From d2b07b24b5d76013cd1b83151ef3d256f61963cc Mon Sep 17 00:00:00 2001 From: jupiterv2 Date: Wed, 17 Jun 2026 17:13:58 +0800 Subject: [PATCH 1/5] feat(driver): vendor shared libs and driver-v3/v4 chain config into core Preparation for moving the streaming driver (driver v3/v4, sentio's driver/controller) into sentio-core. The controller depends on a handful of sentio-local leaf libraries that have no sentio-only dependencies, so they are vendored into sentio-core here (the sentio side will repoint its imports and delete the local copies in a paired change): - common/compress, common/window, common/contract, common/jsonutils, driver/errors It also introduces driver/controller/config, holding the driver v3/v4 per-chain config (ConfigV2 + LoadChainsConfigV2 + NewCustomizedChainConfigV2). The legacy driver v2 config (chain.Config) stays in the sentio repository. common/clickhousemanagerext is intentionally NOT vendored (it must not be open-sourced): the controller's dependency on it will be inverted on the sentio side by injecting the timeseries/entity chx.Controller into the startup config, built by sentio/driver/cmd. Co-Authored-By: Claude Opus 4.8 (1M context) --- common/compress/BUILD.bazel | 16 +++ common/compress/compress.go | 71 +++++++++++++ common/compress/compress_test.go | 32 ++++++ common/contract/BUILD.bazel | 19 ++++ common/contract/contract.go | 148 +++++++++++++++++++++++++++ common/contract/contract_test.go | 83 +++++++++++++++ common/jsonutils/BUILD.bazel | 15 +++ common/jsonutils/jsonutils.go | 74 ++++++++++++++ common/jsonutils/jsonutils_test.go | 103 +++++++++++++++++++ common/window/BUILD.bazel | 16 +++ common/window/window.go | 70 +++++++++++++ common/window/window_test.go | 40 ++++++++ driver/controller/config/BUILD.bazel | 13 +++ driver/controller/config/config.go | 65 ++++++++++++ driver/errors/BUILD.bazel | 9 ++ driver/errors/errors.go | 57 +++++++++++ 16 files changed, 831 insertions(+) create mode 100644 common/compress/BUILD.bazel create mode 100644 common/compress/compress.go create mode 100644 common/compress/compress_test.go create mode 100644 common/contract/BUILD.bazel create mode 100644 common/contract/contract.go create mode 100644 common/contract/contract_test.go create mode 100644 common/jsonutils/BUILD.bazel create mode 100644 common/jsonutils/jsonutils.go create mode 100644 common/jsonutils/jsonutils_test.go create mode 100644 common/window/BUILD.bazel create mode 100644 common/window/window.go create mode 100644 common/window/window_test.go create mode 100644 driver/controller/config/BUILD.bazel create mode 100644 driver/controller/config/config.go create mode 100644 driver/errors/BUILD.bazel create mode 100644 driver/errors/errors.go diff --git a/common/compress/BUILD.bazel b/common/compress/BUILD.bazel new file mode 100644 index 0000000..a908aa7 --- /dev/null +++ b/common/compress/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "compress", + srcs = ["compress.go"], + importpath = "sentioxyz/sentio-core/common/compress", + visibility = ["//visibility:public"], + deps = ["@com_github_pkg_errors//:errors"], +) + +go_test( + name = "compress_test", + srcs = ["compress_test.go"], + embed = [":compress"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/compress/compress.go b/common/compress/compress.go new file mode 100644 index 0000000..795cb07 --- /dev/null +++ b/common/compress/compress.go @@ -0,0 +1,71 @@ +package compress + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "github.com/pkg/errors" + "io" +) + +type compressPayload struct { + CompressMethod string `json:"compress_method,omitempty"` + Data []byte `json:"data,omitempty"` +} + +const ( + compressMethodGZIP = "gzip" +) + +func Load(raw []byte, d any) (err error) { + if len(raw) == 0 { + return nil + } + var payload compressPayload + _ = json.Unmarshal(raw, &payload) + var r io.Reader + switch payload.CompressMethod { + case compressMethodGZIP: + r, err = gzip.NewReader(bytes.NewReader(payload.Data)) + if err != nil { + return errors.Wrapf(err, "try to load as compressed payload failed") + } + default: + r = bytes.NewReader(raw) + } + return json.NewDecoder(r).Decode(d) +} + +func Dump(d any) ([]byte, error) { + return dump(d, compressMethodGZIP) +} + +func dump(d any, compressMethod string) ([]byte, error) { + // prepare WriteCloser by compressMethod + var buf bytes.Buffer + var w io.WriteCloser + switch compressMethod { + case compressMethodGZIP: + w = gzip.NewWriter(&buf) + default: + return nil, errors.Errorf("compress method %s not supported", compressMethod) + } + // json marshal and do compress + err := json.NewEncoder(w).Encode(d) + if err == nil { + err = w.Close() + } + if err != nil { + return nil, errors.Wrapf(err, "dump with compress method %s failed", compressMethod) + } + // build payload and json marshal + var raw []byte + raw, err = json.Marshal(compressPayload{ + CompressMethod: compressMethod, + Data: buf.Bytes(), + }) + if err != nil { + err = errors.Wrapf(err, "dump with compress method %s failed", compressMethod) + } + return raw, err +} diff --git a/common/compress/compress_test.go b/common/compress/compress_test.go new file mode 100644 index 0000000..be88074 --- /dev/null +++ b/common/compress/compress_test.go @@ -0,0 +1,32 @@ +package compress + +import ( + "encoding/json" + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_loadAndDump(t *testing.T) { + const count = 100000 + origin := make([]string, count) + for i := 0; i < count; i++ { + origin[i] = fmt.Sprintf("%08d", i) + } + + d0, _ := json.Marshal(origin) + t.Logf("origin len: %d", len(d0)) + d, err := Dump(origin) + assert.NoError(t, err) + + t.Logf("dump len: %d", len(d)) + t.Logf("dump prefix: %s", string(d[:100])) + t.Logf("dump suffix: %s", string(d[len(d)-100:])) + + var result []string + assert.NoError(t, Load(d, &result)) + assert.Equal(t, origin, result) + + assert.NoError(t, Load(d0, &result)) + assert.Equal(t, origin, result) +} diff --git a/common/contract/BUILD.bazel b/common/contract/BUILD.bazel new file mode 100644 index 0000000..1f29386 --- /dev/null +++ b/common/contract/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "contract", + srcs = ["contract.go"], + importpath = "sentioxyz/sentio-core/common/contract", + visibility = ["//visibility:public"], + deps = [ + "//common/https", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "contract_test", + srcs = ["contract_test.go"], + embed = [":contract"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/contract/contract.go b/common/contract/contract.go new file mode 100644 index 0000000..c3f82db --- /dev/null +++ b/common/contract/contract.go @@ -0,0 +1,148 @@ +package contract + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/https" +) + +type TokenResult struct { + Decimals *int `json:"decimals"` + Logo *string `json:"logo"` + Name string `json:"name"` + Symbol string `json:"symbol"` +} + +type jsonError struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` +} + +type JsonrpcMessage struct { + Version string `json:"jsonrpc,omitempty"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Error *jsonError `json:"error,omitempty"` + Result *TokenResult `json:"result,omitempty"` +} + +// IsERC20 refers from https://docs.alchemy.com/reference/alchemy-gettokenmetadata +func IsERC20(ctx context.Context, apiEndpoint string, tokenAddress string) (bool, error) { + payload := strings.NewReader( + fmt.Sprintf( + "{\"id\":1,\"jsonrpc\":\"2.0\",\"method\":\"alchemy_getTokenMetadata\",\"params\":[\"%s\"]}", + tokenAddress, + ), + ) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, apiEndpoint, payload) + + req.Header.Add("accept", "application/json") + req.Header.Add("content-type", "application/json") + + res, err := https.DefaultClient.Do(req) + + if err != nil { + return false, errors.WithStack(err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return false, errors.New("HTTP request return" + res.Status) + } + + body, _ := io.ReadAll(res.Body) + //bodyString := string(body) + + var jsonrpcRes JsonrpcMessage + err = json.Unmarshal(body, &jsonrpcRes) + if err != nil { + return false, err + } + + if jsonrpcRes.Result != nil { + if jsonrpcRes.Result.Decimals != nil { + return true, nil + } + } + + return false, nil +} + +// IsERC20New refers from https://docs.moralis.io/reference/gettokenmetadata +func IsERC20New(ctx context.Context, apiKey string, chainID string, tokenAddress string) (bool, error) { + info, err := getERC20Info(ctx, apiKey, chainID, []string{tokenAddress}) + if err != nil { + return false, err + } + return info[0].Symbol != "" && info[0].Decimals != "", nil +} + +type ERC20Token struct { + Address string `json:"address"` + Symbol string `json:"symbol"` + Name string `json:"name"` + Decimals string `json:"decimals"` + Logo string `json:"logo,omitempty"` + LogoHash string `json:"logo_hash,omitempty"` + Thumbnail string `json:"thumbnail,omitempty"` + BlockNumber string `json:"block_number,omitempty"` + //Validated string `json:"validated,omitempty"` +} + +// https://docs.moralis.io/reference/gettokenmetadata +func getERC20Info(ctx context.Context, apiKey string, chainID string, tokenAddress []string) ([]ERC20Token, error) { + var chain string + switch chainID { + case "1": + chain = "eth" + case "5": + chain = "goerli" + case "56": + chain = "bsc" + // TDDO add more mappings + default: + return nil, errors.New("chainID not supported") + } + + var addresses = "" + for _, address := range tokenAddress { + addresses += "&addresses=" + address + } + + url := "https://deep-index.moralis.io/api/v2/erc20/metadata?&chain=" + chain + addresses + + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + + req.Header.Add("accept", "application/json") + req.Header.Add("X-API-Key", apiKey) + + res, err := https.DefaultClient.Do(req) + + if err != nil { + return nil, errors.WithStack(err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, errors.New("HTTP request return" + res.Status) + } + + body, _ := io.ReadAll(res.Body) + + var tokenInfo []ERC20Token + err = json.Unmarshal(body, &tokenInfo) + if err != nil { + return nil, errors.WithStack(err) + } + + return tokenInfo, nil +} diff --git a/common/contract/contract_test.go b/common/contract/contract_test.go new file mode 100644 index 0000000..389b043 --- /dev/null +++ b/common/contract/contract_test.go @@ -0,0 +1,83 @@ +package contract + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// FIXME runs into 403 regularly +//func TestIsERC20(t *testing.T) { +// endpoint := "https://eth-mainnet.g.alchemy.com/v2/KLPDGUUQGKmScCbSdPd0iUNO_JufXdg9" +// +// ctx := context.Background() +// +// var res bool +// var err error +// +// //// abtc +// //res, err = IsERC20(ctx, endpoint, "0xC2fcab14Ec1F2dFA82a23C639c4770345085a50F") +// //assert.NoError(t, err) +// //assert.False(t, res) +// +// // weth address +// res, err = IsERC20(ctx, endpoint, "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2") +// assert.NoError(t, err) +// assert.True(t, res) +// +// // invalid address +// res, err = IsERC20(ctx, endpoint, "0x0000000000000000000000000000000000000000") +// assert.NoError(t, err) +// assert.False(t, res) +// +// // token address but does not implement ERC20 +// res, err = IsERC20(ctx, endpoint, "0xa6794DEc66Df7d8B69752956df1b28cA93f77CD7") +// assert.NoError(t, err) +// assert.False(t, res) +// +// // USDC +// res, err = IsERC20(ctx, endpoint, "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48") +// assert.NoError(t, err) +// assert.True(t, res) +// +// res, err = IsERC20(ctx, endpoint, "0x6B89B97169a797d94F057F4a0B01E2cA303155e4") +// assert.NoError(t, err) +// assert.True(t, res) +//} + +func TestIsERC20New(t *testing.T) { + //endpoint := "https://eth-mainnet.g.alchemy.com/v2/z1Q-YhcYg60C5sOQPUzsMFqiDJSvqbsK" + endpoint := "oGDQZsjYX3IdnfkenzRf0K5NA7Lsy0NljUFVKFp0nGDhwajEq5ltjHU7aFp3V8lG" + + ctx := context.Background() + + var res bool + var err error + + //// abtc + //res, err = IsERC20New(ctx, endpoint, "1", "0xC2fcab14Ec1F2dFA82a23C639c4770345085a50F") + //assert.NoError(t, err) + //assert.False(t, res) + + // weth address + res, err = IsERC20New(ctx, endpoint, "1", "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2") + assert.NoError(t, err) + assert.True(t, res) + + // invalid address + // res, err = IsERC20New(ctx, endpoint, "1", "0x0000000000000000000000000000000000000000") + // assert.NoError(t, err) + // assert.False(t, res) + + // token address but does not implement ERC20 + _, err = IsERC20New(ctx, endpoint, "1", "0xa6794DEc66Df7d8B69752956df1b28cA93f77CD7") + assert.NoError(t, err) + // Uncomment this line when the issue is fixed. + // assert.False(t, res) + + // USDC + res, err = IsERC20New(ctx, endpoint, "1", "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48") + assert.NoError(t, err) + assert.True(t, res) +} diff --git a/common/jsonutils/BUILD.bazel b/common/jsonutils/BUILD.bazel new file mode 100644 index 0000000..1990541 --- /dev/null +++ b/common/jsonutils/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "jsonutils", + srcs = ["jsonutils.go"], + importpath = "sentioxyz/sentio-core/common/jsonutils", + visibility = ["//visibility:public"], +) + +go_test( + name = "jsonutils_test", + srcs = ["jsonutils_test.go"], + embed = [":jsonutils"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/jsonutils/jsonutils.go b/common/jsonutils/jsonutils.go new file mode 100644 index 0000000..6745700 --- /dev/null +++ b/common/jsonutils/jsonutils.go @@ -0,0 +1,74 @@ +package jsonutils + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" +) + +type Tracker func(path string, or, pa any) + +func Patch(origin, patch []byte, tracker Tracker) (final []byte, err error) { + var o, p map[string]any + var decoder *json.Decoder + decoder = json.NewDecoder(bytes.NewReader(origin)) + decoder.UseNumber() + if err = decoder.Decode(&o); err != nil { + return + } + decoder = json.NewDecoder(bytes.NewReader(patch)) + decoder.UseNumber() + if err = decoder.Decode(&p); err != nil { + return + } + if err = patchObject("", o, p, tracker); err != nil { + return + } + return json.Marshal(o) +} + +const wildcard = "*" + +func patchObject(path string, origin, patch map[string]any, tracker Tracker) error { + for key, pv := range patch { + if key == wildcard { + for key, ov := range origin { + keyPath := path + "." + key + if reflect.ValueOf(ov).Type().String() != reflect.ValueOf(pv).Type().String() { + continue + } else if reflect.ValueOf(ov).Kind() == reflect.Map { + if err := patchObject(keyPath, ov.(map[string]any), pv.(map[string]any), tracker); err != nil { + return err + } + continue + } + if tracker != nil { + tracker(keyPath, ov, pv) + } + origin[key] = pv + } + continue + } + + ov, has := origin[key] + keyPath := path + "." + key + + if has { + if reflect.ValueOf(ov).Type().String() != reflect.ValueOf(pv).Type().String() { + return fmt.Errorf("cannot patch %s/%T by %T", keyPath, ov, pv) + } else if reflect.ValueOf(ov).Kind() == reflect.Map { + if err := patchObject(keyPath, ov.(map[string]any), pv.(map[string]any), tracker); err != nil { + return err + } + continue + } + } + + if tracker != nil { + tracker(keyPath, ov, pv) + } + origin[key] = pv + } + return nil +} diff --git a/common/jsonutils/jsonutils_test.go b/common/jsonutils/jsonutils_test.go new file mode 100644 index 0000000..9e47740 --- /dev/null +++ b/common/jsonutils/jsonutils_test.go @@ -0,0 +1,103 @@ +package jsonutils + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPatch_Succeed(t *testing.T) { + testcases := [][]string{ + { + `{}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + }, + { + `{"a":123}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + }, + { + `{"a":456,"e":"xxx"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","e":"xxx"}`, + }, + { + `{"a":456,"e":"xxx","f":{"g":123,"h":"yyy"}}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","f":{"h":"zzz","i":true}}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","e":"xxx","f":{"g":123,"h":"zzz","i":true}}`, + }, + } + for i, testcase := range testcases { + r, err := Patch([]byte(testcase[0]), []byte(testcase[1]), func(path string, or, pa any) { + fmt.Printf("#%d %s: %v => %v\n", i, path, or, pa) + }) + assert.NoError(t, err) + assert.Equal(t, testcase[2], string(r), fmt.Sprintf("testcase #%d %v", i, testcase)) + } +} + +func TestPatch_WildcardSucceed(t *testing.T) { + testcases := [][]string{ + { + `{}`, + `{"*":1}`, + `{}`, + }, + { + `{"a":123,"b":456,"c":"ccc"}`, + `{"*":1}`, + `{"a":1,"b":1,"c":"ccc"}`, + }, + { + `{"a":{"x":1},"b":{"x":2},"c":3}`, + `{"*":{"x":4}}`, + `{"a":{"x":4},"b":{"x":4},"c":3}`, + }, + { + `{"a":{"x":1},"b":{"y":2},"c":3}`, + `{"*":{"x":4}}`, + `{"a":{"x":4},"b":{"x":4,"y":2},"c":3}`, + }, + } + for i, testcase := range testcases { + r, err := Patch([]byte(testcase[0]), []byte(testcase[1]), func(path string, or, pa any) { + fmt.Printf("#%d %s: %v => %v\n", i, path, or, pa) + }) + assert.NoError(t, err) + assert.Equal(t, testcase[2], string(r), fmt.Sprintf("testcase #%d %v", i, testcase)) + } +} + +func TestPatch_TypeMismatch(t *testing.T) { + var err error + + _, err = Patch([]byte(`{"a":true}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/bool by string") + + _, err = Patch([]byte(`{"a":123}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/json.Number by string") + + _, err = Patch([]byte(`{"a":{"b":123}}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/map[string]interface {} by string") + + _, err = Patch([]byte(`{"a":123}`), []byte(`{"a":{"b":true}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/json.Number by map[string]interface {}") +} + +func TestPatch_WildcardTypeMismatch(t *testing.T) { + var err error + + _, err = Patch([]byte(`{"a":{"x":true}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/bool by string") + + _, err = Patch([]byte(`{"a":{"x":123}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/json.Number by string") + + _, err = Patch([]byte(`{"a":{"x":{"y":123}}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/map[string]interface {} by string") + + _, err = Patch([]byte(`{"a":{"x":123}}`), []byte(`{"*":{"x":{"y":123}}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/json.Number by map[string]interface {}") +} diff --git a/common/window/BUILD.bazel b/common/window/BUILD.bazel new file mode 100644 index 0000000..e0b2ef5 --- /dev/null +++ b/common/window/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "window", + srcs = ["window.go"], + importpath = "sentioxyz/sentio-core/common/window", + visibility = ["//visibility:public"], + deps = ["@org_golang_x_exp//constraints"], +) + +go_test( + name = "window_test", + srcs = ["window_test.go"], + embed = [":window"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/window/window.go b/common/window/window.go new file mode 100644 index 0000000..d9a7e80 --- /dev/null +++ b/common/window/window.go @@ -0,0 +1,70 @@ +package window + +import ( + "context" + "golang.org/x/exp/constraints" + "time" +) + +// FindFirstStartPoint find the first x in [start,end] that timeWinGetter(x-1) < timeWinGetter(x) +func FindFirstStartPoint[N constraints.Signed]( + ctx context.Context, + start, end N, + winGetter func(ctx context.Context, n N) (time.Time, error), +) (*N, error) { + if start > end { + return nil, nil + } + + var t time.Time + p, err := winGetter(ctx, start-1) + if err != nil { + return nil, err + } + + t, err = winGetter(ctx, end) + if err != nil { + return nil, err + } + + if p.Equal(t) { + // all x in [start,end] that timeWinGetter(start-1) == timeWinGetter(x), so no result + return nil, nil + } + + for start < end { + mid := (start + end) / 2 + t, err = winGetter(ctx, mid) + if err != nil { + return nil, err + } + if t.Equal(p) { + start = mid + 1 + } else { + end = mid + } + } + return &start, nil +} + +// FindStartPoints find first limit points in [start,end] that timeWinGetter(x-1) < timeWinGetter(x) +func FindStartPoints[N constraints.Signed]( + ctx context.Context, + start, end N, + limit int, + winGetter func(ctx context.Context, n N) (time.Time, error), +) ([]N, error) { + var result []N + for i := 0; limit <= 0 || i < limit; i++ { + x, err := FindFirstStartPoint(ctx, start, end, winGetter) + if err != nil { + return nil, err + } + if x == nil { + return result, nil + } + result = append(result, *x) + start = *x + 1 + } + return result, nil +} diff --git a/common/window/window_test.go b/common/window/window_test.go new file mode 100644 index 0000000..48dae6c --- /dev/null +++ b/common/window/window_test.go @@ -0,0 +1,40 @@ +package window + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func Test_findFirstStartPoint(t *testing.T) { + winGetter := func(ctx context.Context, n int) (time.Time, error) { + return time.Unix(int64(n/100), 0), nil + } + + var r *int + for s := 1; s <= 100; s++ { + for e := 100; e < 199; e++ { + r, _ = FindFirstStartPoint[int](context.Background(), s, e, winGetter) + assert.Equal(t, 100, *r) + } + } + for s := 101; s <= 199; s++ { + for e := s; e < 199; e++ { + r, _ = FindFirstStartPoint[int](context.Background(), s, e, winGetter) + assert.Nil(t, r) + } + } + + var rs []int + rs, _ = FindStartPoints[int](context.Background(), 1, 99, 3, winGetter) + assert.Nil(t, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 1, winGetter) + assert.Equal(t, []int{100}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 2, winGetter) + assert.Equal(t, []int{100, 200}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 3, winGetter) + assert.Equal(t, []int{100, 200}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, -1, winGetter) + assert.Equal(t, []int{100, 200}, rs) +} diff --git a/driver/controller/config/BUILD.bazel b/driver/controller/config/BUILD.bazel new file mode 100644 index 0000000..2d499cf --- /dev/null +++ b/driver/controller/config/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "config", + srcs = ["config.go"], + importpath = "sentioxyz/sentio-core/driver/controller/config", + visibility = ["//visibility:public"], + deps = [ + "//common/jsonutils", + "//common/log", + "//service/processor/models", + ], +) diff --git a/driver/controller/config/config.go b/driver/controller/config/config.go new file mode 100644 index 0000000..b209804 --- /dev/null +++ b/driver/controller/config/config.go @@ -0,0 +1,65 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "sentioxyz/sentio-core/common/jsonutils" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/service/processor/models" +) + +// ConfigV2 is the per-chain configuration consumed by the streaming +// (driver v3/v4) controller. The legacy driver v2 configuration (chain.Config) +// stays in the sentio repository. +type ConfigV2 struct { + ChainID string + Endpoint string + StartBlockOverride int64 + ProcessingDelayBlocks uint64 + KeepSuiEventTypePackage bool + SkipStartBlockValidation bool + IsCustomizedEndpoint bool +} + +// PatchChainsConfigEnv is the env var that, when set, carries a JSON patch +// applied on top of the chains config file before it is parsed. +const PatchChainsConfigEnv = "CHAIN_CONFIG_JSON_PATCH" + +func LoadChainsConfigV2( + path string, + patchEnv string, + networkOverrides []models.NetworkOverride, +) (map[string]*ConfigV2, error) { + var file []byte + var err error + if file, err = os.ReadFile(path); err != nil { + return nil, err + } + if patch := strings.TrimSpace(os.Getenv(patchEnv)); patchEnv != "" && patch != "" { + file, err = jsonutils.Patch(file, []byte(patch), func(path string, or, pa any) { + log.Infof("patch chains config %s %v => %v", path, or, pa) + }) + if err != nil { + return nil, fmt.Errorf("patch chain config from env %s failed: %w", patchEnv, err) + } + } + var chainsConfig map[string]*ConfigV2 + if err = json.Unmarshal(file, &chainsConfig); err != nil { + return nil, err + } + for _, no := range networkOverrides { + chainsConfig[no.Chain] = &ConfigV2{ChainID: no.Chain, Endpoint: no.Host, IsCustomizedEndpoint: true} + log.Infof("will use customized host %q in chain %s", no.Host, no.Chain) + } + return chainsConfig, nil +} + +func NewCustomizedChainConfigV2(chainID, endpoint string) *ConfigV2 { + return &ConfigV2{ + ChainID: chainID, + Endpoint: endpoint, + } +} diff --git a/driver/errors/BUILD.bazel b/driver/errors/BUILD.bazel new file mode 100644 index 0000000..8e4b8ee --- /dev/null +++ b/driver/errors/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "errors", + srcs = ["errors.go"], + importpath = "sentioxyz/sentio-core/driver/errors", + visibility = ["//visibility:public"], + deps = ["//common/log"], +) diff --git a/driver/errors/errors.go b/driver/errors/errors.go new file mode 100644 index 0000000..4d6f7db --- /dev/null +++ b/driver/errors/errors.go @@ -0,0 +1,57 @@ +package drivererrors + +import ( + "errors" + "os" + "sentioxyz/sentio-core/common/log" +) + +var ( + ErrConfigUpdate = errors.New("config update error") + ErrCleanUp = errors.New("clean up error") + ErrNeedCheckLatest = errors.New("need check latest") + ErrProcessor = errors.New("user processor error") + ErrProcessorBadUsage = errors.New("user processor bad usage error") + ErrOverQuota = errors.New("over quota error of units") + ErrNeedRestart = errors.New("error need restart") +) + +type ExitCode int + +const ( + AlwaysRetry ExitCode = 1 + LimitedRetry ExitCode = 10 + NeverRetry ExitCode = 11 + OverQuota ExitCode = 12 + RetryAfterOneHour ExitCode = 20 + RetryNextDay ExitCode = 21 + RetryNextMonth ExitCode = 22 +) + +func halt(err error, exitCode ExitCode) { + log.Errore(err) + os.Exit(int(exitCode)) +} + +func Halt(err error) { + switch { + case errors.Is(err, ErrProcessor): + halt(err, LimitedRetry) + case errors.Is(err, ErrProcessorBadUsage): + halt(err, NeverRetry) + case errors.Is(err, ErrOverQuota): + halt(err, OverQuota) + default: + halt(err, AlwaysRetry) + } +} + +func IsProcessorError(err error) bool { + if err == nil { + return false + } + return errors.Is(err, ErrProcessor) || + errors.Is(err, ErrProcessorBadUsage) || + errors.Is(err, ErrOverQuota) || + errors.Is(err, ErrNeedRestart) +} From 2791d69e5c55150efffdb3e3c11e66e31c74901e Mon Sep 17 00:00:00 2001 From: jupiterv2 Date: Thu, 18 Jun 2026 16:05:21 +0800 Subject: [PATCH 2/5] refactor(driver): keep only ExitCode in core driver/errors The vendored driver/errors should expose just the ExitCode type and its constants, which the streaming driver (driver v3/v4) uses. The ErrXXX sentinel errors and the Halt / IsProcessorError helpers are driver v2 specific and stay in the sentio repository. Co-Authored-By: Claude Opus 4.8 (1M context) --- driver/errors/BUILD.bazel | 1 - driver/errors/errors.go | 48 ++++----------------------------------- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/driver/errors/BUILD.bazel b/driver/errors/BUILD.bazel index 8e4b8ee..b17b427 100644 --- a/driver/errors/BUILD.bazel +++ b/driver/errors/BUILD.bazel @@ -5,5 +5,4 @@ go_library( srcs = ["errors.go"], importpath = "sentioxyz/sentio-core/driver/errors", visibility = ["//visibility:public"], - deps = ["//common/log"], ) diff --git a/driver/errors/errors.go b/driver/errors/errors.go index 4d6f7db..50163a6 100644 --- a/driver/errors/errors.go +++ b/driver/errors/errors.go @@ -1,21 +1,9 @@ package drivererrors -import ( - "errors" - "os" - "sentioxyz/sentio-core/common/log" -) - -var ( - ErrConfigUpdate = errors.New("config update error") - ErrCleanUp = errors.New("clean up error") - ErrNeedCheckLatest = errors.New("need check latest") - ErrProcessor = errors.New("user processor error") - ErrProcessorBadUsage = errors.New("user processor bad usage error") - ErrOverQuota = errors.New("over quota error of units") - ErrNeedRestart = errors.New("error need restart") -) - +// ExitCode is the process exit code the streaming driver uses to tell its +// supervisor how to retry. The concrete ErrXXX sentinel errors and the Halt +// helper that map onto these codes are used by driver v2 and stay in the sentio +// repository. type ExitCode int const ( @@ -27,31 +15,3 @@ const ( RetryNextDay ExitCode = 21 RetryNextMonth ExitCode = 22 ) - -func halt(err error, exitCode ExitCode) { - log.Errore(err) - os.Exit(int(exitCode)) -} - -func Halt(err error) { - switch { - case errors.Is(err, ErrProcessor): - halt(err, LimitedRetry) - case errors.Is(err, ErrProcessorBadUsage): - halt(err, NeverRetry) - case errors.Is(err, ErrOverQuota): - halt(err, OverQuota) - default: - halt(err, AlwaysRetry) - } -} - -func IsProcessorError(err error) bool { - if err == nil { - return false - } - return errors.Is(err, ErrProcessor) || - errors.Is(err, ErrProcessorBadUsage) || - errors.Is(err, ErrOverQuota) || - errors.Is(err, ErrNeedRestart) -} From 1e685e717e252509a612786bc5cc91b81b4d1208 Mon Sep 17 00:00:00 2001 From: jupiterv2 Date: Thu, 18 Jun 2026 16:28:55 +0800 Subject: [PATCH 3/5] refactor(driver): rename core driver/errors to driver/exitcode The vendored package holds only the process exit/retry codes the streaming driver uses, so name it driver/exitcode (package exitcode, type Code). The ErrXXX sentinel errors and Halt/IsProcessorError helpers are driver v2 specific and stay in the sentio repository. Co-Authored-By: Claude Opus 4.8 (1M context) --- driver/errors/BUILD.bazel | 8 -------- driver/errors/errors.go | 17 ----------------- driver/exitcode/BUILD.bazel | 8 ++++++++ driver/exitcode/exitcode.go | 17 +++++++++++++++++ 4 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 driver/errors/BUILD.bazel delete mode 100644 driver/errors/errors.go create mode 100644 driver/exitcode/BUILD.bazel create mode 100644 driver/exitcode/exitcode.go diff --git a/driver/errors/BUILD.bazel b/driver/errors/BUILD.bazel deleted file mode 100644 index b17b427..0000000 --- a/driver/errors/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "errors", - srcs = ["errors.go"], - importpath = "sentioxyz/sentio-core/driver/errors", - visibility = ["//visibility:public"], -) diff --git a/driver/errors/errors.go b/driver/errors/errors.go deleted file mode 100644 index 50163a6..0000000 --- a/driver/errors/errors.go +++ /dev/null @@ -1,17 +0,0 @@ -package drivererrors - -// ExitCode is the process exit code the streaming driver uses to tell its -// supervisor how to retry. The concrete ErrXXX sentinel errors and the Halt -// helper that map onto these codes are used by driver v2 and stay in the sentio -// repository. -type ExitCode int - -const ( - AlwaysRetry ExitCode = 1 - LimitedRetry ExitCode = 10 - NeverRetry ExitCode = 11 - OverQuota ExitCode = 12 - RetryAfterOneHour ExitCode = 20 - RetryNextDay ExitCode = 21 - RetryNextMonth ExitCode = 22 -) diff --git a/driver/exitcode/BUILD.bazel b/driver/exitcode/BUILD.bazel new file mode 100644 index 0000000..63efdb5 --- /dev/null +++ b/driver/exitcode/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "exitcode", + srcs = ["exitcode.go"], + importpath = "sentioxyz/sentio-core/driver/exitcode", + visibility = ["//visibility:public"], +) diff --git a/driver/exitcode/exitcode.go b/driver/exitcode/exitcode.go new file mode 100644 index 0000000..7d874db --- /dev/null +++ b/driver/exitcode/exitcode.go @@ -0,0 +1,17 @@ +package exitcode + +// Code is the process exit code the streaming driver uses to tell its +// supervisor how to retry. The concrete ErrXXX sentinel errors and the Halt +// helper that map onto these codes are used by driver v2 and stay in the sentio +// repository. +type Code int + +const ( + AlwaysRetry Code = 1 + LimitedRetry Code = 10 + NeverRetry Code = 11 + OverQuota Code = 12 + RetryAfterOneHour Code = 20 + RetryNextDay Code = 21 + RetryNextMonth Code = 22 +) From f1408a6720f424966eede8b019161304010cf9a0 Mon Sep 17 00:00:00 2001 From: jupiterv2 Date: Thu, 18 Jun 2026 16:41:00 +0800 Subject: [PATCH 4/5] feat(driver): host the streaming driver controller in sentio-core Move the driver v3/v4 streaming controller (~21k LOC across data / fetcher / standard / subgraph / startup) into sentio-core. Imports are repointed to their sentio-core homes: - chain ConfigV2 (and loaders) -> driver/controller/config - driver exit codes -> driver/exitcode (type Code) - common/{compress,window,contract,gonanoid} and service/common/rpc -> sentio-core The controller's couplings to the driver binary's metric instruments and service clients are inverted behind controller.Notifier and startup.ClickhouseConnector (added earlier); their implementations live in the driver binary, which is not part of this repo. The GCP project for the webhook pubsub topic is injected via startup.Config rather than hardcoded. Adds the cloud.google.com/go/pubsub dependency. The driver-binary side (remove the in-repo controller, repoint its consumers) lands in the paired downstream change. --- MODULE.bazel | 2 + driver/controller/BUILD.bazel | 55 + driver/controller/analyse.go | 167 +++ driver/controller/block_builder.go | 332 ++++++ driver/controller/checkpoint.go | 1001 +++++++++++++++++ driver/controller/checkpoint_test.go | 183 +++ driver/controller/config.go | 31 + driver/controller/data/BUILD.bazel | 44 + driver/controller/data/aptos/BUILD.bazel | 41 + driver/controller/data/aptos/block.go | 128 +++ driver/controller/data/aptos/change.go | 110 ++ driver/controller/data/aptos/client.go | 362 ++++++ driver/controller/data/aptos/resource.go | 50 + driver/controller/data/aptos/transaction.go | 140 +++ .../controller/data/aptos/transaction_test.go | 44 + driver/controller/data/block_cache.go | 84 ++ driver/controller/data/block_cache_test.go | 67 ++ driver/controller/data/contract.go | 53 + driver/controller/data/contract_test.go | 30 + driver/controller/data/errors.go | 46 + driver/controller/data/evm/BUILD.bazel | 47 + driver/controller/data/evm/block.go | 67 ++ driver/controller/data/evm/block_extend.go | 63 ++ driver/controller/data/evm/block_main.go | 127 +++ driver/controller/data/evm/client.go | 584 ++++++++++ driver/controller/data/evm/log.go | 238 ++++ driver/controller/data/evm/log_test.go | 90 ++ driver/controller/data/evm/trace.go | 200 ++++ driver/controller/data/fuel/BUILD.bazel | 26 + driver/controller/data/fuel/block.go | 27 + driver/controller/data/fuel/block_main.go | 113 ++ driver/controller/data/fuel/client.go | 196 ++++ driver/controller/data/fuel/transaction.go | 94 ++ driver/controller/data/interval.go | 200 ++++ driver/controller/data/sol/BUILD.bazel | 40 + driver/controller/data/sol/block.go | 32 + driver/controller/data/sol/block_main.go | 249 ++++ driver/controller/data/sol/client.go | 385 +++++++ driver/controller/data/sol/client_test.go | 84 ++ driver/controller/data/sol/native_client.go | 542 +++++++++ driver/controller/data/sol/transaction.go | 108 ++ driver/controller/data/statistics.go | 164 +++ driver/controller/data/statistics_test.go | 31 + driver/controller/data/subscribe.go | 114 ++ driver/controller/data/sui/BUILD.bazel | 37 + driver/controller/data/sui/block.go | 198 ++++ driver/controller/data/sui/block_test.go | 48 + driver/controller/data/sui/client.go | 496 ++++++++ driver/controller/data/sui/grpc/BUILD.bazel | 20 + driver/controller/data/sui/grpc/block.go | 169 +++ .../controller/data/sui/grpc/object_change.go | 53 + .../controller/data/sui/grpc/transaction.go | 54 + driver/controller/data/sui/object_change.go | 89 ++ driver/controller/data/sui/transaction.go | 97 ++ driver/controller/entity.go | 125 ++ driver/controller/errors.go | 152 +++ driver/controller/errors_test.go | 87 ++ driver/controller/fetcher/BUILD.bazel | 40 + driver/controller/fetcher/fetcher.go | 385 +++++++ driver/controller/fetcher/fetcher_test.go | 261 +++++ driver/controller/fetcher/merge.go | 125 ++ driver/controller/fetcher/retry.go | 15 + driver/controller/fetcher/stat.go | 87 ++ driver/controller/fetcher/transfer.go | 418 +++++++ driver/controller/fetcher/transfer_test.go | 183 +++ driver/controller/fetcher/utils.go | 21 + driver/controller/handler.go | 102 ++ driver/controller/main.go | 354 ++++++ driver/controller/main_test.go | 659 +++++++++++ driver/controller/notifier.go | 102 ++ driver/controller/quota.go | 60 + driver/controller/range.go | 582 ++++++++++ driver/controller/range_test.go | 552 +++++++++ driver/controller/standard/BUILD.bazel | 50 + driver/controller/standard/aptos/BUILD.bazel | 32 + .../controller/standard/aptos/block_data.go | 81 ++ driver/controller/standard/aptos/handler.go | 431 +++++++ .../standard/aptos/handler_change.go | 53 + .../standard/aptos/handler_event.go | 65 ++ .../standard/aptos/handler_function.go | 50 + .../standard/aptos/handler_interval.go | 78 ++ driver/controller/standard/binding_data.go | 69 ++ driver/controller/standard/config.go | 16 + driver/controller/standard/convert.go | 140 +++ driver/controller/standard/convert_test.go | 265 +++++ driver/controller/standard/evm/BUILD.bazel | 30 + driver/controller/standard/evm/block_data.go | 93 ++ driver/controller/standard/evm/handler.go | 313 ++++++ .../standard/evm/handler_interval.go | 132 +++ driver/controller/standard/evm/handler_log.go | 158 +++ .../controller/standard/evm/handler_trace.go | 114 ++ .../standard/evm/handler_transaction.go | 71 ++ driver/controller/standard/fuel/BUILD.bazel | 32 + driver/controller/standard/fuel/block_data.go | 78 ++ driver/controller/standard/fuel/handler.go | 277 +++++ .../standard/fuel/handler_interval.go | 53 + .../standard/fuel/handler_receipt.go | 73 ++ .../standard/fuel/handler_transaction.go | 55 + driver/controller/standard/handler.go | 335 ++++++ driver/controller/standard/helper.go | 29 + driver/controller/standard/interval.go | 34 + driver/controller/standard/sol/BUILD.bazel | 30 + driver/controller/standard/sol/block_data.go | 91 ++ driver/controller/standard/sol/handler.go | 205 ++++ .../standard/sol/handler_instruction.go | 133 +++ .../standard/sol/handler_interval.go | 61 + driver/controller/standard/sui/BUILD.bazel | 44 + driver/controller/standard/sui/agents.go | 291 +++++ driver/controller/standard/sui/block_data.go | 87 ++ .../controller/standard/sui/grpc/BUILD.bazel | 56 + .../standard/sui/grpc/block_data.go | 92 ++ .../controller/standard/sui/grpc/handler.go | 238 ++++ .../standard/sui/grpc/handler_change.go | 47 + .../standard/sui/grpc/handler_event.go | 64 ++ .../standard/sui/grpc/handler_function.go | 45 + .../standard/sui/grpc/handler_interval.go | 297 +++++ .../standard/sui/grpc/handler_test.go | 127 +++ driver/controller/standard/sui/handler.go | 225 ++++ .../controller/standard/sui/handler_change.go | 69 ++ .../controller/standard/sui/handler_event.go | 73 ++ .../standard/sui/handler_function.go | 58 + .../standard/sui/handler_interval.go | 449 ++++++++ .../standard/sui/handler_interval_test.go | 50 + driver/controller/standard/task.go | 658 +++++++++++ driver/controller/startup/BUILD.bazel | 89 ++ driver/controller/startup/checkpoint.go | 191 ++++ driver/controller/startup/entity.go | 155 +++ driver/controller/startup/quota.go | 128 +++ driver/controller/startup/standard.go | 368 ++++++ driver/controller/startup/startup.go | 632 +++++++++++ driver/controller/startup/startup_test.go | 43 + driver/controller/startup/subgraph.go | 133 +++ driver/controller/startup/timeseries.go | 138 +++ driver/controller/startup/utils.go | 48 + driver/controller/startup/webhook.go | 176 +++ driver/controller/subgraph/BUILD.bazel | 44 + driver/controller/subgraph/binding_data.go | 28 + driver/controller/subgraph/block_data.go | 325 ++++++ driver/controller/subgraph/handler.go | 478 ++++++++ driver/controller/subgraph/handler_block.go | 61 + driver/controller/subgraph/handler_call.go | 58 + driver/controller/subgraph/handler_event.go | 71 ++ driver/controller/subgraph/instance.go | 787 +++++++++++++ driver/controller/subgraph/task.go | 93 ++ driver/controller/timeseries.go | 50 + driver/controller/webhook.go | 55 + go.mod | 4 +- go.sum | 69 ++ 148 files changed, 23282 insertions(+), 1 deletion(-) create mode 100644 driver/controller/BUILD.bazel create mode 100644 driver/controller/analyse.go create mode 100644 driver/controller/block_builder.go create mode 100644 driver/controller/checkpoint.go create mode 100644 driver/controller/checkpoint_test.go create mode 100644 driver/controller/config.go create mode 100644 driver/controller/data/BUILD.bazel create mode 100644 driver/controller/data/aptos/BUILD.bazel create mode 100644 driver/controller/data/aptos/block.go create mode 100644 driver/controller/data/aptos/change.go create mode 100644 driver/controller/data/aptos/client.go create mode 100644 driver/controller/data/aptos/resource.go create mode 100644 driver/controller/data/aptos/transaction.go create mode 100644 driver/controller/data/aptos/transaction_test.go create mode 100644 driver/controller/data/block_cache.go create mode 100644 driver/controller/data/block_cache_test.go create mode 100644 driver/controller/data/contract.go create mode 100644 driver/controller/data/contract_test.go create mode 100644 driver/controller/data/errors.go create mode 100644 driver/controller/data/evm/BUILD.bazel create mode 100644 driver/controller/data/evm/block.go create mode 100644 driver/controller/data/evm/block_extend.go create mode 100644 driver/controller/data/evm/block_main.go create mode 100644 driver/controller/data/evm/client.go create mode 100644 driver/controller/data/evm/log.go create mode 100644 driver/controller/data/evm/log_test.go create mode 100644 driver/controller/data/evm/trace.go create mode 100644 driver/controller/data/fuel/BUILD.bazel create mode 100644 driver/controller/data/fuel/block.go create mode 100644 driver/controller/data/fuel/block_main.go create mode 100644 driver/controller/data/fuel/client.go create mode 100644 driver/controller/data/fuel/transaction.go create mode 100644 driver/controller/data/interval.go create mode 100644 driver/controller/data/sol/BUILD.bazel create mode 100644 driver/controller/data/sol/block.go create mode 100644 driver/controller/data/sol/block_main.go create mode 100644 driver/controller/data/sol/client.go create mode 100644 driver/controller/data/sol/client_test.go create mode 100644 driver/controller/data/sol/native_client.go create mode 100644 driver/controller/data/sol/transaction.go create mode 100644 driver/controller/data/statistics.go create mode 100644 driver/controller/data/statistics_test.go create mode 100644 driver/controller/data/subscribe.go create mode 100644 driver/controller/data/sui/BUILD.bazel create mode 100644 driver/controller/data/sui/block.go create mode 100644 driver/controller/data/sui/block_test.go create mode 100644 driver/controller/data/sui/client.go create mode 100644 driver/controller/data/sui/grpc/BUILD.bazel create mode 100644 driver/controller/data/sui/grpc/block.go create mode 100644 driver/controller/data/sui/grpc/object_change.go create mode 100644 driver/controller/data/sui/grpc/transaction.go create mode 100644 driver/controller/data/sui/object_change.go create mode 100644 driver/controller/data/sui/transaction.go create mode 100644 driver/controller/entity.go create mode 100644 driver/controller/errors.go create mode 100644 driver/controller/errors_test.go create mode 100644 driver/controller/fetcher/BUILD.bazel create mode 100644 driver/controller/fetcher/fetcher.go create mode 100644 driver/controller/fetcher/fetcher_test.go create mode 100644 driver/controller/fetcher/merge.go create mode 100644 driver/controller/fetcher/retry.go create mode 100644 driver/controller/fetcher/stat.go create mode 100644 driver/controller/fetcher/transfer.go create mode 100644 driver/controller/fetcher/transfer_test.go create mode 100644 driver/controller/fetcher/utils.go create mode 100644 driver/controller/handler.go create mode 100644 driver/controller/main.go create mode 100644 driver/controller/main_test.go create mode 100644 driver/controller/notifier.go create mode 100644 driver/controller/quota.go create mode 100644 driver/controller/range.go create mode 100644 driver/controller/range_test.go create mode 100644 driver/controller/standard/BUILD.bazel create mode 100644 driver/controller/standard/aptos/BUILD.bazel create mode 100644 driver/controller/standard/aptos/block_data.go create mode 100644 driver/controller/standard/aptos/handler.go create mode 100644 driver/controller/standard/aptos/handler_change.go create mode 100644 driver/controller/standard/aptos/handler_event.go create mode 100644 driver/controller/standard/aptos/handler_function.go create mode 100644 driver/controller/standard/aptos/handler_interval.go create mode 100644 driver/controller/standard/binding_data.go create mode 100644 driver/controller/standard/config.go create mode 100644 driver/controller/standard/convert.go create mode 100644 driver/controller/standard/convert_test.go create mode 100644 driver/controller/standard/evm/BUILD.bazel create mode 100644 driver/controller/standard/evm/block_data.go create mode 100644 driver/controller/standard/evm/handler.go create mode 100644 driver/controller/standard/evm/handler_interval.go create mode 100644 driver/controller/standard/evm/handler_log.go create mode 100644 driver/controller/standard/evm/handler_trace.go create mode 100644 driver/controller/standard/evm/handler_transaction.go create mode 100644 driver/controller/standard/fuel/BUILD.bazel create mode 100644 driver/controller/standard/fuel/block_data.go create mode 100644 driver/controller/standard/fuel/handler.go create mode 100644 driver/controller/standard/fuel/handler_interval.go create mode 100644 driver/controller/standard/fuel/handler_receipt.go create mode 100644 driver/controller/standard/fuel/handler_transaction.go create mode 100644 driver/controller/standard/handler.go create mode 100644 driver/controller/standard/helper.go create mode 100644 driver/controller/standard/interval.go create mode 100644 driver/controller/standard/sol/BUILD.bazel create mode 100644 driver/controller/standard/sol/block_data.go create mode 100644 driver/controller/standard/sol/handler.go create mode 100644 driver/controller/standard/sol/handler_instruction.go create mode 100644 driver/controller/standard/sol/handler_interval.go create mode 100644 driver/controller/standard/sui/BUILD.bazel create mode 100644 driver/controller/standard/sui/agents.go create mode 100644 driver/controller/standard/sui/block_data.go create mode 100644 driver/controller/standard/sui/grpc/BUILD.bazel create mode 100644 driver/controller/standard/sui/grpc/block_data.go create mode 100644 driver/controller/standard/sui/grpc/handler.go create mode 100644 driver/controller/standard/sui/grpc/handler_change.go create mode 100644 driver/controller/standard/sui/grpc/handler_event.go create mode 100644 driver/controller/standard/sui/grpc/handler_function.go create mode 100644 driver/controller/standard/sui/grpc/handler_interval.go create mode 100644 driver/controller/standard/sui/grpc/handler_test.go create mode 100644 driver/controller/standard/sui/handler.go create mode 100644 driver/controller/standard/sui/handler_change.go create mode 100644 driver/controller/standard/sui/handler_event.go create mode 100644 driver/controller/standard/sui/handler_function.go create mode 100644 driver/controller/standard/sui/handler_interval.go create mode 100644 driver/controller/standard/sui/handler_interval_test.go create mode 100644 driver/controller/standard/task.go create mode 100644 driver/controller/startup/BUILD.bazel create mode 100644 driver/controller/startup/checkpoint.go create mode 100644 driver/controller/startup/entity.go create mode 100644 driver/controller/startup/quota.go create mode 100644 driver/controller/startup/standard.go create mode 100644 driver/controller/startup/startup.go create mode 100644 driver/controller/startup/startup_test.go create mode 100644 driver/controller/startup/subgraph.go create mode 100644 driver/controller/startup/timeseries.go create mode 100644 driver/controller/startup/utils.go create mode 100644 driver/controller/startup/webhook.go create mode 100644 driver/controller/subgraph/BUILD.bazel create mode 100644 driver/controller/subgraph/binding_data.go create mode 100644 driver/controller/subgraph/block_data.go create mode 100644 driver/controller/subgraph/handler.go create mode 100644 driver/controller/subgraph/handler_block.go create mode 100644 driver/controller/subgraph/handler_call.go create mode 100644 driver/controller/subgraph/handler_event.go create mode 100644 driver/controller/subgraph/instance.go create mode 100644 driver/controller/subgraph/task.go create mode 100644 driver/controller/timeseries.go create mode 100644 driver/controller/webhook.go diff --git a/MODULE.bazel b/MODULE.bazel index 36d50c8..a8655b5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -174,6 +174,7 @@ use_repo( "com_github_vmihailenco_msgpack_v5", "com_github_wasmerio_wasmer_go", "com_google_cloud_go_bigquery", + "com_google_cloud_go_pubsub", "in_gopkg_natefinch_lumberjack_v2", "in_gopkg_yaml_v2", "in_gopkg_yaml_v3", @@ -199,6 +200,7 @@ use_repo( "org_golang_google_protobuf", "org_golang_x_exp", "org_golang_x_net", + "org_golang_x_sync", "org_modernc_mathutil", "org_uber_go_mock", "org_uber_go_zap", diff --git a/driver/controller/BUILD.bazel b/driver/controller/BUILD.bazel new file mode 100644 index 0000000..2581913 --- /dev/null +++ b/driver/controller/BUILD.bazel @@ -0,0 +1,55 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "controller", + srcs = [ + "analyse.go", + "block_builder.go", + "checkpoint.go", + "config.go", + "entity.go", + "errors.go", + "handler.go", + "main.go", + "notifier.go", + "quota.go", + "range.go", + "timeseries.go", + "webhook.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller", + visibility = ["//visibility:public"], + deps = [ + "//common/concurrency", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/sparsify", + "//common/timer", + "//common/timewin", + "//common/utils", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/timeseries", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "controller_test", + srcs = [ + "checkpoint_test.go", + "errors_test.go", + "main_test.go", + "range_test.go", + ], + embed = [":controller"], + deps = [ + "//common/errgroup", + "//common/log", + "//common/utils", + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/analyse.go b/driver/controller/analyse.go new file mode 100644 index 0000000..af6d9a3 --- /dev/null +++ b/driver/controller/analyse.go @@ -0,0 +1,167 @@ +package controller + +import ( + "time" + + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" +) + +type taskStatWindow struct { + startAt time.Time + taskCount map[string]int + taskUsed map[string]time.Duration + fetchWaitUsed time.Duration + taskPreWaitUsed time.Duration + taskPostWaitUsed time.Duration + makeCheckpointUsed time.Duration +} + +func (w *taskStatWindow) GetStartAt() time.Time { + return w.startAt +} + +func (w *taskStatWindow) Merge(a *taskStatWindow) { + for handlerID, v := range a.taskCount { + w.taskCount[handlerID] += v + } + for handlerID, v := range a.taskUsed { + w.taskUsed[handlerID] += v + } + w.fetchWaitUsed += a.fetchWaitUsed + w.taskPreWaitUsed += a.taskPreWaitUsed + w.taskPostWaitUsed += a.taskPostWaitUsed + w.makeCheckpointUsed += a.makeCheckpointUsed +} + +func (w *taskStatWindow) Snapshot(endAt time.Time) any { + taskStat := make(map[string]map[string]any) + for hid, count := range w.taskCount { + used := w.taskUsed[hid] + taskStat[hid] = map[string]any{ + "count": count, + "avgUsed": (w.taskUsed[hid] / time.Duration(count)).String(), + "totalUsed": used.String(), + } + } + return map[string]any{ + "startAt": w.startAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(w.startAt).String(), + "task": taskStat, + "taskTotalUsed": utils.SumMap(w.taskUsed).String(), + // fetchWaitUsed is the time the single producer goroutine blocked inside blockBuilder.Next() + // waiting for the data fetcher. It is NOT covered by taskPreWaitUsed (which only starts after + // Next() returns), so without it a fetch-bound pipeline looks idle on the consumer side. + "fetchWaitUsed": w.fetchWaitUsed.String(), + "taskPreWaitUsed": w.taskPreWaitUsed.String(), + "taskPostWaitUsed": w.taskPostWaitUsed.String(), + "makeCheckpointUsed": w.makeCheckpointUsed.String(), + } +} + +type analyser struct { + *timewin.TimeWindowsManager[*taskStatWindow] +} + +func newAnalyser() analyser { + return analyser{ + TimeWindowsManager: timewin.NewTimeWindowsManager[*taskStatWindow](time.Minute), + } +} + +func (a *analyser) fetchWait(used time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + fetchWaitUsed: used, + }) +} + +func (a *analyser) taskSent(preWait time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + taskPreWaitUsed: preWait, + taskPostWaitUsed: 0, + makeCheckpointUsed: 0, + }) +} + +func (a *analyser) taskComplete(handlerID string, used, postWait time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: map[string]int{handlerID: 1}, + taskUsed: map[string]time.Duration{handlerID: used}, + taskPreWaitUsed: 0, + taskPostWaitUsed: postWait, + makeCheckpointUsed: 0, + }) +} + +func (a *analyser) makeCheckpoint(used time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + taskPreWaitUsed: 0, + taskPostWaitUsed: 0, + makeCheckpointUsed: used, + }) +} + +type checkpointStatWindow struct { + startAt time.Time + checkOverQuotaUsed time.Duration + commitTimeSeriesUsed time.Duration + commitEntityUsed time.Duration + commitWebhookUsed time.Duration + saveUsageUsed time.Duration + saveCheckpointUsed time.Duration + totalBinding uint64 + failedCount int + count int +} + +func (w *checkpointStatWindow) GetStartAt() time.Time { + return w.startAt +} + +func (w *checkpointStatWindow) Merge(a *checkpointStatWindow) { + w.checkOverQuotaUsed += a.checkOverQuotaUsed + w.commitTimeSeriesUsed += a.commitTimeSeriesUsed + w.commitEntityUsed += a.commitEntityUsed + w.commitWebhookUsed += a.commitWebhookUsed + w.saveUsageUsed += a.saveUsageUsed + w.saveCheckpointUsed += a.saveCheckpointUsed + w.totalBinding += a.totalBinding + w.failedCount += a.failedCount + w.count += a.count +} + +func (w *checkpointStatWindow) Snapshot(endAt time.Time) any { + dur := endAt.Sub(w.startAt) + total := w.checkOverQuotaUsed + + w.commitTimeSeriesUsed + + w.commitEntityUsed + + w.commitWebhookUsed + + w.saveUsageUsed + + w.saveCheckpointUsed + return map[string]any{ + "startAt": w.startAt.String(), + "endAt": endAt.String(), + "duration": dur.String(), + "checkOverQuotaUsed": w.checkOverQuotaUsed.String(), + "commitTimeSeriesUsed": w.commitTimeSeriesUsed.String(), + "commitEntityUsed": w.commitEntityUsed.String(), + "commitWebhookUsed": w.commitWebhookUsed.String(), + "saveUsageUsed": w.saveUsageUsed.String(), + "saveCheckpointUsed": w.saveCheckpointUsed.String(), + "totalBinding": w.totalBinding, + "failedCount": w.failedCount, + "count": w.count, + "pressure": float64(total) / float64(dur), + } +} diff --git a/driver/controller/block_builder.go b/driver/controller/block_builder.go new file mode 100644 index 0000000..c8c48ae --- /dev/null +++ b/driver/controller/block_builder.go @@ -0,0 +1,332 @@ +package controller + +import ( + "bytes" + "context" + "fmt" + "strconv" + "sync" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/utils" +) + +type BlockHeader interface { + GetBlockNumber() uint64 + GetBlockParentHash() string + GetBlockHash() string + GetBlockTime() time.Time +} + +func GetBlockSummary(b BlockHeader) string { + const hashPreviewLen = 8 + hash := b.GetBlockHash() + if len(hash) == 0 { + return fmt.Sprintf("%d", b.GetBlockNumber()) + } else if len(hash) < hashPreviewLen { + return fmt.Sprintf("%d/%s", b.GetBlockNumber(), hash) + } else { + return fmt.Sprintf("%d/%s", b.GetBlockNumber(), hash[:hashPreviewLen]) + } +} + +func GetBlockFullText[H BlockHeader](b H) string { + var bf bytes.Buffer + bf.WriteString(strconv.FormatUint(b.GetBlockNumber(), 10)) + if hash := b.GetBlockHash(); len(hash) > 0 { + bf.WriteString("/") + bf.WriteString(hash) + } + bf.WriteString("/") + bf.WriteString(b.GetBlockTime().Format(time.RFC3339Nano)) + if hash := b.GetBlockParentHash(); len(hash) > 0 { + bf.WriteString("->") + bf.WriteString(hash) + } + return bf.String() +} + +type BlockData interface { + BlockHeader + + GetTaskList() []Task // BlockData without task may be not a empty BlockData + CheckpointData() map[string]string // at least should return a empty map + DataSource() string + Size() int +} + +type TaskIndex struct { + Global uint64 + InBlock int + TotalInBlock int +} + +type Task interface { + BlockHeader + + Summary() string + GetHandlerID() HandlerID + Init(ctx context.Context, index TaskIndex, progressbar ProgressBar) + Exec(ctx context.Context, checkpointCtrl CheckpointController) *ExternalError +} + +type ProgressBar struct { + LatestBlock BlockHeader + FullBlockRange BlockRange +} + +type BlockBuilder interface { + // Start Reset the templates, then reconstruct all HandlerAgent, start constructing blockData + // after checkpoint.BlockNumber, and return them one by one through Next. + // If checkpoint is nil, it means starting from the beginning, and the first block to be processed will be + // obtained through all HandlerAgent. + Start( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + ) (agentStat map[string]int, extErr *ExternalError) + + // Next Get the block data of the current block. + // may be waiting for the Fetcher to fetch data, or waiting for the latest block. + // If reorg is detected, the returned reorg will not be nil, and its value indicates that all data from + // that block are invalid. + Next(ctx context.Context) (blockNumber uint64, blockData BlockData, progressBar ProgressBar, reorg *uint64, err error) + + // Finish End the traversal of the block + Finish() + + Snapshot() map[string]any +} + +type Client interface { + // GetLatest return err may be ErrInternalNeedUpgrade + GetLatest(ctx context.Context) (latest BlockHeader, first uint64, err error) + // Subscribe broken in callback may be ErrInternalNeedUpgrade + Subscribe(ctx context.Context, from BlockHeader, callback func(latest BlockHeader, broken error)) + // GetHeaderIgnoreCache get the block header without cache, used to check reorg + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) + // ResetCache should remove the cached data from the target block, used when reorg detected or progress passed + ResetCache(r BlockRange) + Snapshot() any +} + +type FetchTarget interface { + Size() int +} + +type Fetcher[T FetchTarget] interface { + GetName() string + GetFullRange() BlockRange + Snapshot() any + KeepFetch(ctx context.Context) + Get(ctx context.Context, blockNumber uint64) (data T, has bool, latest BlockHeader, err error) + UpdateLatest(latest BlockHeader) + Broken(err error) + MoveStart(start uint64) +} + +type blockBuilder struct { + handlerCtrl HandlerController + client Client + checkLink bool + + dataFetcher Fetcher[BlockData] + + fetchCancel context.CancelFunc + fetchDone sync.WaitGroup + + mu sync.RWMutex + + fullBlockRange BlockRange + currentBlockNumber uint64 + + headerList []BlockHeader +} + +func NewBlockBuilder( + handlerCtrl HandlerController, + client Client, + checkLink bool, +) BlockBuilder { + return &blockBuilder{ + handlerCtrl: handlerCtrl, + client: client, + checkLink: checkLink, + } +} + +func (b *blockBuilder) Start( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, +) (map[string]int, *ExternalError) { + latest, first, err := b.client.GetLatest(ctx) + if err != nil { + if errors.Is(err, ErrInternalNeedUpgrade) { + return nil, NewExternalError(ErrCodeNeedUpgrade, err) + } + return nil, NewExternalError(ErrCodeFetchDataFailed, err) + } + if checkpoint != nil && checkpoint.BlockNumber < first { + first = checkpoint.BlockNumber + } + if extErr := b.handlerCtrl.Prologue(ctx, checkpoint, templates, first, latest); extErr != nil { + return nil, extErr + } + if len(b.handlerCtrl.GetAgentStat()) == 0 { + return nil, NewExternalError(ErrCodeUnexpectedProcessorConfig, errors.Errorf("no handler")) + } + + b.mu.Lock() + defer b.mu.Unlock() + + b.fullBlockRange = b.handlerCtrl.GetBlockRange() + b.currentBlockNumber = b.fullBlockRange.StartBlock + b.headerList = nil + if checkpoint != nil { + b.currentBlockNumber = checkpoint.BlockNumber + 1 + if b.checkLink { + b.headerList = append(b.headerList, checkpoint) + } + } + + b.client.ResetCache(BlockRange{StartBlock: b.currentBlockNumber}) + + _, logger := log.FromContext(ctx) + logger.Infow("will start to fetch data", + "full", b.fullBlockRange.String(), + "current", b.currentBlockNumber, + "latest", GetBlockSummary(latest)) + + b.dataFetcher = b.handlerCtrl.BuildBlockDataFetcher(first, b.currentBlockNumber, latest) + + var fetchCtx context.Context + fetchCtx, b.fetchCancel = context.WithCancel(ctx) + b.fetchDone.Add(2) + go func() { + b.client.Subscribe(fetchCtx, latest, func(latest BlockHeader, broken error) { + if broken != nil { + b.dataFetcher.Broken(broken) + } else { + b.dataFetcher.UpdateLatest(latest) + } + }) + b.fetchDone.Done() + }() + go func() { + b.dataFetcher.KeepFetch(fetchCtx) + b.fetchDone.Done() + }() + return b.handlerCtrl.GetAgentStat(), nil +} + +func (b *blockBuilder) checkReorg(ctx context.Context, cur BlockHeader) (reorg *uint64, err error) { + _, logger := log.FromContext(ctx) + b.mu.RLock() + defer b.mu.RUnlock() + if len(b.headerList) == 0 { + return + } + pre := b.headerList[len(b.headerList)-1] + if pre.GetBlockNumber()+1 == cur.GetBlockNumber() && cur.GetBlockParentHash() == pre.GetBlockHash() { + return + } + for i := len(b.headerList) - 1; i >= 0; i-- { + var h BlockHeader + h, err = b.client.GetHeaderIgnoreCache(ctx, b.headerList[i].GetBlockNumber()) + if err != nil { + return + } + if h.GetBlockHash() == b.headerList[i].GetBlockHash() { + if i < len(b.headerList)-1 { + logger.Warnf("block %s not changed", GetBlockSummary(h)) + // first different block must in [b.headerList[i].GetBlockNumber()+1, b.headerList[i+1].GetBlockNumber()], + // be conservative and clear all data starting from b.headerList[i].GetBlockNumber()+1 + reorg = utils.WrapPointer(h.GetBlockNumber() + 1) + } + return + } + reorg = utils.WrapPointer(b.headerList[i].GetBlockNumber()) + logger.Warnf("block %s changed to %s", GetBlockSummary(b.headerList[i]), GetBlockSummary(h)) + } + return +} + +func (b *blockBuilder) Next(ctx context.Context) ( + blockNumber uint64, + blockData BlockData, + progressBar ProgressBar, + reorg *uint64, + err error, +) { + b.mu.RLock() + blockNumber = b.currentBlockNumber + progressBar.FullBlockRange = b.fullBlockRange + b.mu.RUnlock() + + if !b.fullBlockRange.Contains(blockNumber) { + if blockNumber < b.fullBlockRange.StartBlock { + // unreachable + panic(errors.Errorf("current block number %d is to the left of full block range %s", + blockNumber, b.fullBlockRange)) + } + // out of range, just return + return + } + + var has bool + blockData, has, progressBar.LatestBlock, err = b.dataFetcher.Get(ctx, blockNumber) + if err != nil { + return + } + + // detect reorg + if has && b.checkLink && progressBar.LatestBlock.GetBlockTime().Sub(blockData.GetBlockTime()) < WatchingDelay { + if reorg, err = b.checkReorg(ctx, blockData); reorg != nil || err != nil { + return + } + } + + // progress will go to b.currentBlockNumber + 1, so all cached data in [0,b.currentBlockNumber] are useless now. + if has && progressBar.LatestBlock.GetBlockTime().Sub(blockData.GetBlockTime()) < WatchingDelay { + b.client.ResetCache(BlockRange{EndBlock: &b.currentBlockNumber}) + } + + b.mu.Lock() + if b.checkLink && has { + b.headerList = append(b.headerList, blockData) + b.headerList = sparsify.Sparsify(b.headerList, BlockHeader.GetBlockNumber) + } + b.currentBlockNumber++ + b.dataFetcher.MoveStart(b.currentBlockNumber) + b.mu.Unlock() + return +} + +func (b *blockBuilder) Finish() { + b.fetchCancel() + b.fetchDone.Wait() + b.handlerCtrl.Epilogue() +} + +func (b *blockBuilder) Snapshot() map[string]any { + b.mu.RLock() + defer b.mu.RUnlock() + sn := map[string]any{ + "client": b.client.Snapshot(), + "checkLink": b.checkLink, + "handlerController": b.handlerCtrl.Snapshot(), + } + if b.dataFetcher != nil { + // already started + sn["fetcher"] = b.dataFetcher.Snapshot() + sn["fullRange"] = b.fullBlockRange.String() + sn["currentBlockNumber"] = b.currentBlockNumber + sn["headerList"] = utils.MapSliceNoError(b.headerList, GetBlockSummary) + } + return sn +} diff --git a/driver/controller/checkpoint.go b/driver/controller/checkpoint.go new file mode 100644 index 0000000..1d7abe2 --- /dev/null +++ b/driver/controller/checkpoint.go @@ -0,0 +1,1001 @@ +package controller + +import ( + "context" + "fmt" + "math" + "strconv" + "strings" + "sync" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/timeseries" + + "github.com/pkg/errors" +) + +type TemplateInstance struct { + TemplateID int32 + TemplateName string + Address string + Labels string + + Removed bool + + BlockRange +} + +func (t TemplateInstance) UniqID() string { + if t.Labels == "" { + return fmt.Sprintf("%d/%s", t.TemplateID, t.Address) + } + return fmt.Sprintf("%d/%s/%s", t.TemplateID, t.Address, t.Labels) +} + +func (t TemplateInstance) String() string { + firstPart := strconv.FormatInt(int64(t.TemplateID), 10) + if t.TemplateName != "" { + firstPart += "/" + t.TemplateName + } + var labelPart string + if t.Labels != "" { + labelPart = "::" + t.Labels + } + onOffPart := utils.Select(t.Removed, "OFF", "ON") + return fmt.Sprintf("%s::%s%s::%s-%s", firstPart, t.Address, labelPart, onOffPart, t.BlockRange.String()) +} + +type TplWithCreated struct { + TemplateInstance + + CreatedBlock uint64 +} + +func (t TplWithCreated) String() string { + return fmt.Sprintf("%d#%s", t.CreatedBlock, t.TemplateInstance.String()) +} + +func CountTemplatesByID(orig map[uint64][]TemplateInstance) map[int32]int { + sum := make(map[int32]int) + for _, templates := range orig { + for _, tpl := range templates { + sum[tpl.TemplateID] += 1 + } + } + return sum +} + +type CheckpointData struct { +} + +type Checkpoint struct { + BlockNumber uint64 + BlockHash string `json:"BlockHash,omitempty"` + BlockParentHash string `json:"BlockParentHash,omitempty"` + BlockTime time.Time + + TotalBindings uint64 + + LatestBlockNumber uint64 + LatestBlockHash string `json:"LatestBlockHash,omitempty"` + LatestBlockParentHash string `json:"LatestBlockParentHash,omitempty"` + LatestBlockTime time.Time + + FullBlockRange BlockRange + + Data map[string]string `json:"Data,omitempty"` +} + +func (c Checkpoint) GetBlockNumber() uint64 { + return c.BlockNumber +} + +func (c Checkpoint) GetBlockParentHash() string { + return c.BlockParentHash +} + +func (c Checkpoint) GetBlockHash() string { + return c.BlockHash +} + +func (c Checkpoint) GetBlockTime() time.Time { + return c.BlockTime +} + +func (c Checkpoint) InWatching() bool { + return c.LatestBlockTime.Sub(c.BlockTime) < WatchingDelay +} + +func (c Checkpoint) AllDone() bool { + return c.FullBlockRange.EndBlock != nil && *c.FullBlockRange.EndBlock == c.BlockNumber +} + +func (c Checkpoint) CurrentLastBlockNumber() uint64 { + if c.FullBlockRange.EndBlock != nil && *c.FullBlockRange.EndBlock < c.LatestBlockNumber { + return *c.FullBlockRange.EndBlock + } + return c.LatestBlockNumber +} + +func (c Checkpoint) Rate() float64 { + return float64(c.BlockNumber-c.FullBlockRange.StartBlock+1) / + float64(c.CurrentLastBlockNumber()-c.FullBlockRange.StartBlock+1) +} + +func (c Checkpoint) RateOrDelay() string { + if c.InWatching() { + return fmt.Sprintf("[Delay:%s]", time.Since(c.BlockTime).String()) + } else { + return fmt.Sprintf("[%.1f%%]", c.Rate()*100) + } +} + +func (c Checkpoint) Snapshot() any { + latest := Checkpoint{ + BlockNumber: c.LatestBlockNumber, + BlockHash: c.LatestBlockHash, + BlockParentHash: c.LatestBlockParentHash, + BlockTime: c.LatestBlockTime, + } + return map[string]any{ + "fullBlockRange": c.FullBlockRange.String(), + "current": GetBlockFullText(c), + "latest": GetBlockFullText(&latest), + "totalBindings": c.TotalBindings, + "data": utils.MapMapNoError(c.Data, utils.StringSummaryV2), + } +} + +func (c Checkpoint) String() string { + return GetBlockFullText(c) +} + +type CheckpointController interface { + // GetLatestCheckpoint get the latest checkpoint + GetLatestCheckpoint() *Checkpoint + + // GetSavedLatestCheckpoint get the latest checkpoint + GetSavedLatestCheckpoint() *Checkpoint + + // GetTemplates return all templates + // There may be template instances later than the checkpoint, which are generated when the binding data of the + // later block is processed, but these are only temporary. + GetTemplates() map[uint64][]TemplateInstance + + // Ready Clean up invalid time series data and entity data by checkpoint, executed at the start of a round + Ready(ctx context.Context, agentStat map[string]int) *ExternalError + + // CleanCheckpoint Delete checkpoints and templates greater than or equal to blockNumberGE, executed after reorg + CleanCheckpoint(ctx context.Context, curBlockNumber, blockNumberGE uint64) *ExternalError + + // MakeCheckpoint Try to construct a checkpoint at the blockData block, indicating that all bindings of this block + // have been processed. + MakeCheckpoint( + ctx context.Context, + blockData BlockDataSummary, + progressBar ProgressBar, + ) (hasNewTemplate bool, err *ExternalError) + + // Save Actually try to save checkpoints + Save(ctx context.Context, saveAll bool) *ExternalError + + // KeepSave Continuously try to save checkpoints, + // if all checkpoint was made, allMade will be closed, and KeepSave will return nil after all Checkpoint was saved. + KeepSave(ctx context.Context, allMade chan struct{}) error + + // SaveError An error occurred during processing, save the error information + SaveError(ctx context.Context, err *ExternalError) error + + // NewTemplateInstance Declare a new template instance in the specified task + NewTemplateInstance(ctx context.Context, task Task, templates []TemplateInstance) *ExternalError + + // InsertTimeSeriesData Insert time series data into the specified block + InsertTimeSeriesData(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) + + // InsertWebhookData Insert webhook data in the specified block + InsertWebhookData(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) + + // GetEntityOrInterfaceType Get entity or interface declaration by name + GetEntityOrInterfaceType(entity string) schema.EntityOrInterface + // GetEntityType Get entity declaration by name + GetEntityType(entity string) *schema.Entity + // GetEntity get entity + GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + // GetEntityInBlock get entity + GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + // ListEntity list entity + ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, + ) (boxes []*persistent.EntityBox, next *string, err *ExternalError) + // ListRelated list related entity to / with the relationship definition in . + ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, + ) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) + + // SetEntity set or delete entity + SetEntity(ctx context.Context, entityType *schema.Entity, box persistent.UncommittedEntityBox) *ExternalError + + Snapshot() map[string]any +} + +type CheckpointStore interface { + Load(ctx context.Context) ([]Checkpoint, map[uint64][]TemplateInstance, error) + Save( + ctx context.Context, + checkpoints []Checkpoint, + templates map[uint64][]TemplateInstance, + agentStat map[string]int, + ) error + SaveError(ctx context.Context, err *ExternalError) error +} + +var ErrCheckpointsTooBig = errors.New("checkpoints too big") + +var _ CheckpointController = &checkpointController{} + +type checkpointController struct { + chainID string + saveDelay time.Duration + saveInterval time.Duration + maxKeepCheckpointCount uint64 + + checkpointStore CheckpointStore + checkpoints []Checkpoint + savedCheckpoints int // checkpoints[:savedCheckpoints] are all saved in checkpointStore + checkpointSparsifyMultiplier uint64 + + // lastSavedAt and lastSavedCheckpoint are used to estimate Watching needed, + lastSavedAt time.Time + lastSavedCheckpoint *Checkpoint + actuallyLastSavedAt time.Time + + templates map[uint64][]TemplateInstance // key is blockNumber + unsavedTemplates map[uint64][]TemplateInstance // key is blockNumber + + agentStat map[string]int + + quotaService QuotaService + + commitCtxBuilder func(ctx context.Context, chainID string, cur Checkpoint) context.Context + + timeSeriesCtrl TimeSeriesController + entityCtrl EntityController + webhookCtrl WebhookController + + stopped bool + printProcessedExecutor *timer.MinimumIntervalExecutor + + stat *timewin.TimeWindowsManager[*checkpointStatWindow] + + mu sync.Mutex +} + +func NewCheckpointController( + ctx context.Context, + chainID string, + saveDelay time.Duration, + saveInterval time.Duration, + maxKeepCheckpointCount uint64, + checkpointStore CheckpointStore, + quotaService QuotaService, + timeSeriesCtrl TimeSeriesController, + entityCtrl EntityController, + webhookCtrl WebhookController, + commitCtxBuilder func(ctx context.Context, chainID string, cur Checkpoint) context.Context, +) (CheckpointController, error) { + c := checkpointController{ + chainID: chainID, + saveDelay: saveDelay, + saveInterval: saveInterval, + maxKeepCheckpointCount: maxKeepCheckpointCount, + checkpointStore: checkpointStore, + quotaService: quotaService, + commitCtxBuilder: commitCtxBuilder, + timeSeriesCtrl: timeSeriesCtrl, + entityCtrl: entityCtrl, + webhookCtrl: webhookCtrl, + printProcessedExecutor: timer.NewMinimumIntervalExecutor(PrintProcessedInterval), + stat: timewin.NewTimeWindowsManager[*checkpointStatWindow](time.Minute), + } + checkpoints, templates, err := checkpointStore.Load(ctx) + if err != nil { + return nil, err + } + c.checkpoints = checkpoints + c.savedCheckpoints = len(checkpoints) + if templates != nil { + c.templates = templates + } else { + c.templates = make(map[uint64][]TemplateInstance) + } + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + return &c, nil +} + +func (c *checkpointController) findCheckpoint(blockNumberLE uint64) int { + var p int + for p < len(c.checkpoints) && c.checkpoints[p].BlockNumber <= blockNumberLE { + p++ + } + return p - 1 +} + +func (c *checkpointController) getLatestCheckpoint(saved bool) *Checkpoint { + cc := len(c.checkpoints) + if saved { + cc = c.savedCheckpoints + } + if cc == 0 { + return nil + } + return &c.checkpoints[cc-1] +} + +func (c *checkpointController) GetLatestCheckpoint() *Checkpoint { + c.mu.Lock() + defer c.mu.Unlock() + return c.getLatestCheckpoint(false) +} + +func (c *checkpointController) GetSavedLatestCheckpoint() *Checkpoint { + c.mu.Lock() + defer c.mu.Unlock() + return c.getLatestCheckpoint(true) +} + +func (c *checkpointController) getTemplates(savedOnly bool) map[uint64][]TemplateInstance { + if savedOnly { + if c.savedCheckpoints == 0 { + return make(map[uint64][]TemplateInstance) + } + savedBlockNumber := c.checkpoints[c.savedCheckpoints-1].BlockNumber + return utils.FilterMap(c.templates, func(u uint64) bool { + return u <= savedBlockNumber + }) + } + return c.templates +} + +func (c *checkpointController) GetTemplates() map[uint64][]TemplateInstance { + c.mu.Lock() + defer c.mu.Unlock() + return c.getTemplates(false) +} + +func (c *checkpointController) Ready(ctx context.Context, agentStat map[string]int) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + c.agentStat = agentStat + var checkpoint *Checkpoint + if len(c.checkpoints) > 0 { + checkpoint = &c.checkpoints[len(c.checkpoints)-1] + } + _, logger := log.FromContext(ctx, "checkpoint", utils.NullOrToString(checkpoint)) + logger.Debug("will clean data") + g, gctx := errgroup.WithContext(ctx) + g.Go(func() error { + if extErr := c.entityCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in entity controller failed") + return extErr + } + return nil + }) + g.Go(func() error { + if extErr := c.timeSeriesCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in time series controller failed") + return extErr + } + return nil + }) + g.Go(func() error { + if extErr := c.webhookCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in webhook controller failed") + return extErr + } + return nil + }) + if err := g.Wait(); err != nil { + var extErr *ExternalError + if errors.As(err, &extErr) { + return extErr + } + // unreachable + return NewExternalError(ErrCodeSystem, err) + } + c.stopped = false + return nil +} + +func (c *checkpointController) CleanCheckpoint( + ctx context.Context, + curBlockNumber, + blockNumberGE uint64, +) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + _, logger := log.FromContext(ctx) + logger = logger.UserVisible() + detectedMsg := fmt.Sprintf("Reorg detected when processing block %d, all blocks from block %d are invalid", + curBlockNumber, blockNumberGE) + c.stopped = true + var cc int + for cc < len(c.checkpoints) && c.checkpoints[cc].BlockNumber < blockNumberGE { + cc++ + } + // c.checkpoints[cc:] need to delete + if cc > 0 { + // we have at least one checkpoint + tcc := len(c.checkpoints) + c.checkpoints = c.checkpoints[:cc] + backMsg := fmt.Sprintf("progress will back to %s", c.checkpoints[cc-1].String()) + if cc < c.savedCheckpoints { + logger.Warnf("%s, will remove all %d unsaved checkpoints and %d saved checkpoints in checkpoint store, %s", + detectedMsg, tcc-c.savedCheckpoints, c.savedCheckpoints-cc, backMsg) + // saved checkpoint should rollback + if err := c.checkpointStore.Save(ctx, c.checkpoints, c.getTemplates(false), c.agentStat); err != nil { + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + c.savedCheckpoints = len(c.checkpoints) + } else if cc == c.savedCheckpoints { + logger.Warnf("%s, will remove all %d unsaved checkpoints, %s", detectedMsg, tcc-cc, backMsg) + } else { + logger.Warnf("%s, will remove %d unsaved checkpoints, %s", detectedMsg, tcc-cc, backMsg) + } + deleteFilter := func(bn uint64) bool { + return bn > c.checkpoints[cc-1].BlockNumber + } + utils.MapDelete(c.templates, deleteFilter) + utils.MapDelete(c.unsavedTemplates, deleteFilter) + } else { + logger.Warnf("%s, will remove all checkpoints", detectedMsg) + // all checkpoints are invalid, it means all data are useless + if err := c.checkpointStore.Save(ctx, nil, nil, c.agentStat); err != nil { + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + c.checkpoints = nil + c.savedCheckpoints = 0 + c.templates = make(map[uint64][]TemplateInstance) + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + } + return nil +} + +func (c *checkpointController) MakeCheckpoint( + ctx context.Context, + blockData BlockDataSummary, + progressBar ProgressBar, +) (templatesChanged bool, extErr *ExternalError) { + _, logger := log.FromContext(ctx, + "current", GetBlockSummary(blockData), + "latest", GetBlockSummary(progressBar.LatestBlock)) + + c.mu.Lock() + defer c.mu.Unlock() + + if c.stopped { + logger.Warnf("try to make checkpoint after checkpoint controller stopped") + return + } + + ck := Checkpoint{ + BlockNumber: blockData.GetBlockNumber(), + BlockHash: blockData.GetBlockHash(), + BlockParentHash: blockData.GetBlockParentHash(), + BlockTime: blockData.GetBlockTime(), + TotalBindings: uint64(blockData.TaskCount), + LatestBlockNumber: progressBar.LatestBlock.GetBlockNumber(), + LatestBlockHash: progressBar.LatestBlock.GetBlockHash(), + LatestBlockParentHash: progressBar.LatestBlock.GetBlockParentHash(), + LatestBlockTime: progressBar.LatestBlock.GetBlockTime(), + FullBlockRange: progressBar.FullBlockRange, + Data: blockData.CheckpointData, + } + processedMsg := fmt.Sprintf("Processed %s[%d/%s/%d] with %d bindings", + ck.RateOrDelay(), + ck.FullBlockRange.StartBlock, + GetBlockSummary(blockData), + ck.CurrentLastBlockNumber(), + ck.TotalBindings) + + var templates []TemplateInstance + if templates, templatesChanged = c.unsavedTemplates[blockData.GetBlockNumber()]; templatesChanged { + // If there are new template instances, temporarily confirm these template instances, + // then return ErrInternalHasNewTemplate and wait for the next time. + c.templates[blockData.GetBlockNumber()] = append(c.templates[blockData.GetBlockNumber()], templates...) + // New template instances that come later need to be ignored because the current block may have new data + // at the beginning, and the generation process of those new template instances may change as a result. + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + // No more templates will be accepted in this run. + c.stopped = true + var minStartBlock uint64 = math.MaxUint64 + for _, tpl := range templates { + minStartBlock = min(minStartBlock, tpl.StartBlock) + } + logger = logger.UserVisible() + processedMsg += fmt.Sprintf(" and %d new templates [%s]", + len(templates), strings.Join(utils.MapSliceNoError(templates, TemplateInstance.String), ",")) + if minStartBlock == ck.BlockNumber { + logger.Warn(processedMsg + ", but this block will be re-processed because has new template from this block") + } else { + // The checkpoint of the current block is still acceptable, but because a new template is created later, + // the BlockBuilder need to be reset and restarted. + c.checkpoints = append(c.checkpoints, ck) + logger.Info(processedMsg) + } + return + } + c.checkpoints = append(c.checkpoints, ck) + + printProcessed := func() { + logger.UserVisible().Info(processedMsg) + } + if ck.InWatching() || ck.TotalBindings > 0 { + printProcessed() + } else { + c.printProcessedExecutor.ExecSimple(printProcessed) + } + + if c.saveDelay == 0 && ck.InWatching() { + // realtime mode + extErr = c.save(ctx, false, true) + } else if uint64(len(c.checkpoints)) >= c.maxKeepCheckpointCount { + logger.Info("will try to save checkpoint because there are too many checkpoints") + extErr = c.save(ctx, false, false) + } else if c.webhookCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted webhook message") + extErr = c.save(ctx, false, false) + } else if c.timeSeriesCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted time series data") + extErr = c.save(ctx, false, false) + } else if c.entityCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted entity changes") + extErr = c.save(ctx, false, false) + } + return +} + +func (c *checkpointController) Save(ctx context.Context, saveAll bool) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + return c.save(ctx, saveAll, true) +} + +func (c *checkpointController) estimateWatchingNeed(cur *Checkpoint) string { + if cur == nil { + return "" + } + if cur.InWatching() { + return "0" + } + passed := time.Since(c.lastSavedAt) + if c.lastSavedCheckpoint == nil || passed < time.Minute { + return "" + } + last := *c.lastSavedCheckpoint + growthSpeed := float64(cur.CurrentLastBlockNumber()-last.CurrentLastBlockNumber()) / passed.Seconds() + processSpeed := float64(cur.BlockNumber-last.BlockNumber) / passed.Seconds() + if processSpeed <= growthSpeed { + return "INF" + } + eta := time.Second * time.Duration(float64(cur.CurrentLastBlockNumber()-cur.BlockNumber)/(processSpeed-growthSpeed)) + return eta.String() +} + +func (c *checkpointController) save(ctx context.Context, saveAll bool, checkInterval bool) (extErr *ExternalError) { + _, logger := log.FromContext(ctx) + + if c.stopped { + return + } + + if !saveAll && checkInterval && time.Since(c.actuallyLastSavedAt) < c.saveInterval/2 { + // This is not the final save, and not long enough since the last save, just ignore + return + } + + tm := timer.NewTimer() + startTm := tm.Start("ALL") + + // Find the checkpoints should be saved + var cc int + if saveAll { + cc = len(c.checkpoints) + } else { + cc = c.savedCheckpoints + for cc < len(c.checkpoints) && time.Since(c.checkpoints[cc].BlockTime) > c.saveDelay { + cc++ + } + } + if cc == 0 || cc == c.savedCheckpoints { + // No new checkpoints or all new checkpoints are too close to the current time, so there is nothing to save. + return + } + + cur := c.checkpoints[cc-1] + pre := cur.FullBlockRange.StartBlock + if c.savedCheckpoints > 0 { + pre = c.checkpoints[c.savedCheckpoints-1].BlockNumber + 1 + } + var totalBindings uint64 + for i := c.savedCheckpoints; i < cc; i++ { + totalBindings += c.checkpoints[i].TotalBindings + } + if cc > c.savedCheckpoints { // if c.allDone() is true, cc may be equal to c.savedCheckpoints + // checkpoints[:cc] should be saved. + // However, considering that checkpoint data is only generated when each data store commits, + // so only c.checkpoints[cc-1] should to save, + // so unsaved checkpoints in c.checkpoints[c.savedCheckpoints:cc-1] should be removed + c.checkpoints = utils.RemoveSubSeq(c.checkpoints, c.savedCheckpoints, cc-1-c.savedCheckpoints) + cc = c.savedCheckpoints + 1 + } + + win := checkpointStatWindow{startAt: time.Now(), count: 1} + defer func() { + if extErr == nil { + win.totalBinding = totalBindings + } else { + win.failedCount = 1 + } + c.stat.Append(&win) + }() + + checkOverQuotaTm := tm.Start("O") + over, err := c.quotaService.CheckOverQuota(ctx) + win.checkOverQuotaUsed = checkOverQuotaTm.End() + if err != nil { + return NewExternalError(ErrCodeQuotaServiceError, err) + } else if over != nil { + logger.UserVisible().Errorf("Over quota: %s", over.Detail) + return NewExternalError(ErrCodeOverQuota, errors.Errorf("over quota: %s", over.Msg)) + } + + var usage Usage + + defer func() { + startTm.End() + logger = logger.UserVisible().With("used", tm.ReportDistribution("ALL", "*")) + progress := fmt.Sprintf("%s[%d/%d-%d/%d] with %d bindings", + cur.RateOrDelay(), + cur.FullBlockRange.StartBlock, + pre, + cur.BlockNumber, + cur.CurrentLastBlockNumber(), + totalBindings) + if extErr != nil { + // commits below may be completed partly, so all unsaved checkpoint should be clean, and need to re-init + c.checkpoints = c.checkpoints[:c.savedCheckpoints] + c.templates = c.getTemplates(true) + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + c.stopped = true + if extErr.IsUserError() { + logger.Warnf("Save %s failed: %s", progress, extErr.Error()) + } else { + logger.Warnf("Save %s failed, progress will back to %s", progress, c.getLatestCheckpoint(true)) + } + } else { + // succeed + logger.Infof("Saved %s and %s", progress, usage.String()) + c.actuallyLastSavedAt = time.Now() + if c.lastSavedCheckpoint == nil || time.Since(c.lastSavedAt) > time.Minute*5 { + c.lastSavedAt, c.lastSavedCheckpoint = c.actuallyLastSavedAt, &cur + } + } + }() + + if c.commitCtxBuilder != nil { + ctx = c.commitCtxBuilder(ctx, c.chainID, cur) + } + + // Save the time series data up to cur.BlockNumber + commitTimeSeriesTm := tm.Start("T") + usage.TimeSeries, extErr = c.timeSeriesCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitTimeSeriesUsed = commitTimeSeriesTm.End() + if extErr != nil { + return extErr + } + + // Save entity data up to cur.BlockNumber + commitEntityTm := tm.Start("E") + usage.EntityCreated, usage.EntityUpdated, extErr = c.entityCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitEntityUsed = commitEntityTm.End() + if extErr != nil { + return extErr + } + + // Save webhook data up to cur.BlockNumber + commitWebhookTm := tm.Start("W") + usage.Export, extErr = c.webhookCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitWebhookUsed = commitWebhookTm.End() + if extErr != nil { + return extErr + } + + // Save usage + saveUsageTm := tm.Start("U") + err = c.quotaService.SaveUsage(ctx, usage, cur.InWatching()) + win.saveUsageUsed = saveUsageTm.End() + if err != nil { + return NewExternalError(ErrCodeQuotaServiceError, err) + } + + // Sparse these cc checkpoints and save them + for { + multiplier := max(c.checkpointSparsifyMultiplier, 2) + saveCheckpointTm := tm.Start("S") + remove := sparsify.RemoveWithMultiplier(c.checkpoints[:cc], Checkpoint.GetBlockNumber, multiplier/2, multiplier) + c.checkpoints = utils.RemoveByIndex(c.checkpoints, remove) + cc -= len(remove) + c.savedCheckpoints = cc + err = c.checkpointStore.Save(ctx, c.checkpoints[:c.savedCheckpoints], c.getTemplates(true), c.agentStat) + win.saveCheckpointUsed = saveCheckpointTm.End() + if err != nil { + if errors.Is(err, ErrCheckpointsTooBig) { + logger.With("sparsifyMultiplier", multiplier).Warnfe(err, "checkpoints too big, will reduce it") + c.checkpointSparsifyMultiplier = multiplier * 2 + continue + } + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + return nil + } +} + +func (c *checkpointController) KeepSave(ctx context.Context, allMade chan struct{}) error { + _, logger := log.FromContext(ctx) + logger.Info("keep save checkpoint started") + defer func() { + logger.Info("keep save checkpoint finished") + }() + ticker := time.NewTicker(c.saveInterval) + defer ticker.Stop() + for { + if extErr := c.Save(ctx, false); extErr != nil { + return extErr + } + select { + case <-allMade: + // last save + if extErr := c.Save(ctx, true); extErr != nil { + return extErr + } + return nil + case <-ticker.C: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (c *checkpointController) SaveError(ctx context.Context, err *ExternalError) error { + return c.checkpointStore.SaveError(ctx, err) +} + +func (c *checkpointController) NewTemplateInstance( + ctx context.Context, + task Task, + templates []TemplateInstance, +) *ExternalError { + blockNumber := task.GetBlockNumber() + c.mu.Lock() + defer c.mu.Unlock() + _, logger := log.FromContext(ctx) + if c.stopped { + logger.Warnw("try to make new template instance after checkpoint controller stopped", + "blockNumber", blockNumber, + "templates", utils.MapSliceNoError(templates, TemplateInstance.String)) + return nil + } + exists := make(map[string][]TplWithCreated) // key is /
/ + for _, bn := range utils.GetOrderedMapKeys(c.templates) { + for _, tpl := range c.templates[bn] { + exists[tpl.UniqID()] = append(exists[tpl.UniqID()], TplWithCreated{TemplateInstance: tpl, CreatedBlock: bn}) + } + } + for _, bn := range utils.GetOrderedMapKeys(c.unsavedTemplates) { + if bn > blockNumber { + break + } + for _, tpl := range c.unsavedTemplates[bn] { + exists[tpl.UniqID()] = append(exists[tpl.UniqID()], TplWithCreated{TemplateInstance: tpl, CreatedBlock: bn}) + } + } + for _, tpl := range templates { + if tpl.StartBlock < blockNumber && !tpl.Removed { + logger.Warnf("start block of the template instance %s less then the block %d created it, will be reset to %d", + tpl, blockNumber, blockNumber) + tpl.StartBlock = blockNumber + } + if exist := exists[tpl.UniqID()]; len(exist) > 0 { + existText := strings.Join(utils.MapSliceNoError(exist, TplWithCreated.String), ",") + on := EmptyBlockRangeSet + for _, ex := range exist { + if ex.Removed { + on = on.Remove(ex.BlockRange) + } else { + on = on.Union(ex.BlockRange) + } + } + on = on.Intersection(BlockRange{StartBlock: blockNumber}) + intersection := on.Intersection(tpl.BlockRange) + if tpl.Removed && intersection.IsEmpty() || !tpl.Removed && intersection.Include(tpl.BlockRange) { + logger.Warnf("try to create new template instance %s at %s, but already created %s, will be ignored", + tpl, task.Summary(), existText) + continue + } + if tpl.Removed { + on = on.Remove(tpl.BlockRange) + } else { + on = on.Union(tpl.BlockRange) + } + if len(on.Holes) > 0 { + return NewExternalError(ErrCodeCreateTemplateFailed, errors.Errorf( + "new template instance %s at %s is invalid, already created %s, "+ + "after created the enable block range will have hole", tpl, task.Summary(), existText)) + } + logger.Infof("has new template %s at %s, and already created %s", tpl, task.Summary(), existText) + } else { + logger.Infof("has new template %s at %s", tpl, task.Summary()) + } + c.unsavedTemplates[blockNumber] = append(c.unsavedTemplates[blockNumber], tpl) + } + return nil +} + +func (c *checkpointController) InsertTimeSeriesData( + blockNumber uint64, + taskIndex TaskIndex, + data []timeseries.Dataset, +) { + c.mu.Lock() + defer c.mu.Unlock() + c.timeSeriesCtrl.Insert(blockNumber, taskIndex, data) +} + +func (c *checkpointController) InsertWebhookData(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) { + c.mu.Lock() + defer c.mu.Unlock() + c.webhookCtrl.Insert(blockNumber, taskIndex, messages) +} + +func (c *checkpointController) GetEntityOrInterfaceType(entity string) schema.EntityOrInterface { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityOrInterfaceType(entity) +} + +func (c *checkpointController) GetEntityType(entity string) *schema.Entity { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityType(entity) +} + +func (c *checkpointController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntity(ctx, typ, id, blockNumber) +} + +func (c *checkpointController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityInBlock(ctx, typ, id, blockNumber) +} + +func (c *checkpointController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) (boxes []*persistent.EntityBox, next *string, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.ListEntity(ctx, entityType, filters, cursor, limit, blockNumber) +} + +func (c *checkpointController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.ListRelated(ctx, entityType, id, fieldName, blockNumber) +} + +func (c *checkpointController) SetEntity( + ctx context.Context, + entityType *schema.Entity, + box persistent.UncommittedEntityBox, +) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.SetEntity(ctx, entityType, box) +} + +func (c *checkpointController) unsavedSnapshot() any { + count := len(c.checkpoints) - c.savedCheckpoints + sn := map[string]any{ + "checkpointCount": count, + } + if count > 0 { + var tb uint64 + for i := c.savedCheckpoints; i < len(c.checkpoints); i++ { + tb += c.checkpoints[i].TotalBindings + } + sn["totalBindings"] = tb + sn["latestCheckpoint"] = c.checkpoints[len(c.checkpoints)-1].Snapshot() + } + return sn +} + +func (c *checkpointController) Snapshot() map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + lastSaved := c.getLatestCheckpoint(true) + watching := lastSaved != nil && lastSaved.InWatching() + return map[string]any{ + "chainID": c.chainID, + "config": map[string]any{ + "saveDelay": c.saveDelay.String(), + "saveInterval": c.saveInterval.String(), + "maxKeepCheckpointCount": c.maxKeepCheckpointCount, + "watchingDelay": WatchingDelay.String(), + }, + "checkpointCount": len(c.checkpoints), + "stores": map[string]any{ + "timeSeries": c.timeSeriesCtrl.Snapshot(), + "entity": c.entityCtrl.Snapshot(), + "webhook": c.webhookCtrl.Snapshot(), + }, + "saved": map[string]any{ + "checkpoints": utils.MapSliceNoError(c.checkpoints[:c.savedCheckpoints], Checkpoint.Snapshot), + "inWatching": watching, + "estimateWatchingNeed": c.estimateWatchingNeed(lastSaved), + "checkpointSparsifyMultiplier": c.checkpointSparsifyMultiplier, + }, + "unsaved": c.unsavedSnapshot(), + "templates": c.getTemplates(false), + "statistics": c.stat.Snapshot(), + } +} diff --git a/driver/controller/checkpoint_test.go b/driver/controller/checkpoint_test.go new file mode 100644 index 0000000..2721a38 --- /dev/null +++ b/driver/controller/checkpoint_test.go @@ -0,0 +1,183 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type testCheckpointStore struct { + checkpoints []Checkpoint + templates map[uint64][]TemplateInstance + err *ExternalError +} + +func (cs *testCheckpointStore) Load(ctx context.Context) ([]Checkpoint, map[uint64][]TemplateInstance, error) { + return cs.checkpoints, cs.templates, nil +} + +func (cs *testCheckpointStore) Save( + ctx context.Context, + checkpoints []Checkpoint, + templates map[uint64][]TemplateInstance, + agentStat map[string]int, +) error { + cs.checkpoints = checkpoints + cs.templates = templates + return nil +} + +func (cs *testCheckpointStore) SaveError(ctx context.Context, err *ExternalError) error { + cs.err = err + return nil +} + +type testBlockData struct { + BlockHeader + tasks []Task +} + +func (b testBlockData) DataSource() string { + return "test-data-source" +} + +func (b testBlockData) GetTaskList() []Task { + return b.tasks +} + +func (b testBlockData) CheckpointData() map[string]string { + return nil +} + +func (b testBlockData) Size() int { + return 1 +} + +func newSimpleTestBlockData(blockNumber uint64) testBlockData { + return newTestBlockData(newTestBlockHeader(blockNumber, "", "")) +} + +func newSimpleTestBlockDataSummary(blockNumber uint64) BlockDataSummary { + h := newTestBlockHeader(blockNumber, "", "") + return BlockDataSummary{ + BlockNumber: h.GetBlockNumber(), + BlockParentHash: h.GetBlockParentHash(), + BlockHash: h.GetBlockHash(), + BlockTime: h.GetBlockTime(), + } +} + +func newTestBlockData(header BlockHeader, tasks ...Task) testBlockData { + return testBlockData{ + BlockHeader: header, + tasks: tasks, + } +} + +func Test_save(t *testing.T) { + cs := &testCheckpointStore{} + ctx := context.Background() + cc, err := NewCheckpointController( + context.Background(), + "1", + 0, + time.Hour, + 10000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + assert.NoError(t, err) + + progressBar := ProgressBar{ + LatestBlock: newSimpleTestBlockData(1000), + } + makeCheckpoints := func(bs ...uint64) []Checkpoint { + r := make([]Checkpoint, len(bs)) + for i, bn := range bs { + r[i] = Checkpoint{ + BlockNumber: bn, + BlockTime: newSimpleTestBlockData(bn).GetBlockTime(), + LatestBlockNumber: progressBar.LatestBlock.GetBlockNumber(), + LatestBlockTime: progressBar.LatestBlock.GetBlockTime(), + } + } + return r + } + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(0), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(1), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 1), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(2), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 1, 2), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(3), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 3), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(4), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 3, 4), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(5), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 4, 5), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(6), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 5, 6), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(7), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 7), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(8), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 7, 8), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(9), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 8, 9), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(10), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 8, 9, 10), cs.checkpoints) + + for i := uint64(11); i <= 100; i++ { + // all even number has non-empty checkpoint + if i%2 == 0 { + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(i), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + } + } + assert.Equal(t, makeCheckpoints(0, 64, 80, 88, 92, 96, 98, 100), cs.checkpoints) + + for i := uint64(101); i <= 200; i++ { + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(i), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + } + assert.Equal(t, makeCheckpoints(0, 128, 160, 176, 192, 196, 198, 199, 200), cs.checkpoints) +} diff --git a/driver/controller/config.go b/driver/controller/config.go new file mode 100644 index 0000000..91ef736 --- /dev/null +++ b/driver/controller/config.go @@ -0,0 +1,31 @@ +package controller + +import ( + "time" + + "sentioxyz/sentio-core/common/envconf" +) + +var ( + ProcessConcurrency = envconf.LoadUInt64("PROCESS_CONCURRENCY", 20, + envconf.WithMax(200), envconf.WithMin(1)) + SaveCheckpointDelay = envconf.LoadDuration("SENTIO_SAVE_CHECKPOINT_DELAY", 0, + envconf.WithMinDuration(0), envconf.WithMaxDuration(time.Minute*10)) + SaveCheckpointInterval = envconf.LoadDuration("SENTIO_SAVE_CHECKPOINT_INTERVAL", time.Second*20, + envconf.WithMinDuration(time.Second)) + MaxKeepCheckpointCount = envconf.LoadUInt64("SENTIO_KEEP_CHECKPOINT_COUNT", 1000000, + envconf.WithMin(10000)) + SubscribeMinWatchInterval = envconf.LoadDuration("SENTIO_SUBSCRIBE_MIN_WATCH_INTERVAL", time.Second) + ClientMaxConcurrency = envconf.LoadUInt64("SENTIO_CLIENT_MAX_CONCURRENCY", 100, envconf.WithMin(10)) + PrintProcessedInterval = envconf.LoadDuration("SENTIO_PRINT_PROCESSED_INTERVAL", time.Second) + SkipStartBlockValidation = envconf.LoadBool("SENTIO_SKIP_START_BLOCK_VALIDATION", false) +) + +const ( + // WatchingDelay If the difference between the time of a processed block and the latest block is less than this value, + // then the block is considered to be in the watching state. + WatchingDelay = time.Minute * 5 + + // RunWaiting If a round got an external error that can be retried, wait this long before starting the next round + RunWaiting = time.Second * 30 +) diff --git a/driver/controller/data/BUILD.bazel b/driver/controller/data/BUILD.bazel new file mode 100644 index 0000000..872dff7 --- /dev/null +++ b/driver/controller/data/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "data", + srcs = [ + "block_cache.go", + "contract.go", + "errors.go", + "interval.go", + "statistics.go", + "subscribe.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data", + visibility = ["//visibility:public"], + deps = [ + "//common/envconf", + "//common/log", + "//common/queue", + "//common/timehist", + "//common/timewin", + "//common/utils", + "//common/window", + "//driver/controller", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + "@org_golang_x_sync//singleflight", + ], +) + +go_test( + name = "data_test", + srcs = [ + "block_cache_test.go", + "contract_test.go", + "statistics_test.go", + ], + embed = [":data"], + deps = [ + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/driver/controller/data/aptos/BUILD.bazel b/driver/controller/data/aptos/BUILD.bazel new file mode 100644 index 0000000..5096af5 --- /dev/null +++ b/driver/controller/data/aptos/BUILD.bazel @@ -0,0 +1,41 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "aptos", + srcs = [ + "block.go", + "change.go", + "client.go", + "resource.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/aptos", + visibility = ["//visibility:public"], + deps = [ + "//chain/aptos", + "//common/concurrency", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_aptos_labs_aptos_go_sdk//:aptos-go-sdk", + "@com_github_aptos_labs_aptos_go_sdk//api", + "@com_github_ethereum_go_ethereum//common", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "aptos_test", + srcs = ["transaction_test.go"], + embed = [":aptos"], + deps = [ + "//chain/aptos", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/aptos/block.go b/driver/controller/data/aptos/block.go new file mode 100644 index 0000000..d12e0c7 --- /dev/null +++ b/driver/controller/data/aptos/block.go @@ -0,0 +1,128 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type BlockMainData struct { + SimpleTxn *aptos.MinimalistTransaction + Txn *aptos.Transaction + Changes []Change + Intervals []data.IntervalConfig +} + +type DataRequirement struct { + Interval []data.IntervalRequirement + Changes []ChangeRequirement + Txn []TransactionRequirement +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 +} + +func (d BlockMainData) Size() int { + return len(d.Changes) + len(d.Intervals) + utils.Select(d.Txn == nil, 0, 10) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetMinimalistTransaction(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Changes = MergeChangeRequirements(currentBlockNumber, req.Changes) + req.Txn = MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Changes { + fetchers = append(fetchers, BuildChangeFetcher( + namePrefix+fmt.Sprintf("ChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // Changes and Txn never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + if box.Txn != nil { + data.Txn = box.Txn + } + if box.SimpleTxn != nil { + data.SimpleTxn = box.SimpleTxn + } + data.Changes = append(data.Changes, box.Changes...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/aptos/change.go b/driver/controller/data/aptos/change.go new file mode 100644 index 0000000..9e05af2 --- /dev/null +++ b/driver/controller/data/aptos/change.go @@ -0,0 +1,110 @@ +package aptos + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type Change struct { + Raw string + + *aptos.WriteSetChange +} + +func (e *Change) UnmarshalJSON(raw []byte) error { + if err := json.Unmarshal(raw, &e.WriteSetChange); err != nil { + return err + } + e.Raw = string(raw) + return nil +} + +type MinimalistTransactionWithChanges struct { + aptos.MinimalistTransaction `json:",inline"` + + Changes []Change `json:"changes"` +} + +type ChangeRequirement struct { + controller.BlockRange + aptos.ChangeFilter +} + +func (r ChangeRequirement) String() string { + return fmt.Sprintf("ChangeRequirement[%s]%s", r.ChangeFilter.String(), r.BlockRange.String()) +} + +func (r ChangeRequirement) Snapshot() any { + return map[string]any{ + "filter": r.ChangeFilter, + "range": r.BlockRange.String(), + } +} + +// MergeChangeRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeChangeRequirements(current uint64, reqs []ChangeRequirement) (result []ChangeRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r ChangeRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []aptos.ChangeFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.ChangeFilter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, ChangeRequirement{ + ChangeFilter: aptos.MergeChangeFilters(filters...), + BlockRange: r, + }) + } + return result +} + +func BuildChangeFetcher( + name string, + req ChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 1000000, + 100000, + 10000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetChanges(ctx, start, end, req.ChangeFilter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData, len(txs)) + for _, tx := range txs { + result[tx.Version] = BlockMainData{SimpleTxn: &tx.MinimalistTransaction, Changes: tx.Changes} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/aptos/client.go b/driver/controller/data/aptos/client.go new file mode 100644 index 0000000..68344be --- /dev/null +++ b/driver/controller/data/aptos/client.go @@ -0,0 +1,362 @@ +package aptos + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + aptossdk "github.com/aptos-labs/aptos-go-sdk" + "github.com/aptos-labs/aptos-go-sdk/api" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetTransaction(ctx context.Context, txnVersion uint64) (aptos.Transaction, error) + GetMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) + GetHeaderIgnoreCache(ctx context.Context, txnVersion uint64) (controller.BlockHeader, error) + + GetChanges( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.ChangeFilter, + ) ([]MinimalistTransactionWithChanges, error) + GetTransactions( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.TransactionFilter, + fetchConfig aptos.TransactionFetchConfig, + ) ([]aptos.Transaction, error) + // GetAccountResources key of requirement is Address, + // value is the set of ResourceType, empty set means need all Resource of the Account + GetAccountResources( + ctx context.Context, + txnVersion uint64, + requirement map[string][]string, // requirement[] is null means need all resource of + ) ([]AccountResource, error) + + GetAddressStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstTxnVersion int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + rpcCli *rpc.Client + rawCli *aptossdk.NodeClient + + cachedMinimalistTxn *data.BlockCache[MinimalistTransaction] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstTxnVersion int64, + watchLatestInterval time.Duration, +) (Client, error) { + cli := &client{ + endpoint: endpoint, + firstTxnVersion: firstTxnVersion, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + var err error + if cli.rpcCli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.rawCli, err = aptossdk.NewNodeClientWithHttpClient( + fmt.Sprintf("%s/v1", strings.TrimRight(endpoint, "/")), + 0, + https.NewClient(https.WithTimeout(time.Minute))) + if err != nil { + return nil, errors.Wrapf(err, "build aptos node client with endpoint %s failed", endpoint) + } + cli.cachedMinimalistTxn, _ = data.NewBlockCache[MinimalistTransaction](100000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) (err error) { + startAt := time.Now() + // waiting concurrency control token + var release func() + release, err = c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + waitEndAt := time.Now() + defer func() { + c.stat.Called(method, args, err, startAt, waitEndAt) + }() + if err != nil { + return err // always be context.Canceled or context.DeadlineExceeded + } + defer func() { + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + }() + defer release() + // actually call + switch method { + case "raw_getAccountResourcesAll": + txnVersion := args[0].(uint64) + address := args[1].(string) + addr := aptossdk.AccountAddress(common.HexToHash(address)) + var resources []aptossdk.AccountResourceInfo + resources, err = c.rawCli.AccountResourcesByPages(addr, txnVersion, 0) + if err != nil { + return err + } + r := result.(*[]AccountResource) + (*r), err = utils.MapSlice(resources, func(res aptossdk.AccountResourceInfo) (AccountResource, error) { + raw, marshalErr := json.Marshal(res) + if marshalErr != nil { + return AccountResource{}, marshalErr + } + return AccountResource{ + Raw: string(raw), + Address: address, + Type: res.Type, + }, nil + }) + return err + case "raw_getAccountResource": + txnVersion := args[0].(uint64) + address := args[1].(string) + resourceType := args[2].(string) + addr := aptossdk.AccountAddress(common.HexToHash(address)) + var res map[string]any + res, err = c.rawCli.AccountResource(addr, resourceType, txnVersion) + if err != nil { + return err + } + r := result.(*AccountResource) + if raw, marshalErr := json.Marshal(res); marshalErr != nil { + return marshalErr + } else { + r.Raw = string(raw) + r.Address = address + r.Type = resourceType + } + return nil + case "raw_getTxByVersion": + txnVersion := args[0].(uint64) + var tx *api.CommittedTransaction + tx, err = c.rawCli.TransactionByVersion(txnVersion) + if err != nil { + return err + } + // will return error if not found, so here tx will always be non-null + r := result.(*api.CommittedTransaction) + *r = *tx + return nil + default: + return c.rpcCli.CallContext(ctx, &result, method, args...) + } +} + +func (c *client) GetLatest(ctx context.Context) (controller.BlockHeader, uint64, error) { + var resp aptos.GetLatestMinimalistTransactionResponse + err := c.callContext(ctx, &resp, 0, "aptosV2_getLatestMinimalistTransaction", 0) + if err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest := MinimalistTransaction(resp.Transaction) + return latest, data.GetFirst(c.firstTxnVersion, latest.Version), err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, txVersionGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp aptos.GetLatestMinimalistTransactionResponse + err = c.callContext(ctx, &resp, 0, "aptosV2_getLatestMinimalistTransaction", txVersionGt) + if err == nil { + latest, broken = MinimalistTransaction(resp.Transaction), resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) GetTransaction(ctx context.Context, txnVersion uint64) (aptos.Transaction, error) { + var raw api.CommittedTransaction + if err := c.callContext(ctx, &raw, txnVersion, "raw_getTxByVersion", txnVersion); err != nil { + return aptos.Transaction{}, err + } + return aptos.NewTransaction(&raw), nil +} + +func (c *client) GetMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + // Cache + singleflight: concurrent fetchers asking for the same version share one RPC. + return c.cachedMinimalistTxn.GetOrFetch(txnVersion, func() (MinimalistTransaction, error) { + return c.fetchMinimalistTransaction(ctx, txnVersion) + }) +} + +func (c *client) fetchMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + var txn *MinimalistTransaction + if err := c.callContext(ctx, &txn, txnVersion, "aptosV2_getMinimalistTransaction", txnVersion); err != nil { + return MinimalistTransaction{}, err + } + if txn == nil { + return MinimalistTransaction{}, errors.Errorf("transaction %d not found", txnVersion) + } + return *txn, nil +} + +func (c *client) getMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + txn, err := c.fetchMinimalistTransaction(ctx, txnVersion) + if err == nil { + c.cachedMinimalistTxn.Add(txnVersion, txn) + } + return txn, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, txnVersion uint64) (controller.BlockHeader, error) { + return c.getMinimalistTransaction(ctx, txnVersion) +} + +func (c *client) GetChanges( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.ChangeFilter, +) (result []MinimalistTransactionWithChanges, err error) { + args := aptos.GetResourceChangesRequest{ + FromVersion: startTxnVersion, + ToVersion: endTxnVersion, + Filter: filter, + } + err = c.callContext(ctx, &result, startTxnVersion, "aptosV2_getResourceChanges", args) + return +} + +func (c *client) GetTransactions( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.TransactionFilter, + fetchConfig aptos.TransactionFetchConfig, +) (result []aptos.Transaction, err error) { + args := aptos.GetTransactionsRequest{ + FromVersion: startTxnVersion, + ToVersion: endTxnVersion, + Filter: filter, + FetchConfig: fetchConfig, + } + err = c.callContext(ctx, &result, startTxnVersion, "aptosV2_getTransactions", args) + return +} + +func (c *client) GetAccountResources( + ctx context.Context, + txnVersion uint64, + requirement map[string][]string, +) ([]AccountResource, error) { + if int64(txnVersion) < c.firstTxnVersion || len(requirement) == 0 { + return nil, nil + } + var result utils.SafeSlice[AccountResource] + g, gctx := errgroup.WithContext(ctx) + for address_, types := range requirement { + address := address_ + if types == nil { // need all resources + g.Go(func() error { + var resources []AccountResource + err := c.callContext(gctx, &resources, txnVersion, "raw_getAccountResourcesAll", txnVersion, address) + if err == nil { + result.Append(resources...) + } + return err + }) + } else { + for _, typ_ := range set.New(types...).DumpValues() { + typ := typ_ + g.Go(func() error { + var resource AccountResource + err := c.callContext(gctx, &resource, txnVersion, "raw_getAccountResource", txnVersion, address, typ) + if err == nil { + result.Append(resource) + } + return err + }) + } + } + } + if err := g.Wait(); err != nil { + return nil, err + } + return result.Dump(), nil +} + +func (c *client) GetAddressStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) { + var result *uint64 + err := c.callContext(ctx, &result, start, "aptosV2_getAddressStartTxVersion", address, latest) + if err != nil { + return 0, false, err + } + if result == nil { + return 0, false, nil + } + return max(*result, start), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedMinimalistTxn.Keys() { + if r.Contains(bn) { + c.cachedMinimalistTxn.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstTxnVersion": c.firstTxnVersion, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cachedMinimalistTxn": c.cachedMinimalistTxn.Snapshot(10, controller.GetBlockFullText[MinimalistTransaction]), + } +} diff --git a/driver/controller/data/aptos/resource.go b/driver/controller/data/aptos/resource.go new file mode 100644 index 0000000..1c08d76 --- /dev/null +++ b/driver/controller/data/aptos/resource.go @@ -0,0 +1,50 @@ +package aptos + +import ( + "fmt" + + "sentioxyz/sentio-core/common/utils" +) + +type AccountResourceFilter struct { + Address string + + // nil means need all resources of the account, empty means do not need any resource + // only contract move interval handler will use empty ResourceType + ResourceType []string +} + +func (c AccountResourceFilter) NeedNothing() bool { + return c.ResourceType != nil && len(c.ResourceType) == 0 +} + +func (c AccountResourceFilter) String() string { + return fmt.Sprintf("Address:%s,ResourceType:%s", c.Address, utils.ArrSummary(c.ResourceType, 10)) +} + +func (c AccountResourceFilter) Check(ar AccountResource) bool { + return c.Address == ar.Address && (c.ResourceType == nil || utils.IndexOf(c.ResourceType, ar.Type) >= 0) +} + +func MergeAccountResourceFilters(filters []AccountResourceFilter) map[string][]string { + result := make(map[string][]string) + for _, filter := range filters { + if types, has := result[filter.Address]; !has { + result[filter.Address] = filter.ResourceType + } else if types != nil { + if filter.ResourceType == nil { + result[filter.Address] = nil + } else { + result[filter.Address] = append(types, filter.ResourceType...) + } + } + } + return result +} + +type AccountResource struct { + Raw string + + Address string + Type string +} diff --git a/driver/controller/data/aptos/transaction.go b/driver/controller/data/aptos/transaction.go new file mode 100644 index 0000000..e7e88e4 --- /dev/null +++ b/driver/controller/data/aptos/transaction.go @@ -0,0 +1,140 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type MinimalistTransaction aptos.MinimalistTransaction + +func (t MinimalistTransaction) GetBlockNumber() uint64 { + return t.Version +} + +func (t MinimalistTransaction) GetBlockHash() string { + return t.Hash +} + +func (t MinimalistTransaction) GetBlockParentHash() string { + return "" +} + +func (t MinimalistTransaction) GetBlockTime() time.Time { + return time.UnixMicro(t.TimestampMS) +} + +type Transaction aptos.Transaction + +func (t *Transaction) GetBlockNumber() uint64 { + return t.Version() +} + +func (t *Transaction) GetBlockHash() string { + return t.Hash() +} + +func (t *Transaction) GetBlockParentHash() string { + return "" +} + +func (t *Transaction) GetBlockTime() time.Time { + if tm := (*aptos.Transaction)(t).Time(); tm != nil { + return *tm + } + return time.Time{} +} + +type TransactionRequirement struct { + controller.BlockRange + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (r TransactionRequirement) String() string { + return fmt.Sprintf("TransactionRequirement[Filter:[%s],FetchConfig:[%s]]%s", + r.Filter.String(), r.FetchConfig.String(), r.BlockRange.String()) +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "fetchConfig": r.FetchConfig, + "range": r.BlockRange.String(), + } +} + +func MergeTxnRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + rr := TransactionRequirement{BlockRange: r} + first := true + for _, req := range reqs { + if !req.BlockRange.Include(r) { + continue + } + if first { + rr.Filter = req.Filter + rr.FetchConfig = req.FetchConfig + first = false + } else { + rr.Filter = rr.Filter.Merge(req.Filter) + rr.FetchConfig = rr.FetchConfig.Merge(req.FetchConfig) + } + } + if first { + continue + } + result = append(result, rr) + } + return result + +} + +func BuildTxnFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 100000, + 10000, // size of transaction is 10, so will cache 1000 transactions + 5000, // the target is that each query got no more than 500 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for i := range txs { + result[txs[i].Version()] = BlockMainData{Txn: &txs[i]} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/aptos/transaction_test.go b/driver/controller/data/aptos/transaction_test.go new file mode 100644 index 0000000..b5391a2 --- /dev/null +++ b/driver/controller/data/aptos/transaction_test.go @@ -0,0 +1,44 @@ +package aptos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/chain/aptos" +) + +func Test_txJsonMarshal(t *testing.T) { + raw := aptos.MinimalistTransaction{ + Version: 123, + Hash: "abc", + TimestampMS: 123, + } + b0, err := json.Marshal(raw) + //t.Logf("%s", string(b0)) + assert.NoError(t, err) + + var tx MinimalistTransaction + assert.NoError(t, json.Unmarshal(b0, &tx)) + assert.Equal(t, raw.Version, tx.Version) + assert.Equal(t, raw.Hash, tx.Hash) + assert.Equal(t, raw.TimestampMS, tx.TimestampMS) + + b1, err := json.Marshal(tx) + //t.Logf("%s", string(b1)) + assert.NoError(t, err) + assert.Equal(t, b0, b1) +} + +func Test_diffBetweenNullAndEmptyArr(t *testing.T) { + var a []int + assert.True(t, a == nil) + assert.True(t, len(a) == 0) + a = []int{} + assert.False(t, a == nil) + assert.True(t, len(a) == 0) + a = []int{1} + assert.False(t, a == nil) + assert.False(t, len(a) == 0) +} diff --git a/driver/controller/data/block_cache.go b/driver/controller/data/block_cache.go new file mode 100644 index 0000000..d1ff989 --- /dev/null +++ b/driver/controller/data/block_cache.go @@ -0,0 +1,84 @@ +package data + +import ( + "strconv" + + "sentioxyz/sentio-core/common/utils" + + lru "github.com/sentioxyz/golang-lru" + "golang.org/x/sync/singleflight" +) + +// BlockCache is an LRU keyed by block number combined with singleflight. Chain clients prefetch +// headers/blocks concurrently from several fetchers, so the same block is frequently requested by +// multiple goroutines at nearly the same time. BlockCache caches the value and collapses concurrent +// misses for the same block into a single fetch, which keeps per-block RPCs (e.g. a header lookup) +// off the hot path and from being duplicated. +type BlockCache[V any] struct { + cache *lru.Cache[uint64, V] + sf singleflight.Group +} + +// NewBlockCache creates a BlockCache holding up to size entries. It errors only when size <= 0. +func NewBlockCache[V any](size int) (*BlockCache[V], error) { + cache, err := lru.New[uint64, V](size) + if err != nil { + return nil, err + } + return &BlockCache[V]{cache: cache}, nil +} + +// Get returns the cached value for blockNumber, if present. +func (c *BlockCache[V]) Get(blockNumber uint64) (V, bool) { + return c.cache.Get(blockNumber) +} + +// Add stores v for blockNumber, overwriting any existing entry. +func (c *BlockCache[V]) Add(blockNumber uint64, v V) { + c.cache.Add(blockNumber, v) +} + +// Remove drops blockNumber from the cache. +func (c *BlockCache[V]) Remove(blockNumber uint64) { + c.cache.Remove(blockNumber) +} + +// Keys returns the currently cached block numbers (used to evict a reorged range). +func (c *BlockCache[V]) Keys() []uint64 { + return c.cache.Keys() +} + +// GetOrFetch returns the cached value for blockNumber, or fetches it via fetch and caches the result. +// Concurrent misses for the same block are coalesced into a single fetch. fetch is not invoked even +// when a caller arrives just after an in-flight fetch finished: the cache is re-checked inside the +// flight, so the just-fetched value is reused rather than fetched again. +func (c *BlockCache[V]) GetOrFetch(blockNumber uint64, fetch func() (V, error)) (V, error) { + // Fast path: avoids the strconv + singleflight bookkeeping when the block is already cached. + if v, ok := c.cache.Get(blockNumber); ok { + return v, nil + } + v, err, _ := c.sf.Do(strconv.FormatUint(blockNumber, 10), func() (any, error) { + // Re-check inside the flight (double-checked): a preceding flight for this block may have + // finished and populated the cache between the miss above and entering Do. + if v, ok := c.cache.Get(blockNumber); ok { + return v, nil + } + v, err := fetch() + if err != nil { + return nil, err + } + c.cache.Add(blockNumber, v) + return v, nil + }) + if err != nil { + var zero V + return zero, err + } + return v.(V), nil +} + +// Snapshot renders up to maxCount entries for the debug tracker, using valuePreview to stringify each +// value. It mirrors utils.CacheSnapshot so callers don't need to reach for the underlying cache. +func (c *BlockCache[V]) Snapshot(maxCount int, valuePreview func(V) string) any { + return utils.CacheSnapshot(c.cache, maxCount, valuePreview) +} diff --git a/driver/controller/data/block_cache_test.go b/driver/controller/data/block_cache_test.go new file mode 100644 index 0000000..b3dde55 --- /dev/null +++ b/driver/controller/data/block_cache_test.go @@ -0,0 +1,67 @@ +package data + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlockCacheGetOrFetch(t *testing.T) { + c, err := NewBlockCache[uint64](1024) + require.NoError(t, err) + + var fetches atomic.Int64 + fetch := func() (uint64, error) { + fetches.Add(1) + return 70, nil + } + + // Concurrent callers that all miss the same block must collapse into a single fetch. Run under + // -race, this also guards that BlockCache itself has no internal data race. + const concurrent = 50 + results := make([]uint64, concurrent) + errs := make([]error, concurrent) + var wg sync.WaitGroup + for i := 0; i < concurrent; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + results[i], errs[i] = c.GetOrFetch(7, fetch) + }() + } + wg.Wait() + + for i := 0; i < concurrent; i++ { + require.NoError(t, errs[i]) + require.Equal(t, uint64(70), results[i]) + } + require.EqualValues(t, 1, fetches.Load(), "concurrent misses for the same block fetch once") + + // A later call hits the cache without fetching again. + v, err := c.GetOrFetch(7, fetch) + require.NoError(t, err) + require.Equal(t, uint64(70), v) + require.EqualValues(t, 1, fetches.Load()) +} + +func TestBlockCacheGetOrFetchError(t *testing.T) { + c, err := NewBlockCache[uint64](16) + require.NoError(t, err) + + boom := errors.New("boom") + _, err = c.GetOrFetch(1, func() (uint64, error) { return 0, boom }) + require.ErrorIs(t, err, boom) + + // Errors are not cached: a subsequent successful fetch for the same block still runs and caches. + v, err := c.GetOrFetch(1, func() (uint64, error) { return 42, nil }) + require.NoError(t, err) + require.Equal(t, uint64(42), v) + + got, ok := c.Get(1) + require.True(t, ok) + require.Equal(t, uint64(42), got) +} diff --git a/driver/controller/data/contract.go b/driver/controller/data/contract.go new file mode 100644 index 0000000..77912fc --- /dev/null +++ b/driver/controller/data/contract.go @@ -0,0 +1,53 @@ +package data + +import ( + "context" + + "sentioxyz/sentio-core/common/utils" +) + +func BinarySearchContractStart( + ctx context.Context, + start, end uint64, + checker func(context.Context, uint64) (bool, error), +) (uint64, bool, error) { + if has, err := checker(ctx, end); err != nil { + return 0, false, err + } else if !has { + return 0, false, nil + } + low, high := start, end + for p := uint64(1); p > 0; p <<= 1 { + if p <= start { + low = p + } + if p >= end { + high = p + break + } + } + internalChecker := func(ctx context.Context, bn uint64) (bool, error) { + if bn < start { + return false, nil + } + if bn >= end { + return true, nil + } + return checker(ctx, bn) + } + for low < high { + mid := (low + high) / 2 + if has, err := internalChecker(ctx, mid); err != nil { + return 0, false, err + } else if has { + high = mid + } else { + low = mid + 1 + } + } + return low, true, nil +} + +func GetFirst(firstConfig int64, latest uint64) uint64 { + return uint64(max(0, utils.Select(firstConfig < 0, int64(latest)+firstConfig, firstConfig))) +} diff --git a/driver/controller/data/contract_test.go b/driver/controller/data/contract_test.go new file mode 100644 index 0000000..574624a --- /dev/null +++ b/driver/controller/data/contract_test.go @@ -0,0 +1,30 @@ +package data + +import ( + "context" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func Test_BinarySearchContractStart(t *testing.T) { + const startBlock = 500 + checker := func(_ context.Context, bn uint64) (bool, error) { + if bn < 100 || bn > 1000 { + return false, errors.Errorf("out of range") + } + return bn >= startBlock, nil + } + + for s := uint64(100); s <= 1000; s++ { + for e := s; e <= 1000; e++ { + n, has, err := BinarySearchContractStart(context.Background(), s, e, checker) + assert.NoError(t, err) + assert.Equalf(t, startBlock <= e, has, "s=%d,e=%d", s, e) + if has { + assert.Equalf(t, max(startBlock, s), n, "s=%d,e=%d", s, e) + } + } + } +} diff --git a/driver/controller/data/errors.go b/driver/controller/data/errors.go new file mode 100644 index 0000000..e667adf --- /dev/null +++ b/driver/controller/data/errors.go @@ -0,0 +1,46 @@ +package data + +import ( + "errors" + "fmt" +) + +// NewClientRetryableError marks a failure returned by a chain data client constructor (the NewClient +// functions under data/, e.g. data/sol.NewClient) that is transient and worth retrying, +// rather than a permanent misconfiguration. For example, a data endpoint that keeps returning +// HTTP/timeout errors while probing its capabilities may simply be temporarily unavailable; failing +// permanently would strand the processor, whereas retrying (by restarting the pod) gives it a fresh +// chance once the endpoint recovers. +// +// This package intentionally knows nothing about how callers act on the error; it only signals that +// the NewClient failure is retryable. The startup controller (which depends on this package) inspects +// construction errors with IsNewClientRetryable / errors.As and chooses to restart the pod instead of +// failing permanently when it matches. +type NewClientRetryableError struct { + // Reason is a human-readable description of why client construction failed. + Reason string + // Err is the underlying error that triggered the retryable failure, if any. + Err error +} + +// NewClientRetryable wraps err into a NewClientRetryableError with the given reason. +func NewClientRetryable(reason string, err error) *NewClientRetryableError { + return &NewClientRetryableError{Reason: reason, Err: err} +} + +func (e *NewClientRetryableError) Error() string { + if e.Err != nil { + return fmt.Sprintf("retryable NewClient error: %s: %v", e.Reason, e.Err) + } + return fmt.Sprintf("retryable NewClient error: %s", e.Reason) +} + +func (e *NewClientRetryableError) Unwrap() error { + return e.Err +} + +// IsNewClientRetryable reports whether err is, or wraps, a *NewClientRetryableError. +func IsNewClientRetryable(err error) bool { + var target *NewClientRetryableError + return errors.As(err, &target) +} diff --git a/driver/controller/data/evm/BUILD.bazel b/driver/controller/data/evm/BUILD.bazel new file mode 100644 index 0000000..4671065 --- /dev/null +++ b/driver/controller/data/evm/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "evm", + srcs = [ + "block.go", + "block_extend.go", + "block_main.go", + "client.go", + "log.go", + "trace.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/evm", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//common/concurrency", + "//common/contract", + "//common/envconf", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//:go-ethereum", + "@com_github_ethereum_go_ethereum//common/hexutil", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + ], +) + +go_test( + name = "evm_test", + srcs = ["log_test.go"], + embed = [":evm"], + deps = [ + "//common/set", + "//driver/controller", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/evm/block.go b/driver/controller/data/evm/block.go new file mode 100644 index 0000000..9da1fc3 --- /dev/null +++ b/driver/controller/data/evm/block.go @@ -0,0 +1,67 @@ +package evm + +import ( + "encoding/json" + "math" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +type BlockHeader struct { + Raw json.RawMessage + + BlockNumber uint64 + BlockHash string + BlockTime time.Time + ParentBlockHash string + TxHashes []string +} + +func (b *BlockHeader) UnmarshalJSON(raw []byte) error { + var payload *struct { + Number hexutil.Uint64 `json:"number"` + Hash string `json:"hash"` + Timestamp hexutil.Uint64 `json:"timestamp"` + ParentHash string `json:"parentHash"` + Transactions []string `json:"transactions"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return err + } + if payload == nil { + b = nil + return nil + } + b.Raw = raw + b.BlockNumber = uint64(payload.Number) + b.BlockHash = payload.Hash + b.ParentBlockHash = payload.ParentHash + b.TxHashes = payload.Transactions + if payload.Timestamp < math.MaxInt32 { + b.BlockTime = time.Unix(int64(payload.Timestamp), 0) + } else if payload.Timestamp < math.MaxInt32*1000 { + b.BlockTime = time.UnixMilli(int64(payload.Timestamp)) + } else if payload.Timestamp < math.MaxInt32*1000000 { + b.BlockTime = time.UnixMicro(int64(payload.Timestamp)) + } else { + b.BlockTime = time.Unix(0, int64(payload.Timestamp)) + } + return nil +} + +func (b BlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b BlockHeader) GetBlockParentHash() string { + return b.ParentBlockHash +} + +func (b BlockHeader) GetBlockHash() string { + return b.BlockHash +} + +func (b BlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} diff --git a/driver/controller/data/evm/block_extend.go b/driver/controller/data/evm/block_extend.go new file mode 100644 index 0000000..3e0668c --- /dev/null +++ b/driver/controller/data/evm/block_extend.go @@ -0,0 +1,63 @@ +package evm + +import ( + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/utils" +) + +type BlockExtendData struct { + Transactions map[string]evm.RPCTransaction + Receipts map[string]evm.ExtendedReceipt + Traces map[string][]Trace +} + +type BlockExtendRequirement struct { + AllTransactions bool + SpecialTransactions []string + + AllTransactionReceipts bool + SpecialTransactionReceipts []string + + AllTransactionReceiptLogs bool + SpecialTransactionReceiptLogs []string + + AllTraces bool +} + +func (r *BlockExtendRequirement) Merge(a BlockExtendRequirement) { + r.AllTransactions = r.AllTransactions || a.AllTransactions + r.AllTransactionReceipts = r.AllTransactionReceipts || a.AllTransactionReceipts + r.AllTransactionReceiptLogs = r.AllTransactionReceiptLogs || a.AllTransactionReceiptLogs + r.AllTraces = r.AllTraces || a.AllTraces + r.SpecialTransactions = append(r.SpecialTransactions, a.SpecialTransactions...) + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, a.SpecialTransactionReceipts...) + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, a.SpecialTransactionReceiptLogs...) + r.Trim() +} + +func (r *BlockExtendRequirement) IsEmpty() bool { + return !r.AllTransactions && len(r.SpecialTransactions) == 0 && + !r.AllTransactionReceipts && len(r.SpecialTransactionReceipts) == 0 && + !r.AllTransactionReceiptLogs && len(r.SpecialTransactionReceiptLogs) == 0 && + !r.AllTraces +} + +func (r *BlockExtendRequirement) Trim() { + if r.AllTransactions { + r.SpecialTransactions = nil + } else { + r.SpecialTransactions = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactions)) + } + + if r.AllTransactionReceipts { + r.SpecialTransactionReceipts = nil + } else { + r.SpecialTransactionReceipts = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactionReceipts)) + } + + if r.AllTransactionReceiptLogs { + r.SpecialTransactionReceiptLogs = nil + } else { + r.SpecialTransactionReceiptLogs = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactionReceiptLogs)) + } +} diff --git a/driver/controller/data/evm/block_main.go b/driver/controller/data/evm/block_main.go new file mode 100644 index 0000000..3e70ed8 --- /dev/null +++ b/driver/controller/data/evm/block_main.go @@ -0,0 +1,127 @@ +package evm + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type DataRequirement struct { + Log []LogRequirement + Trace []TraceRequirement + Interval []data.IntervalRequirement + Exact []uint64 +} + +type BlockMainData struct { + Logs []types.Log + Traces []Trace + Intervals []data.IntervalConfig + Exact bool +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 && !d.Exact +} + +func (d BlockMainData) Size() int { + return len(d.Logs) + len(d.Traces) + len(d.Intervals) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetHeader(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 100, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Log = MergeLogRequirements(currentBlockNumber, req.Log) + req.Trace = MergeTraceRequirement(currentBlockNumber, req.Trace) + req.Interval = data.MergeIntervalRequirements(req.Interval) + exact := set.New(req.Exact...) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Log { + fetchers = append(fetchers, BuildLogFetcher( + namePrefix+fmt.Sprintf("LogFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Trace { + fetchers = append(fetchers, BuildTraceFetcher( + namePrefix+fmt.Sprintf("TraceFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + data.Exact = exact.Contains(bn) + has = len(from) > 0 || data.Exact + // Logs and traces will never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Logs = append(data.Logs, box.Logs...) + data.Traces = append(data.Traces, box.Traces...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/evm/client.go b/driver/controller/data/evm/client.go new file mode 100644 index 0000000..590577e --- /dev/null +++ b/driver/controller/data/evm/client.go @@ -0,0 +1,584 @@ +package evm + +import ( + "context" + "math/big" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/contract" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + lru "github.com/sentioxyz/golang-lru" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetBlock( + ctx context.Context, + blockNumber uint64, + req BlockExtendRequirement, + ) (BlockExtendData, error) + + GetLogs( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + topics [][]string, + ) ([]types.Log, error) + + GetTraces( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + ) ([]Trace, error) + + GetContractStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) + IsERC20Address(ctx context.Context, address string) (bool, error) + GetChainID(ctx context.Context) (uint64, error) + CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + latestDelayBlockNumber uint64 + watchLatestInterval time.Duration + getLatestTimeout time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + unsupportedMethods set.Set[string] + + cachedHeaders *data.BlockCache[BlockHeader] + cachedERC20AddrCheckResult *lru.Cache[string, bool] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + latestDelayBlockNumber uint64, + watchLatestInterval time.Duration, + getLatestTimeout time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + latestDelayBlockNumber: latestDelayBlockNumber, + watchLatestInterval: watchLatestInterval, + getLatestTimeout: getLatestTimeout, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + unsupportedMethods: set.NewSafe[string](), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[BlockHeader](10000) + cli.cachedERC20AddrCheckResult, _ = lru.New[string, bool](100000) + return cli, nil +} + +// fullBlockFetchThreshold is the number of special transactions/receipts above which it is +// cheaper to fetch the whole block in a single request than to issue one request per hash. +var fullBlockFetchThreshold = envconf.LoadUInt64("SENTIO_EVM_FULL_BLOCK_FETCH_THRESHOLD", 10) + +var ( + errMethodNotSupported = errors.New("method not supported") + + invalidMethodErrorMatcher = []*regexp.Regexp{ + regexp.MustCompile(`unsupported method`), + regexp.MustCompile("method.*not available"), + regexp.MustCompile(`method.*not support`), + regexp.MustCompile(`method.*not found`), + regexp.MustCompile(`method.*not allowed`), + } +) + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + if utils.MatchAny(strings.ToLower(err.Error()), invalidMethodErrorMatcher) { + err = errors.Wrapf(errMethodNotSupported, err.Error()) + } else { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + _, logger := log.FromContext(ctx) + for { + var resp evm.GetLatestBlockNumberResponse + callCtx, cancel := context.WithTimeout(ctx, time.Minute) + err := c.callContext(callCtx, &resp, 0, "eth_getLatestBlockNumber", 0) + cancel() + if err == nil { + break + } + if errors.Is(err, errMethodNotSupported) { + logger.Warn("do not support eth_getLatestBlockNumber, will use polling mode with method eth_blockNumber") + data.SubscribeUsingPolling( + ctx, + c.watchLatestInterval, + c.getLatestTimeout, + from, + func(ctx context.Context) (h controller.BlockHeader, err error) { + h, _, err = c.GetLatest(ctx) + return h, err + }, + callback) + return + } + logger.Warnfe(err, "call eth_getLatestBlockNumber failed, will retry after %s", c.watchLatestInterval.String()) + select { + case <-time.After(c.watchLatestInterval): + case <-ctx.Done(): + return + } + } + // have eth_getLatestBlockNumber method, use wait latest mode + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp evm.GetLatestBlockNumberResponse + err = c.callContext(ctx, &resp, 0, "eth_getLatestBlockNumber", blockNumberGt+c.latestDelayBlockNumber) + if err == nil { + if broken = resp.CheckAPIVersion(); broken == nil { + latest, err = c.GetHeaderIgnoreCache(ctx, resp.LatestBlockNumber-c.latestDelayBlockNumber) + } else { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + } + return + }, + callback) +} + +func (c *client) GetLatest(ctx context.Context) (controller.BlockHeader, uint64, error) { + var result hexutil.Uint64 + if err := c.callContext(ctx, &result, 0, "eth_blockNumber"); err != nil { + return BlockHeader{}, 0, err + } + latest := uint64(result) + latest -= min(c.latestDelayBlockNumber, latest) + h, err := c.GetHeaderIgnoreCache(ctx, latest) + return h, data.GetFirst(c.firstBlockNumber, latest), err +} + +func (c *client) GetHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one eth_getBlockByNumber. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (BlockHeader, error) { + return c.fetchHeader(ctx, blockNumber) + }) +} + +func (c *client) fetchHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + var h *BlockHeader + err := c.callContext(ctx, &h, blockNumber, "eth_getBlockByNumber", hexutil.Uint64(blockNumber), false) + if err != nil { + return BlockHeader{}, errors.Wrapf(err, "failed to get header of block %d", blockNumber) + } + if h == nil { + return BlockHeader{}, errors.Errorf("block %d not found", blockNumber) + } + return *h, nil +} + +func (c *client) getHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + h, err := c.fetchHeader(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, h) + } + return h, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getHeaderIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetBlock( + ctx context.Context, + blockNumber uint64, + req BlockExtendRequirement, +) (r BlockExtendData, err error) { + if req.IsEmpty() { + return + } + r = BlockExtendData{ + Transactions: make(map[string]evm.RPCTransaction), + Receipts: make(map[string]evm.ExtendedReceipt), + Traces: make(map[string][]Trace), + } + g, gctx := errgroup.WithContext(ctx) + var lock sync.Mutex + // get transactions + // fetchFullBlockTxs fetches the whole block in a single request. When keep is nil every + // transaction is kept (AllTransactions); otherwise only the requested hashes are kept and a + // missing one is treated as an error, matching the per-hash behavior. + fetchFullBlockTxs := func(keep set.Set[string]) func() error { + return func() (err error) { + var block *struct { + Transactions []evm.RPCTransaction `json:"transactions"` + } + err = c.callContext(ctx, &block, blockNumber, "eth_getBlockByNumber", hexutil.Uint64(blockNumber), true) + if err != nil { + return + } + found := set.New[string]() + if block != nil { + for _, tx := range block.Transactions { + hash := tx.Hash.String() + if keep != nil && !keep.Contains(hash) { + continue + } + r.Transactions[hash] = tx + found.Add(hash) + } + } + if keep != nil { + for _, txHash := range req.SpecialTransactions { + if !found.Contains(txHash) { + return errors.Errorf("transaction %d/%s not found", blockNumber, txHash) + } + } + } + return + } + } + switch { + case req.AllTransactions: + g.Go(fetchFullBlockTxs(nil)) + case uint64(len(req.SpecialTransactions)) >= fullBlockFetchThreshold: + // too many special transactions, one full-block request is cheaper than one per hash + g.Go(fetchFullBlockTxs(set.New(req.SpecialTransactions...))) + default: + for _, txHash_ := range req.SpecialTransactions { + txHash := txHash_ + g.Go(func() (err error) { + var tx *evm.RPCTransaction + if err = c.callContext(ctx, &tx, blockNumber, "eth_getTransactionByHash", txHash); err != nil { + return + } + if tx == nil { + return errors.Errorf("transaction %d/%s not found", blockNumber, txHash) + } + lock.Lock() + defer lock.Unlock() + r.Transactions[txHash] = *tx + return + }) + } + } + // get receipts + // fetchFullBlockReceipts fetches all receipts of the block in a single request. When keep is + // nil every receipt is kept (AllTransactionReceipts); otherwise only the requested hashes are + // kept and a missing one is treated as an error, matching the per-hash behavior. + fetchFullBlockReceipts := func(keep set.Set[string]) func() error { + return func() (err error) { + var receipts []evm.ExtendedReceipt + if receipts, err = c.GetBlockReceipts(gctx, blockNumber); err != nil { + return + } + found := set.New[string]() + for _, receipt := range receipts { + hash := receipt.TxHash.String() + if keep != nil && !keep.Contains(hash) { + continue + } + r.Receipts[hash] = receipt + found.Add(hash) + } + if keep != nil { + for _, txHash := range req.SpecialTransactionReceipts { + if !found.Contains(txHash) { + return errors.Errorf("transaction receipt %d/%s not found", blockNumber, txHash) + } + } + } + return + } + } + switch { + case req.AllTransactionReceipts: + g.Go(fetchFullBlockReceipts(nil)) + case uint64(len(req.SpecialTransactionReceipts)) >= fullBlockFetchThreshold && + !c.unsupportedMethods.Contains("eth_getBlockReceipts"): + // too many special receipts, one eth_getBlockReceipts request is cheaper than one per hash. + // only take this path when eth_getBlockReceipts is supported, otherwise GetBlockReceipts + // falls back to one request per transaction in the block, which may be even more requests. + g.Go(fetchFullBlockReceipts(set.New(req.SpecialTransactionReceipts...))) + default: + for _, txHash_ := range req.SpecialTransactionReceipts { + txHash := txHash_ + g.Go(func() (err error) { + var receipt *evm.ExtendedReceipt + if err = c.callContext(gctx, &receipt, blockNumber, "eth_getTransactionReceipt", txHash); err != nil { + return + } + if receipt == nil { + return errors.Errorf("transaction receipt %d/%s not found", blockNumber, txHash) + } + lock.Lock() + defer lock.Unlock() + r.Receipts[txHash] = *receipt + return + }) + } + } + // get traces + if req.AllTraces { + g.Go(func() (err error) { + var traces []Trace + traces, err = c.GetTraces(gctx, blockNumber, blockNumber, nil) + if err != nil { + return + } + r.Traces = utils.Group(traces, func(trace Trace) string { + return trace.TransactionHash + }) + return + }) + } + err = g.Wait() + // check receipt logs + if err == nil && !req.AllTransactionReceiptLogs { + need := set.New(req.SpecialTransactionReceiptLogs...) + for txHash, receipt := range r.Receipts { + if !need.Contains(txHash) { + receipt.Logs = nil + r.Receipts[txHash] = receipt + } + } + } + if err == nil { + _, logger := log.FromContext(ctx) + logger.Debugw("GetBlock succeed", "blockNumber", blockNumber, "req", req, "result", r) + } + return +} + +func (c *client) GetBlockReceipts(ctx context.Context, blockNumber uint64) (r []evm.ExtendedReceipt, err error) { + if !c.unsupportedMethods.Contains("eth_getBlockReceipts") { + err = c.callContext(ctx, &r, blockNumber, "eth_getBlockReceipts", hexutil.Uint64(blockNumber)) + if err == nil || !errors.Is(err, errMethodNotSupported) { + return + } + c.unsupportedMethods.Add("eth_getBlockReceipts") + } + var h BlockHeader + if h, err = c.GetHeader(ctx, blockNumber); err != nil { + return + } + g, gctx := errgroup.WithContext(ctx) + r = make([]evm.ExtendedReceipt, len(h.TxHashes)) + for i_, txHash_ := range h.TxHashes { + i, txHash := i_, txHash_ + g.Go(func() error { + return c.callContext(gctx, &r[i], blockNumber, "eth_getTransactionReceipt", txHash) + }) + } + err = g.Wait() + return +} + +func (c *client) GetLogs( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + topics [][]string, +) (result []types.Log, err error) { + arg := map[string]any{ + "fromBlock": hexutil.Uint64(fromBlock).String(), + "toBlock": hexutil.Uint64(toBlock).String(), + "address": address, + "topics": topics, + } + err = c.callContext(ctx, &result, fromBlock, "eth_getLogs", arg) + return +} + +func (c *client) GetTraces( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, +) (result []Trace, err error) { + arg := map[string]any{ + "fromBlock": hexutil.Uint64(fromBlock).String(), + "toBlock": hexutil.Uint64(toBlock).String(), + "toAddress": address, + } + err = c.callContext(ctx, &result, fromBlock, "trace_filter", arg) + if err != nil { + return + } + return +} + +func (c *client) HasCode(ctx context.Context, address string, blockNumber uint64) (bool, error) { + var result hexutil.Bytes + err := c.callContext(ctx, &result, blockNumber, "eth_getCode", address, hexutil.Uint64(blockNumber)) + return len(result) > 0, err +} + +func (c *client) GetContractStartBlock( + ctx context.Context, + address string, + start, latest uint64, +) (uint64, bool, error) { + return data.BinarySearchContractStart(ctx, start, latest, func(ctx context.Context, bn uint64) (bool, error) { + return c.HasCode(ctx, address, bn) + }) +} + +func (c *client) IsERC20Address(ctx context.Context, address string) (bool, error) { + if is, has := c.cachedERC20AddrCheckResult.Get(address); has { + return is, nil + } + is, err := c.IsERC20AddressIgnoreCache(ctx, address) + if err != nil { + return false, err + } + c.cachedERC20AddrCheckResult.Add(address, is) + return is, nil +} + +func (c *client) IsERC20AddressIgnoreCache(ctx context.Context, address string) (bool, error) { + // TODO need improvement + const endpoint = "https://eth-mainnet.g.alchemy.com/v2/z1Q-YhcYg60C5sOQPUzsMFqiDJSvqbsK" + res, err := contract.IsERC20(ctx, endpoint, address) + if err != nil { + return false, errors.Wrapf(err, "detect address %s is erc20 failed", address) + } + return res, err +} + +func (c *client) GetChainID(ctx context.Context) (uint64, error) { + var chainID hexutil.Uint64 + if err := c.callContext(ctx, &chainID, 0, "eth_chainId"); err != nil { + return 0, err + } + return uint64(chainID), nil +} + +// CallContract Referenced github.com/ethereum/go-ethereum/ethclient.Client.CallContract +func (c *client) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + if blockNumber.Sign() < 0 { + return nil, errors.New("block number for eth_call cannot use tag") + } + arg := map[string]interface{}{ + "from": msg.From, + "to": msg.To, + } + if len(msg.Data) > 0 { + arg["input"] = hexutil.Bytes(msg.Data) + } + if msg.Value != nil { + arg["value"] = (*hexutil.Big)(msg.Value) + } + if msg.Gas != 0 { + arg["gas"] = hexutil.Uint64(msg.Gas) + } + if msg.GasPrice != nil { + arg["gasPrice"] = (*hexutil.Big)(msg.GasPrice) + } + if msg.GasFeeCap != nil { + arg["maxFeePerGas"] = (*hexutil.Big)(msg.GasFeeCap) + } + if msg.GasTipCap != nil { + arg["maxPriorityFeePerGas"] = (*hexutil.Big)(msg.GasTipCap) + } + if msg.AccessList != nil { + arg["accessList"] = msg.AccessList + } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } + var bn = blockNumber.Uint64() + var hex hexutil.Bytes + err := c.callContext(ctx, &hex, bn, "eth_call", arg, hexutil.Uint64(bn).String()) + return hex, err +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "latestDelayBlockNumber": c.latestDelayBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + "getLatestTimeout": c.getLatestTimeout.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "unsupportedMethods": c.unsupportedMethods.DumpValues(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, controller.GetBlockFullText[BlockHeader]), + "cachedERC20AddrCheckResult": utils.CacheSnapshot(c.cachedERC20AddrCheckResult, 100, strconv.FormatBool), + }, + } +} diff --git a/driver/controller/data/evm/log.go b/driver/controller/data/evm/log.go new file mode 100644 index 0000000..2b19296 --- /dev/null +++ b/driver/controller/data/evm/log.go @@ -0,0 +1,238 @@ +package evm + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// LogFilter has 2 parts, there are linked by AND +type LogFilter struct { + // topic condition + Topics [][]string + // address condition + Address []string + AddressShouldBeERC20 bool +} + +// FilterLogs all filters is linked by OR +func FilterLogs(ctx context.Context, cli Client, logs []types.Log, filters ...LogFilter) ([]types.Log, error) { + checkers := utils.MapSliceNoError(filters, func(f LogFilter) func(log types.Log) (bool, error) { + return f.BuildChecker(ctx, cli) + }) + return utils.FilterArrWithErr(logs, func(log types.Log) (bool, error) { + for _, ck := range checkers { + if fok, err := ck(log); err != nil { + return false, err + } else if fok { + return true, nil + } + } + return false, nil + }) +} + +func (f LogFilter) BuildChecker(ctx context.Context, cli Client) func(log types.Log) (bool, error) { + addrSet := set.New(f.Address...) + topicsSet := utils.MapSliceNoError(f.Topics, func(ss []string) set.Set[string] { + return set.New(ss...) + }) + return func(log types.Log) (bool, error) { + for i, topic := range log.Topics { + if i < len(topicsSet) && !topicsSet[i].Empty() && !topicsSet[i].Contains(topic.String()) { + return false, nil + } + } + if !addrSet.Empty() && !addrSet.Contains(strings.ToLower(log.Address.String())) { + return false, nil + } + if f.AddressShouldBeERC20 { + return cli.IsERC20Address(ctx, log.Address.String()) + } + return true, nil + } +} + +func (f LogFilter) String() string { + return fmt.Sprintf("Topics:[%s],Addr:[%s],AddrIsERC20:%v", + strings.Join(utils.MapSliceNoError(f.Topics, func(t []string) string { + if len(t) == 0 { + return "nil" + } + return "[" + strings.Join(t, ",") + "]" + }), ","), + utils.ArrSummary(f.Address, 10), + f.AddressShouldBeERC20, + ) +} + +// Merge logs match f always match r, logs match a also always match r. Logs(r) >= Logs(f) + Logs(a) +func (f LogFilter) Merge(a LogFilter) (r LogFilter) { + r.Topics = make([][]string, min(len(f.Topics), len(a.Topics))) + for i := 0; i < len(r.Topics); i++ { + if len(f.Topics[i]) > 0 && len(a.Topics[i]) > 0 { + r.Topics[i] = set.SmartNew[string](f.Topics[i], a.Topics[i]).DumpValues() + } + } + if len(f.Address) > 0 && len(a.Address) > 0 { + r.Address = set.SmartNew[string](f.Address, a.Address).DumpValues() + } + r.AddressShouldBeERC20 = f.AddressShouldBeERC20 && a.AddressShouldBeERC20 + return r +} + +func MergeLogFilers(filters []LogFilter) (r LogFilter) { + if len(filters) == 0 { + panic("filters is empty") + } + // Topics + for i := 0; ; i++ { + miss := false + s := set.New[string]() + for _, f := range filters { + if len(f.Topics) <= i { + miss = true + break + } else if len(f.Topics[i]) == 0 { + s = set.New[string]() + break + } else { + s.Add(f.Topics[i]...) + } + } + if miss { + break + } + r.Topics = append(r.Topics, s.DumpValues()) + } + // Address + s := set.New[string]() + for _, f := range filters { + if len(f.Address) == 0 { + s = set.New[string]() + break + } + s.Add(f.Address...) + } + r.Address = s.DumpValues() + // AddressShouldBeERC20 + r.AddressShouldBeERC20 = true + for _, f := range filters { + if !f.AddressShouldBeERC20 { + r.AddressShouldBeERC20 = false + break + } + } + return r +} + +type LogRequirement struct { + controller.BlockRange + LogFilter +} + +func (r LogRequirement) String() string { + return fmt.Sprintf("LogRequirement[%s]%s", r.LogFilter.String(), r.BlockRange.String()) +} + +func (r LogRequirement) Snapshot() any { + return map[string]any{ + "filter": r.LogFilter, + "range": r.BlockRange.String(), + } +} + +// MergeLogRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeLogRequirements(current uint64, reqs []LogRequirement) (result []LogRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r LogRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []LogFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.LogFilter) + } + } + if len(filters) == 0 { + continue + } + + result = append(result, LogRequirement{ + LogFilter: MergeLogFilers(filters), + BlockRange: r, + }) + } + return result +} + +var ( + ethGetLogsMaxAddressLen = envconf.LoadUInt64("SENTIO_ETH_GETLOGS_MAX_ADDR_LEN", 200) + ethGetLogsMaxTopicLen = envconf.LoadUInt64("SENTIO_ETH_GETLOGS_MAX_TOPIC_LEN", 100) +) + +func BuildLogFetcher( + name string, + req LogRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10, + 1000, + 10000, + 2000, // the target is that each query got no more than 2000 logs + time.Second*5, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + address := req.Address + if uint64(len(req.Address)) > ethGetLogsMaxAddressLen { + address = nil // the set is too big + } + topics := make([][]string, len(req.Topics)) + for i, s := range req.Topics { + if uint64(len(s)) > ethGetLogsMaxTopicLen { + topics[i] = nil // the set is too big + } else { + topics[i] = s + } + } + allLogs, err := client.GetLogs(ctx, start, end, address, topics) + if err != nil { + return nil, err + } + allLogs, err = FilterLogs(ctx, client, allLogs, req.LogFilter) + if err != nil { + return nil, err + } + blockLogs := utils.Group(allLogs, func(log types.Log) uint64 { + return log.BlockNumber + }) + result := make(map[uint64]BlockMainData) + for bn, logs := range blockLogs { + result[bn] = BlockMainData{Logs: logs} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/evm/log_test.go b/driver/controller/data/evm/log_test.go new file mode 100644 index 0000000..8d50a51 --- /dev/null +++ b/driver/controller/data/evm/log_test.go @@ -0,0 +1,90 @@ +package evm + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" +) + +func Test_logFilter(t *testing.T) { + var ev types.Log + evRaw := `{ + "address": "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x000000000000000000000000fae23c30d383df59d3e031c325a73d454e8721a6" + ], + "data": "0x000000000000000000000000000000000000000000000000000000003b9aca00", + "blockNumber": "0xd8110d", + "transactionHash": "0xa4cc1d25099cc2f8fc18ae8a54e079076a89db38dad54696f7453afa3adcfbe8", + "transactionIndex": "0x8c", + "blockHash": "0x5b22e991900f1e961ff296dd37973279a91dc97ecdb2ae290680e512b3dec7fe", + "logIndex": "0xaf", + "removed": false + }` + assert.NoError(t, json.Unmarshal([]byte(evRaw), &ev)) + f := LogFilter{ + Topics: [][]string{ + { + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + }, + }, + Address: []string{ + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + }, + } + ok, err := f.BuildChecker(nil, nil)(ev) + assert.NoError(t, err) + assert.True(t, ok) +} + +func Test_MergeLogRequirements(t *testing.T) { + var reqs []LogRequirement + var addrs []string + for i := 0; i < 20000; i++ { + addr := fmt.Sprintf("0x%040x", i) + addrs = append(addrs, addr) + reqs = append(reqs, LogRequirement{ + BlockRange: controller.BlockRange{StartBlock: uint64(i * 1000)}, + LogFilter: LogFilter{ + Topics: [][]string{{ + "0x503ec09c1b4597d114eca849f7013c8c457988802d1dc5da49a0c461b5f88658", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x53a0f15ecd2c4989821244d3c7363d7de154e125eac5b2a7c9bb008152778de1", + "0xd51a9c61267aa6196961883ecf5ff2da6619c37dac0fa92122513fb32c032d2d", + "0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0", + "0x043a568d47b1a65cdc989ff14c921411b62abf40132dc4b5e78675ba8d0bc9df", + }}, + Address: []string{addr}, + }, + }) + } + r := MergeLogRequirements(199991000, reqs) + assert.Equal(t, 1, len(r)) + assert.Equal(t, set.New(addrs...), set.New(r[0].Address...)) + assert.False(t, r[0].AddressShouldBeERC20) + assert.Equal(t, 1, len(r[0].Topics)) + assert.Equal(t, + set.New( + "0x503ec09c1b4597d114eca849f7013c8c457988802d1dc5da49a0c461b5f88658", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x53a0f15ecd2c4989821244d3c7363d7de154e125eac5b2a7c9bb008152778de1", + "0xd51a9c61267aa6196961883ecf5ff2da6619c37dac0fa92122513fb32c032d2d", + "0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0", + "0x043a568d47b1a65cdc989ff14c921411b62abf40132dc4b5e78675ba8d0bc9df"), + set.New(r[0].Topics[0]...), + ) + assert.Equal(t, controller.BlockRange{StartBlock: 199991000}, r[0].BlockRange) + //for i, x := range r { + // fmt.Printf("[%d]: %s\n", i, x.String()) + //} + //fmt.Printf("used1: %s\n", used1.String()) + //fmt.Printf("used2: %s\n", used2.String()) +} diff --git a/driver/controller/data/evm/trace.go b/driver/controller/data/evm/trace.go new file mode 100644 index 0000000..301930b --- /dev/null +++ b/driver/controller/data/evm/trace.go @@ -0,0 +1,200 @@ +package evm + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type Trace struct { + Raw json.RawMessage + + BlockNumber uint64 + BlockHash string + TransactionHash string + TransactionIndex int32 + + Error string + Address string + Signature string +} + +func (t *Trace) UnmarshalJSON(raw []byte) error { + var payload *struct { + Action struct { + Input string `json:"input"` + To string `json:"to"` + } `json:"action"` + BlockHash string `json:"blockHash"` + BlockNumber uint64 `json:"blockNumber"` + TransactionPosition int32 `json:"transactionPosition"` + TransactionHash string `json:"transactionHash"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return err + } + if payload == nil { + t = nil + return nil + } + t.Raw = raw + t.BlockNumber = payload.BlockNumber + t.BlockHash = payload.BlockHash + t.TransactionHash = payload.TransactionHash + t.TransactionIndex = payload.TransactionPosition + t.Error = payload.Error + t.Address = payload.Action.To + if len(payload.Action.Input) >= 10 { + t.Signature = payload.Action.Input[:10] + } + return nil +} + +// TraceFilter has 2 parts, there are linked by AND +type TraceFilter struct { + Signature []string + Address []string +} + +func (f TraceFilter) Check(trace Trace) bool { + if len(trace.Error) > 0 { + return false + } + if len(f.Address) > 0 && utils.IndexOf(f.Address, strings.ToLower(trace.Address)) < 0 { + return false + } + if len(f.Signature) > 0 && utils.IndexOf(f.Signature, trace.Signature) < 0 { + return false + } + return true +} + +func (t TraceFilter) String() string { + return fmt.Sprintf("Sig:[%s],Addr:%s", utils.ArrSummary(t.Signature, 10), utils.ArrSummary(t.Address, 10)) +} + +// Merge traces match t always match r, traces match a also always match r. Traces(r) >= Traces(t) + Traces(a) +func (t TraceFilter) Merge(a TraceFilter) (r TraceFilter) { + if len(t.Signature) > 0 && len(a.Signature) > 0 { + r.Signature = set.SmartNew[string](t.Signature, a.Signature).DumpValues() + } + if len(t.Address) > 0 && len(a.Address) > 0 { + r.Address = set.SmartNew[string](t.Address, a.Address).DumpValues() + } + return r +} + +func MergeTraceFilters(filters ...TraceFilter) TraceFilter { + if len(filters) == 0 { + panic("filters is empty") + } + signatures := set.New[string]() + for _, filter := range filters { + if len(filter.Signature) == 0 { + signatures = set.New[string]() + break + } + signatures.Add(filter.Signature...) + } + addresses := set.New[string]() + for _, filter := range filters { + if len(filter.Address) == 0 { + addresses = set.New[string]() + break + } + addresses.Add(filter.Address...) + } + return TraceFilter{ + Signature: signatures.DumpValues(), + Address: addresses.DumpValues(), + } +} + +type TraceRequirement struct { + controller.BlockRange + TraceFilter +} + +func (r TraceRequirement) String() string { + return fmt.Sprintf("TraceRequirement[%s]%s", r.TraceFilter.String(), r.BlockRange.String()) +} + +func (r TraceRequirement) Snapshot() any { + return map[string]any{ + "filter": r.TraceFilter, + "range": r.BlockRange.String(), + } +} + +// MergeTraceRequirement it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeTraceRequirement(current uint64, reqs []TraceRequirement) (result []TraceRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r TraceRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []TraceFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.TraceFilter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, TraceRequirement{ + TraceFilter: MergeTraceFilters(filters...), + BlockRange: r, + }) + } + return result +} + +func BuildTraceFetcher( + name string, + req TraceRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1, + 100, + 10000, + 2000, // the target is that each query got no more than 2000 traces + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + allTraces, err := client.GetTraces(ctx, start, end, req.TraceFilter.Address) + if err != nil { + return nil, err + } + allTraces = utils.FilterArr(allTraces, req.TraceFilter.Check) + blockTraces := utils.Group(allTraces, func(trace Trace) uint64 { + return trace.BlockNumber + }) + result := make(map[uint64]BlockMainData) + for bn, traces := range blockTraces { + result[bn] = BlockMainData{Traces: traces} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/fuel/BUILD.bazel b/driver/controller/data/fuel/BUILD.bazel new file mode 100644 index 0000000..6180c64 --- /dev/null +++ b/driver/controller/data/fuel/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fuel", + srcs = [ + "block.go", + "block_main.go", + "client.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/fuel", + visibility = ["//visibility:public"], + deps = [ + "//chain/fuel", + "//common/concurrency", + "//common/https", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_fuel_go//types", + ], +) diff --git a/driver/controller/data/fuel/block.go b/driver/controller/data/fuel/block.go new file mode 100644 index 0000000..9968983 --- /dev/null +++ b/driver/controller/data/fuel/block.go @@ -0,0 +1,27 @@ +package fuel + +import ( + "time" + + "github.com/sentioxyz/fuel-go/types" +) + +type Block struct { + types.Header +} + +func (b Block) GetBlockNumber() uint64 { + return uint64(b.Height) +} + +func (b Block) GetBlockParentHash() string { + return "" +} + +func (b Block) GetBlockHash() string { + return b.Id.String() +} + +func (b Block) GetBlockTime() time.Time { + return b.Time.Time +} diff --git a/driver/controller/data/fuel/block_main.go b/driver/controller/data/fuel/block_main.go new file mode 100644 index 0000000..40887eb --- /dev/null +++ b/driver/controller/data/fuel/block_main.go @@ -0,0 +1,113 @@ +package fuel + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type DataRequirement struct { + Tx []TransactionRequirement + Interval []data.IntervalRequirement +} + +type BlockMainData struct { + Txs []fuel.WrappedTransaction + Intervals []data.IntervalConfig +} + +func (d BlockMainData) Size() int { + return len(d.Txs) + len(d.Intervals) +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Tx = MergeTxRequirements(currentBlockNumber, req.Tx) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Tx { + fetchers = append(fetchers, BuildTxFetcher( + namePrefix+fmt.Sprintf("TxFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // Txs will never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Txs = append(data.Txs, box.Txs...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/fuel/client.go b/driver/controller/data/fuel/client.go new file mode 100644 index 0000000..ec39a7f --- /dev/null +++ b/driver/controller/data/fuel/client.go @@ -0,0 +1,196 @@ +package fuel + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + "github.com/sentioxyz/fuel-go/types" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetBlock(ctx context.Context, blockNumber uint64) (Block, error) + GetTransactions(ctx context.Context, param fuel.GetTransactionsParam) ([]fuel.WrappedTransaction, error) + GetContractCreateBlockHeight(ctx context.Context, contractID string, startBlock uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedHeaders *data.BlockCache[Block] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *client) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + var resp fuel.GetLatestBlockResponse + if err = c.callContext(ctx, &resp, 0, "fuel_getLatestHeader", 0); err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest = Block{Header: resp.Header} + return latest, data.GetFirst(c.firstBlockNumber, latest.GetBlockNumber()), err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockHeightGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp fuel.GetLatestBlockResponse + err = c.callContext(ctx, &resp, 0, "fuel_getLatestHeader", blockHeightGt) + if err == nil { + latest, broken = Block{Header: resp.Header}, resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var header types.Header + if err := c.callContext(ctx, &header, blockNumber, "fuel_getBlockHeader", blockNumber); err != nil { + return Block{}, err + } + return Block{Header: header}, nil +} + +func (c *client) getHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (Block, error) { + block, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, block) + } + return block, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getHeaderIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one fuel_getBlockHeader. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +func (c *client) GetTransactions(ctx context.Context, param fuel.GetTransactionsParam) ([]fuel.WrappedTransaction, error) { + var txs []fuel.WrappedTransaction + err := c.callContext(ctx, &txs, param.StartHeight, "fuel_getTransactions", param) + return txs, err +} + +func (c *client) GetContractCreateBlockHeight( + ctx context.Context, + contractID string, + startBlock uint64, +) (blockNumber uint64, has bool, err error) { + var tx *fuel.WrappedTransaction + if err = c.callContext(ctx, &tx, 0, "fuel_getContractCreateTransaction", contractID); err != nil { + return 0, false, err + } + if tx == nil { + return 0, false, nil + } + return max(tx.BlockHeight, startBlock), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, controller.GetBlockFullText[Block]), + }, + } +} diff --git a/driver/controller/data/fuel/transaction.go b/driver/controller/data/fuel/transaction.go new file mode 100644 index 0000000..cf68bb9 --- /dev/null +++ b/driver/controller/data/fuel/transaction.go @@ -0,0 +1,94 @@ +package fuel + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type TransactionRequirement struct { + controller.BlockRange + + Filters []fuel.TransactionFilter +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filters": r.Filters, + "range": r.BlockRange.String(), + } +} + +// MergeTxRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeTxRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + var filters []fuel.TransactionFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.Filters...) + } + } + if len(filters) == 0 { + continue + } + result = append(result, TransactionRequirement{ + Filters: filters, + BlockRange: r, + }) + } + return result +} + +func BuildTxFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1, + 100, + 100000, + 1000, // the target is that each query got no more than 1000 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetTransactions(ctx, fuel.GetTransactionsParam{ + StartHeight: start, + EndHeight: end, + Filters: req.Filters, + }) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, tx := range txs { + bd, _ := result[tx.BlockHeight] + bd.Txs = append(bd.Txs, tx) + result[tx.BlockHeight] = bd + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/interval.go b/driver/controller/data/interval.go new file mode 100644 index 0000000..4c4536d --- /dev/null +++ b/driver/controller/data/interval.go @@ -0,0 +1,200 @@ +package data + +import ( + "context" + "fmt" + "sort" + "time" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/window" + "sentioxyz/sentio-core/driver/controller" +) + +type BlockInterval struct { + Backfill uint64 + Watching uint64 +} + +type TimeInterval struct { + Backfill time.Duration + Watching time.Duration +} + +type IntervalConfig struct { + BlockInterval *BlockInterval + TimeInterval *TimeInterval +} + +func (c IntervalConfig) String() string { + if c.BlockInterval != nil { + if c.BlockInterval.Backfill == c.BlockInterval.Watching { + return fmt.Sprintf("Per:%d", c.BlockInterval.Watching) + } else { + return fmt.Sprintf("Backfill:%d,Watching:%d", c.BlockInterval.Backfill, c.BlockInterval.Watching) + } + } + if c.TimeInterval != nil { + if c.TimeInterval.Backfill == c.TimeInterval.Watching { + return fmt.Sprintf("Per:%s", c.TimeInterval.Watching) + } else { + return fmt.Sprintf("Backfill:%s,Watching:%s", c.TimeInterval.Backfill, c.TimeInterval.Watching) + } + } + return "empty" +} + +// watchingDiffers reports whether the finer watching window differs from the backfill window, i.e. +// whether the watching pass can produce points the backfill pass does not. +func (c IntervalConfig) watchingDiffers() bool { + if c.BlockInterval != nil { + return c.BlockInterval.Watching != c.BlockInterval.Backfill + } + if c.TimeInterval != nil { + return c.TimeInterval.Watching != c.TimeInterval.Backfill + } + return false +} + +func (c IntervalConfig) Equal(a IntervalConfig) bool { + if c.BlockInterval != nil { + return a.BlockInterval != nil && + a.BlockInterval.Backfill == c.BlockInterval.Backfill && + a.BlockInterval.Watching == c.BlockInterval.Watching + } + if c.TimeInterval != nil { + return a.TimeInterval != nil && + a.TimeInterval.Backfill == c.TimeInterval.Backfill && + a.TimeInterval.Watching == c.TimeInterval.Watching + } + return false +} + +func ContainsInterval(list []IntervalConfig, target IntervalConfig) bool { + for _, item := range list { + if item.Equal(target) { + return true + } + } + return false +} + +type IntervalRequirement struct { + controller.BlockRange + IntervalConfig +} + +func (r IntervalRequirement) String() string { + return fmt.Sprintf("IntervalRequirement[%s]%s", r.IntervalConfig.String(), r.BlockRange.String()) +} + +func (r IntervalRequirement) Snapshot() any { + return map[string]any{ + "config": r.IntervalConfig.String(), + "range": r.BlockRange.String(), + } +} + +// MergeIntervalRequirements Will remove the requirement of being fully included +func MergeIntervalRequirements(reqs []IntervalRequirement) (result []IntervalRequirement) { + index := func(r IntervalRequirement) int { + if r.BlockInterval != nil { + return 0 + } + return 1 + } + // It will ensure that all block intervals are in front of the time interval + sort.Slice(reqs, func(i, j int) bool { + if ix, jx := index(reqs[i]), index(reqs[j]); ix != jx { + return ix < jx + } + if reqs[i].StartBlock != reqs[j].StartBlock { + return reqs[i].StartBlock < reqs[j].StartBlock + } + return reqs[i].BlockRange.Include(reqs[j].BlockRange) + }) + for i, req := range reqs { + var included bool + for j := i - 1; j >= 0 && index(req) == index(reqs[j]); j-- { + pre := reqs[j] + if pre.BlockRange.Include(req.BlockRange) && pre.IntervalConfig.Equal(req.IntervalConfig) { + included = true + break + } + } + if !included { + result = append(result, req) + } + } + return +} + +func QueryInterval( + ctx context.Context, + start, end uint64, + first uint64, + latest controller.BlockHeader, + req IntervalRequirement, + timeGetter func(ctx context.Context, blockNumber uint64) (time.Time, error), +) ([]uint64, error) { + itv := req.IntervalConfig + // inWatching is only used when the watching window differs from the backfill window; when they + // are equal the watching scan/points just duplicate the backfill ones, so there is no need to + // even fetch end's block time to decide it. + var inWatching bool + if itv.watchingDiffers() { + endTime, err := timeGetter(ctx, end) + if err != nil { + return nil, err + } + inWatching = latest.GetBlockTime().Sub(endTime) < controller.WatchingDelay + } + var blockNumbers []uint64 + if itv.BlockInterval != nil { + for n := start; n <= end; n++ { + if n%itv.BlockInterval.Backfill == 0 || (inWatching && n%itv.BlockInterval.Watching == 0) { + blockNumbers = append(blockNumbers, n) + } + } + } else if itv.TimeInterval != nil { + bns, err := window.FindStartPoints[int64]( + ctx, int64(start), int64(end), 0, + func(ctx context.Context, n int64) (time.Time, error) { + // start may equal to first and window.FindStartPoints will query the time of start-1, + // so here need to check whether n is less than first + if n < int64(first) { + return time.Time{}, nil + } + t, getErr := timeGetter(ctx, uint64(n)) + if getErr != nil { + return time.Time{}, getErr + } + return t.Truncate(itv.TimeInterval.Backfill), nil + }) + if err != nil { + return nil, err + } + ns := utils.BuildSet(bns) + // inWatching is already false when the watching window equals the backfill window, so this + // extra scan only runs when it can add new points. + if inWatching { + bns, err = window.FindStartPoints[int64]( + ctx, int64(start), int64(end), 0, + func(ctx context.Context, n int64) (time.Time, error) { + t, getErr := timeGetter(ctx, uint64(n)) + if getErr != nil { + return time.Time{}, getErr + } + return t.Truncate(itv.TimeInterval.Watching), nil + }) + if err != nil { + return nil, err + } + utils.MergeInto(ns, bns) + } + for n := range ns { + blockNumbers = append(blockNumbers, uint64(n)) + } + } + return blockNumbers, nil +} diff --git a/driver/controller/data/sol/BUILD.bazel b/driver/controller/data/sol/BUILD.bazel new file mode 100644 index 0000000..3b1b901 --- /dev/null +++ b/driver/controller/data/sol/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sol", + srcs = [ + "block.go", + "block_main.go", + "client.go", + "native_client.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sol", + visibility = ["//visibility:public"], + deps = [ + "//chain/sol", + "//common/concurrency", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_gagliardetto_solana_go//:solana-go", + "@com_github_gagliardetto_solana_go//rpc", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "sol_test", + srcs = ["client_test.go"], + embed = [":sol"], + deps = [ + "//driver/controller/data", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/sol/block.go b/driver/controller/data/sol/block.go new file mode 100644 index 0000000..6ec432f --- /dev/null +++ b/driver/controller/data/sol/block.go @@ -0,0 +1,32 @@ +package sol + +import ( + "time" + + "github.com/gagliardetto/solana-go/rpc" +) + +type Block struct { + Slot uint64 + *rpc.GetBlockResult +} + +func (b Block) Skipped() bool { + return b.GetBlockResult == nil +} + +func (b Block) GetBlockNumber() uint64 { + return b.Slot +} + +func (b Block) GetBlockParentHash() string { + return b.PreviousBlockhash.String() +} + +func (b Block) GetBlockHash() string { + return b.Blockhash.String() +} + +func (b Block) GetBlockTime() time.Time { + return b.BlockTime.Time() +} diff --git a/driver/controller/data/sol/block_main.go b/driver/controller/data/sol/block_main.go new file mode 100644 index 0000000..0b8e102 --- /dev/null +++ b/driver/controller/data/sol/block_main.go @@ -0,0 +1,249 @@ +package sol + +import ( + "context" + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +const ( + // targetKeepBytes is the buffered-data backpressure target for the main-data fetchers. + // BlockMainData.Size() estimates real memory use (full transactions can be large), so this is + // sized in bytes, not in block/transaction counts. + targetKeepBytes = 100 * 1024 * 1024 // 100MB + + // targetQueryBytes is the per-query data-size target the fetcher grows a range toward, so each + // query fetches roughly this much data. + targetQueryBytes = 10 * 1024 * 1024 // 10MB + + // intervalMinQuerySize is the smallest interval fetch range. It matches the super node's + // per-query block cap: GetBlocksByInterval returns at most one block per slot, so a range this + // size can always fit under the cap, letting the fetcher shrink down to it on an over-cap error. + intervalMinQuerySize = 500 +) + +// DataRequirement is the data demand of all handlers of one processor. +type DataRequirement struct { + // Tx is the instruction handlers' requirements (one per handler, merged into disjoint ranges + // when building the fetchers). + Tx []TransactionRequirement + // Interval is the interval handlers' requirements. + Interval []data.IntervalRequirement +} + +func (r DataRequirement) Snapshot() any { + txs := make([]any, len(r.Tx)) + for i, t := range r.Tx { + txs[i] = t.Snapshot() + } + intervals := make([]any, len(r.Interval)) + for i, iv := range r.Interval { + intervals[i] = iv.Snapshot() + } + return map[string]any{ + "tx": txs, + "intervals": intervals, + } +} + +// BlockMainData is the per-block data produced by the main-data fetchers and consumed directly by +// the handlers; it already carries everything needed to build the binding data, so the block-data +// phase no longer fetches anything. +type BlockMainData struct { + // Slot / Blockhash / PreviousBlockhash / BlockTime are the block header, always set when data + // is present, so the block data can be built without a separate getBlock call. + Slot uint64 + Blockhash string + PreviousBlockhash string + BlockTime *solana.UnixTimeSeconds + // Intervals are the interval configs this block is a target of (interval handler). + Intervals []data.IntervalConfig + // Block is the block header with signatures (interval handler), nil when not an interval target. + Block *Block + // Transactions are the full transactions invoking the requested programs (instruction handler). + Transactions []solcore.WrappedTransaction +} + +func (d BlockMainData) IsEmpty() bool { + return len(d.Intervals) == 0 && d.Block == nil && len(d.Transactions) == 0 +} + +func (d BlockMainData) Size() int { + size := 0 + if d.Block != nil && d.Block.GetBlockResult != nil { + size += len(d.Block.Signatures) * 90 + } + for _, tx := range d.Transactions { + size += estimateTransactionSize(tx) + } + return size +} + +func estimateTransactionSize(tx solcore.WrappedTransaction) int { + size := 256 // base overhead for signature/version/envelope + if tx.Transaction != nil { + for _, in := range tx.Transaction.Message.Instructions { + size += len(in.Data) + len(in.Accounts)*45 + } + } + if tx.Meta != nil { + for _, inner := range tx.Meta.InnerInstructions { + for _, in := range inner.Instructions { + size += len(in.Data) + len(in.Accounts)*45 + } + } + } + return size +} + +func windowsOf(cfg data.IntervalConfig) (backfill, watching solcore.IntervalWindow) { + if cfg.BlockInterval != nil { + return solcore.IntervalWindow{BlockWindow: cfg.BlockInterval.Backfill}, + solcore.IntervalWindow{BlockWindow: cfg.BlockInterval.Watching} + } + return solcore.IntervalWindow{TimeWindow: cfg.TimeInterval.Backfill}, + solcore.IntervalWindow{TimeWindow: cfg.TimeInterval.Watching} +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + backfill, watching := windowsOf(req.IntervalConfig) + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + // minQuerySize is intervalMinQuerySize (not maxQuerySize) so the fetcher can shrink down to + // a range the super node's block cap can always satisfy when it returns an over-cap error. + intervalMinQuerySize, + 10000, + targetKeepBytes, + targetQueryBytes, + time.Second*15, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + result := make(map[uint64]BlockMainData) + // Builds the interval handler's binding data from each window's first block. Only block + // header fields are used (slot, blockhash, parent hash, time). NOTE: blocks served from + // the BigQuery archival tier (slots below the ClickHouse range) carry NO transaction + // signatures — a deliberate BigQuery cost optimization (see GetBlocksByInterval and + // the archival store). That is fine here because interval binding data is header-only; do not + // add a dependency on b's transaction signatures in this path. + add := func(blocks []Block) { + for i := range blocks { + b := blocks[i] + if b.Skipped() { + continue + } + bd := result[b.Slot] + if !data.ContainsInterval(bd.Intervals, req.IntervalConfig) { + bd.Intervals = append(bd.Intervals, req.IntervalConfig) + } + bd.Slot = b.Slot + bd.Blockhash = b.GetBlockHash() + bd.PreviousBlockhash = b.GetBlockParentHash() + bd.BlockTime = b.BlockTime + bd.Block = &b + result[b.Slot] = bd + } + } + // The super node caps the result and errors when exceeded; that error propagates and the + // fetcher halves the range, so no client-side saturation check is needed. + backfillBlocks, err := client.GetBlocksByInterval(ctx, start, end, backfill) + if err != nil { + return nil, err + } + add(backfillBlocks) + // The finer "watching" window only applies near the head. Like data.QueryInterval, decide + // by time: the range is "in watching" when end's block time is within WatchingDelay of the + // latest. end may itself be skipped, so look up the nearest non-skipped block at or before + // end (slot < end+1) for its time; if there is none we conservatively run the watching + // query and rely on the per-block time filter below. + if watching != backfill { + inWatching := true + endBlock, getErr := client.GetPreviousUnskippedBlock(ctx, end+1) + if getErr != nil { + return nil, getErr + } + if endBlock.Found && endBlock.BlockTime != nil { + inWatching = latest.GetBlockTime().Sub(endBlock.BlockTime.Time()) < controller.WatchingDelay + } + if inWatching { + watchingBlocks, err := client.GetBlocksByInterval(ctx, start, end, watching) + if err != nil { + return nil, err + } + // Keep only blocks whose own time is within WatchingDelay of the latest. + cutoff := latest.GetBlockTime().Add(-controller.WatchingDelay) + inWatch := make([]Block, 0, len(watchingBlocks)) + for _, b := range watchingBlocks { + if !b.Skipped() && b.BlockTime != nil && b.BlockTime.Time().After(cutoff) { + inWatch = append(inWatch, b) + } + } + add(inWatch) + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Tx = MergeTxRequirements(currentBlockNumber, req.Tx) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Tx { + fetchers = append(fetchers, BuildTxFetcher( + namePrefix+fmt.Sprintf("TxFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (result BlockMainData, has bool, _ error) { + result.Slot = bn + for _, box := range from { + if box.Blockhash != "" { + result.Blockhash = box.Blockhash + result.PreviousBlockhash = box.PreviousBlockhash + } + if box.BlockTime != nil { + result.BlockTime = box.BlockTime + } + result.Intervals = append(result.Intervals, box.Intervals...) + if box.Block != nil { + result.Block = box.Block + } + result.Transactions = append(result.Transactions, box.Transactions...) + } + return result, !result.IsEmpty(), nil + }) +} diff --git a/driver/controller/data/sol/client.go b/driver/controller/data/sol/client.go new file mode 100644 index 0000000..08ca63a --- /dev/null +++ b/driver/controller/data/sol/client.go @@ -0,0 +1,385 @@ +package sol + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + ethrpc "github.com/ethereum/go-ethereum/rpc" + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + // GetBlock returns the block header (no transactions). The result may be skipped. + GetBlock(ctx context.Context, blockNumber uint64) (Block, error) + + // GetBlocksByInterval returns the first non-skipped block of each window in [from, to], for the + // interval handler. The super node caps the result and errors when exceeded. + // + // Signatures: blocks within the ClickHouse range carry their transaction signatures; blocks + // served from the BigQuery archival tier (slots below the ClickHouse range) carry headers only, + // with NO signatures — a deliberate BigQuery cost optimization (see the archival store). The interval + // handler only uses block headers (slot/hash/time), so this is fine; callers must not rely on + // per-block signatures from this method. + GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, + ) ([]Block, error) + // FindTransactions returns, grouped by block, the full transactions in [from, to] invoking any + // of the given programs, for the instruction handler. The super node caps the result and errors + // when exceeded (except for a single-block range). + FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, + ) ([]solcore.BlockTransactions, error) + GetContractStartBlock(ctx context.Context, address solana.PublicKey, start, latest uint64) (uint64, bool, error) + // GetPreviousUnskippedBlock returns the nearest non-skipped block with slot < beforeSlot (pass + // slot+1 to include the slot itself), used to learn the chain time around a possibly-skipped slot. + GetPreviousUnskippedBlock(ctx context.Context, beforeSlot uint64) (solcore.PreviousUnskippedBlock, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +// client talks to the Solana super node over JSON-RPC. The super node answers the sol_* methods +// from its latest-slot cache and ClickHouse, so the driver no longer talks to a node directly. +type supernodeClient struct { + endpoint string + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *ethrpc.Client + + cachedHeaders *data.BlockCache[Block] + + savedLatestBlockNumber atomic.Uint64 +} + +// NewClient selects the data client for a sol chain. It uses the native Solana RPC client when the +// processor is old (DriverVersion < 2) or the endpoint is not a super node (no sol_getLatestHeader), +// and the super-node client otherwise. Only sol_mainnet runs a super node (ClickHouse + BigQuery); +// other sol chains fall back to native RPC. +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, + driverVersion int32, +) (Client, error) { + _, logger := log.FromContext(ctx) + if driverVersion < 2 { + logger.Infof("sol: driver version %d < 2, using native Solana RPC at %s", driverVersion, endpoint) + return newNativeClient(endpoint, maxConcurrency, firstBlockNumber, watchLatestInterval) + } + supported, err := endpointSupportsSuperNode(ctx, endpoint) + if err != nil { + // The probe kept hitting transient/HTTP/timeout errors and never got a definitive answer. + // Surface the retryable NewClient error so the controller restarts the pod and tries again, + // rather than failing permanently (NeverRetry) on what may be a temporary outage. + return nil, err + } + if !supported { + logger.Infof("sol: endpoint %s does not support sol_getLatestHeader, using native Solana RPC", endpoint) + return newNativeClient(endpoint, maxConcurrency, firstBlockNumber, watchLatestInterval) + } + // firstBlockNumber is intentionally not passed: the super node is backed by ClickHouse + BigQuery, + // so it serves the full available history and bounds the start range itself (range store + + // BigQuery retention floor). The driver imposes no client-side first block. + return newSupernodeClient(ctx, endpoint, maxConcurrency, watchLatestInterval) +} + +func newSupernodeClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + watchLatestInterval time.Duration, +) (Client, error) { + cli := &supernodeClient{ + endpoint: endpoint, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + var err error + if cli.cli, err = ethrpc.DialOptions(ctx, endpoint, ethrpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return cli, nil +} + +// superNodeProbeCache memoizes endpointSupportsSuperNode by endpoint, so each endpoint is probed at +// most once across controllers. +var superNodeProbeCache sync.Map // endpoint string -> bool + +// superNodeProbeMaxAttempts bounds how many times endpointSupportsSuperNode retries a transient +// (HTTP/timeout/dial) probe failure before giving up with a retryable NewClient error. +const superNodeProbeMaxAttempts = 20 + +// endpointSupportsSuperNode probes whether the endpoint implements the super-node sol_getLatestHeader +// method (gt=0 returns immediately when supported), returning: +// +// - (true, nil) — the endpoint answers sol_getLatestHeader, so it is a super node. +// - (false, nil) — the endpoint returns -32601 / "method not found", so it is a plain Solana RPC +// endpoint and the caller should fall back to native RPC. +// - (false, err) — the probe kept failing with transient errors (HTTP/timeout/dial) across all +// attempts; err is a *data.NewClientRetryableError so the caller can restart the pod and retry, +// instead of incorrectly assuming either kind of endpoint. +// +// Transient failures are retried with exponential backoff up to superNodeProbeMaxAttempts times, each +// logged. Only definitive answers (super node / not a super node) are cached per endpoint. +func endpointSupportsSuperNode(ctx context.Context, endpoint string) (bool, error) { + if v, ok := superNodeProbeCache.Load(endpoint); ok { + return v.(bool), nil + } + _, logger := log.FromContext(ctx) + + var supported bool + attempt := 0 + operation := func() error { + attempt++ + probe, err := ethrpc.DialOptions(ctx, endpoint, ethrpc.WithHTTPClient(https.DefaultClient)) + if err != nil { + logger.Warnf("sol: probe dial to %s failed (attempt %d/%d): %v; will retry", + endpoint, attempt, superNodeProbeMaxAttempts, err) + return err + } + defer probe.Close() + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + var blk Block + err = probe.CallContext(probeCtx, &blk, "sol_getLatestHeader", uint64(0)) + if err == nil { + supported = true + return nil + } + // A -32601 / "method not found" is a definitive negative: the endpoint is a plain Solana RPC + // node. Stop retrying by returning nil with supported left false. + var rpcErr ethrpc.Error + if errors.As(err, &rpcErr) && rpcErr.ErrorCode() == -32601 { + supported = false + return nil + } + if msg := strings.ToLower(err.Error()); strings.Contains(msg, "method not found") || strings.Contains(msg, "does not exist") { + supported = false + return nil + } + // Anything else (HTTP error, timeout, connection reset, ...) is transient: retry. + logger.Warnf("sol: probe sol_getLatestHeader at %s failed (attempt %d/%d): %v; will retry", + endpoint, attempt, superNodeProbeMaxAttempts, err) + return err + } + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 200 * time.Millisecond + bo.MaxInterval = 10 * time.Second + // WithMaxRetries(n) allows n retries after the initial attempt → superNodeProbeMaxAttempts total. + retry := backoff.WithContext(backoff.WithMaxRetries(bo, superNodeProbeMaxAttempts-1), ctx) + if err := backoff.Retry(operation, retry); err != nil { + return false, data.NewClientRetryable( + fmt.Sprintf("sol: probing super-node support at %s kept failing after %d attempts", endpoint, superNodeProbeMaxAttempts), + err, + ) + } + superNodeProbeCache.Store(endpoint, supported) + return supported, nil +} + +func (c *supernodeClient) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *supernodeClient) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlock", blockNumber); err != nil { + return Block{}, err + } + return blk, nil +} + +func (c *supernodeClient) getBlock(ctx context.Context, blockNumber uint64) (Block, error) { + blk, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, blk) + } + return blk, err +} + +// getLatestHeader long-polls the super node for the latest non-skipped block header with slot > gt. +// The super node blocks until such a block exists, so the driver never has to poll, and the +// returned header is guaranteed non-skipped (BlockTime set). +func (c *supernodeClient) getLatestHeader(ctx context.Context, gt uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, 0, "sol_getLatestHeader", gt); err != nil { + return Block{}, err + } + c.savedLatestBlockNumber.Store(blk.Slot) + return blk, nil +} + +// GetLatest returns the latest non-skipped header. first is 0: the super node serves the full +// available history (ClickHouse + BigQuery) and bounds the start range itself, so the driver imposes +// no client-side first block (the processor's own start / GetContractStartBlock decide it). +func (c *supernodeClient) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + blk, err := c.getLatestHeader(ctx, 0) + if err != nil { + return nil, 0, err + } + return blk, 0, nil +} + +func (c *supernodeClient) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + blk, err := c.getLatestHeader(ctx, blockNumberGt) + if err != nil { + return nil, nil, err + } + return blk, nil, nil + }, + callback) +} + +func (c *supernodeClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getBlock(ctx, blockNumber) +} + +func (c *supernodeClient) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one sol_getBlock. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +func (c *supernodeClient) GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, +) ([]Block, error) { + var blocks []Block + param := solcore.GetBlocksByIntervalParam{From: from, To: to, Window: window} + if err := c.callContext(ctx, &blocks, to, "sol_getBlocksByInterval", param); err != nil { + return nil, errors.Wrapf(err, "get interval blocks in [%d,%d] failed", from, to) + } + return blocks, nil +} + +func (c *supernodeClient) FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, +) ([]solcore.BlockTransactions, error) { + var result []solcore.BlockTransactions + param := solcore.FindTransactionsParam{From: from, To: to, ProgramIDs: programs} + if err := c.callContext(ctx, &result, to, "sol_findTransactions", param); err != nil { + return nil, errors.Wrapf(err, "find transactions in [%d,%d] for %d programs failed", from, to, len(programs)) + } + return result, nil +} + +// GetContractStartBlock asks the super node for the contract's earliest appearance block, then maps +// it to [start, latest]: an appearance before start clamps to start; an appearance after latest (or +// no appearance) is treated as not yet in range. +func (c *supernodeClient) GetContractStartBlock( + ctx context.Context, + address solana.PublicKey, + start, latest uint64, +) (uint64, bool, error) { + var result solcore.GetContractStartBlockResult + if err := c.callContext(ctx, &result, 0, "sol_getContractStartBlock", address); err != nil { + return 0, false, err + } + if !result.Found || result.Slot > latest { + return 0, false, nil + } + if result.Slot < start { + return start, true, nil + } + return result.Slot, true, nil +} + +func (c *supernodeClient) GetPreviousUnskippedBlock( + ctx context.Context, + beforeSlot uint64, +) (solcore.PreviousUnskippedBlock, error) { + var result solcore.PreviousUnskippedBlock + if err := c.callContext(ctx, &result, beforeSlot, "sol_getPreviousUnskippedBlock", beforeSlot); err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + return result, nil +} + +func (c *supernodeClient) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *supernodeClient) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "savedLatestBlockNumber": c.savedLatestBlockNumber.Load(), + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, func(block Block) string { + if block.Skipped() { + return fmt.Sprintf("%d/", block.Slot) + } + return controller.GetBlockFullText(block) + }), + }, + } +} diff --git a/driver/controller/data/sol/client_test.go b/driver/controller/data/sol/client_test.go new file mode 100644 index 0000000..6ce1a65 --- /dev/null +++ b/driver/controller/data/sol/client_test.go @@ -0,0 +1,84 @@ +package sol + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/driver/controller/data" +) + +func Test_getSkippedBlock(t *testing.T) { + t.Skip("used external endpoint") + + cli, _ := NewClient( + context.Background(), + "https://solana-rpc.publicnode.com", + 1, + 10000, + time.Second, + 0, // driver version < 2 ⇒ native Solana RPC (this endpoint is a plain node) + ) + + blk, err := cli.GetBlock(context.Background(), 392086804) + assert.NoError(t, err) + assert.False(t, blk.Skipped()) + + blk, err = cli.GetBlock(context.Background(), 392086803) + assert.NoError(t, err) + assert.True(t, blk.Skipped()) +} + +func Test_queryInterval(t *testing.T) { + t.Skip("used external endpoint") + + cli, _ := NewClient( + context.Background(), + "https://solana-rpc.publicnode.com", + 1, + 0, + time.Second, + 0, // driver version < 2 ⇒ native Solana RPC (this endpoint is a plain node) + ) + + ctx := context.Background() + + latest, first, err := cli.GetLatest(ctx) + assert.NoError(t, err) + + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + for n := blockNumber; n >= 0; n-- { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + h, err := cli.GetBlock(getCtx, n) + cancel() + if err != nil { + return time.Time{}, err + } + if !h.Skipped() { + fmt.Printf("!!! got block %d/%d: %s\n", blockNumber, n, h.GetBlockTime().Format(time.RFC3339)) + return h.GetBlockTime(), nil + } + fmt.Printf("!!! got block %d/%d but skipped\n", blockNumber, n) + // Slot n was skipped, try n-1 next + } + // all slot in [0,blockNumber] was skipped, just return zero time + return time.Time{}, nil + } + s := uint64(393722931) + e := uint64(393825051) + + req := data.IntervalRequirement{ + IntervalConfig: data.IntervalConfig{ + TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 4, + Watching: time.Hour, + }, + }, + } + bns, err := data.QueryInterval(ctx, s, e, first, latest, req, timeGetter) + assert.NoError(t, err) + fmt.Printf("!!! result: %v\n", bns) +} diff --git a/driver/controller/data/sol/native_client.go b/driver/controller/data/sol/native_client.go new file mode 100644 index 0000000..664d7c4 --- /dev/null +++ b/driver/controller/data/sol/native_client.go @@ -0,0 +1,542 @@ +package sol + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +// nativeClient implements Client against a plain Solana JSON-RPC endpoint (no super node). It is used +// for sol chains that have no super node (anything but sol_mainnet) and for old processors +// (DriverVersion < 2). It reproduces the super-node sol_* capabilities over native primitives +// (getSlot / getBlock / getBlocks / getSignaturesForAddress), porting the pre-super-node driver +// (commit 87d8d1455). These calls are far more RPC-heavy than the super node — acceptable for the +// low-volume chains / legacy processors that need this path. +type nativeClient struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + getLatestTimeout time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedHeaders *data.BlockCache[Block] + + savedLatestBlockNumber atomic.Uint64 + savedFirstBlockNumber atomic.Uint64 +} + +func newNativeClient( + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (Client, error) { + c := &nativeClient{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + getLatestTimeout: 30 * time.Second, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + cli: rpc.New(endpoint), + } + c.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return c, nil +} + +var slotSkippedErrorMatcher = regexp.MustCompile(`slot.*was skipped`) + +func isSlotSkippedError(err error) bool { + return err != nil && slotSkippedErrorMatcher.FindString(strings.ToLower(err.Error())) != "" +} + +// parsedBlockResult is the subset of getBlock's jsonParsed/full response we need to assemble +// BlockTransactions (header + ordered transactions). +type parsedBlockResult struct { + Blockhash solana.Hash `json:"blockhash"` + PreviousBlockhash solana.Hash `json:"previousBlockhash"` + BlockTime *solana.UnixTimeSeconds `json:"blockTime"` + Transactions []parsedTransactionWithMeta `json:"transactions"` +} + +type parsedTransactionWithMeta struct { + Transaction *rpc.ParsedTransaction `json:"transaction"` + Meta *rpc.ParsedTransactionMeta `json:"meta"` +} + +// callContext applies the concurrency token + statistics around a native Solana RPC call, dispatched +// by an internal method name (kept sol_*-style for statistics continuity with the super-node client). +func (c *nativeClient) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always context.Canceled + } + defer release() + callStartAt := time.Now() + switch method { + case "sol_getLatestBlockNumber": + r := result.(*uint64) + *r, err = c.cli.GetSlot(ctx, rpc.CommitmentFinalized) + case "sol_getBlock": + opt := rpc.GetBlockOpts{TransactionDetails: rpc.TransactionDetailsSignatures} + r := result.(*Block) + r.Slot = args[0].(uint64) + r.GetBlockResult, err = c.cli.GetBlockWithOpts(ctx, r.Slot, &opt) + if err != nil && isSlotSkippedError(err) { + r.GetBlockResult, err = nil, nil // skipped slot + } + case "sol_getBlockFull": + // solana-go's GetBlockWithOpts rejects the jsonParsed encoding, so issue the raw getBlock + // call to fetch every transaction's parsed detail (and the header) in one request. + obj := rpc.M{ + "encoding": solana.EncodingJSONParsed, + "transactionDetails": rpc.TransactionDetailsFull, + "maxSupportedTransactionVersion": uint64(0), + "rewards": false, + } + r := result.(*parsedBlockResult) + err = c.cli.RPCCallForInto(ctx, r, "getBlock", []any{args[0].(uint64), obj}) + if err != nil && isSlotSkippedError(err) { + *r, err = parsedBlockResult{}, nil // skipped slot + } + case "sol_getBlocks": + end := args[1].(uint64) + r := result.(*rpc.BlocksResult) + *r, err = c.cli.GetBlocks(ctx, args[0].(uint64), &end, rpc.CommitmentFinalized) + case "sol_getSignaturesForAddress": + limit := args[3].(int) + opt := rpc.GetSignaturesForAddressOpts{ + Until: args[1].(solana.Signature), + Before: args[2].(solana.Signature), + Limit: &limit, + } + r := result.(*[]*rpc.TransactionSignature) + *r, err = c.cli.GetSignaturesForAddressWithOpts(ctx, args[0].(solana.PublicKey), &opt) + default: + panic(errors.Errorf("unsupported method %q", method)) + } + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *nativeClient) getLatestBlockNumber(ctx context.Context) (uint64, error) { + var latest uint64 + if err := c.callContext(ctx, &latest, 0, "sol_getLatestBlockNumber"); err != nil { + return 0, err + } + c.savedLatestBlockNumber.Store(latest) + c.savedFirstBlockNumber.CompareAndSwap(0, data.GetFirst(c.firstBlockNumber, latest)) + return latest, nil +} + +func (c *nativeClient) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlock", blockNumber); err != nil { + return Block{}, err + } + return blk, nil +} + +func (c *nativeClient) getBlock(ctx context.Context, blockNumber uint64) (Block, error) { + blk, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, blk) + } + return blk, err +} + +func (c *nativeClient) getFullBlock(ctx context.Context, blockNumber uint64) (parsedBlockResult, error) { + var blk parsedBlockResult + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlockFull", blockNumber); err != nil { + return parsedBlockResult{}, err + } + return blk, nil +} + +// getBlocks returns the existing (non-skipped) slots in [start, end] in ascending order. +func (c *nativeClient) getBlocks(ctx context.Context, start, end uint64) ([]uint64, error) { + var res rpc.BlocksResult + if err := c.callContext(ctx, &res, end, "sol_getBlocks", start, end); err != nil { + return nil, err + } + return res, nil +} + +func (c *nativeClient) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + latestBlockNumber, err := c.getLatestBlockNumber(ctx) + if err != nil { + return nil, 0, err + } + // The finalized slot is never skipped, but its block data may lag a moment; retry with exponential + // backoff (cancelled with ctx). + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 200 * time.Millisecond + var blk Block + if err = backoff.Retry(func() error { + blk, err = c.getBlock(ctx, latestBlockNumber) + return err + }, backoff.WithContext(backoff.WithMaxRetries(bo, 10), ctx)); err != nil { + return nil, 0, err + } + return blk, c.savedFirstBlockNumber.Load(), nil +} + +func (c *nativeClient) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingPolling( + ctx, + c.watchLatestInterval, + c.getLatestTimeout, + from, + func(ctx context.Context) (controller.BlockHeader, error) { + h, _, err := c.GetLatest(ctx) + return h, err + }, + callback) +} + +func (c *nativeClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getBlock(ctx, blockNumber) +} + +func (c *nativeClient) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one sol_getBlock. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +// timeGetter returns blockNumber's block time, walking back over skipped slots. +func (c *nativeClient) timeGetter(ctx context.Context, blockNumber uint64) (time.Time, error) { + for n := blockNumber; ; n-- { + h, err := c.GetBlock(ctx, n) + if err != nil { + return time.Time{}, err + } + if !h.Skipped() { + return h.GetBlockTime(), nil + } + if n == 0 { + return time.Time{}, nil // all slots through 0 skipped + } + } +} + +func windowToConfig(w solcore.IntervalWindow) data.IntervalConfig { + if w.IsBlockWindow() { + return data.IntervalConfig{BlockInterval: &data.BlockInterval{Backfill: w.BlockWindow, Watching: w.BlockWindow}} + } + return data.IntervalConfig{TimeInterval: &data.TimeInterval{Backfill: w.TimeWindow, Watching: w.TimeWindow}} +} + +func (c *nativeClient) GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, +) ([]Block, error) { + latest, _, err := c.GetLatest(ctx) + if err != nil { + return nil, err + } + req := data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: from, EndBlock: &to}, + IntervalConfig: windowToConfig(window), + } + // backfill==watching window ⇒ QueryInterval needs no end-time lookup; timeGetter only matters for + // time windows. + bns, err := data.QueryInterval(ctx, from, to, c.savedFirstBlockNumber.Load(), latest, req, c.timeGetter) + if err != nil { + return nil, errors.Wrapf(err, "query interval blocks in [%d,%d] failed", from, to) + } + blocks := make([]Block, 0, len(bns)) + for _, bn := range bns { + blk, err := c.GetBlock(ctx, bn) + if err != nil { + return nil, err + } + if blk.Skipped() { + continue + } + blocks = append(blocks, blk) + } + return blocks, nil +} + +const findSignaturesPageSize = 1000 + +// findProgramSignatures returns the signatures of transactions in [fromBlock, toBlock] referencing +// address, using getSignaturesForAddress paginated by the surrounding non-skipped blocks' border +// signatures (ported from the pre-super-node driver). Result is in descending slot order. +func (c *nativeClient) findProgramSignatures( + ctx context.Context, + fromBlock, toBlock uint64, + address solana.PublicKey, +) (result []*rpc.TransactionSignature, err error) { + latest := c.savedLatestBlockNumber.Load() + first := c.savedFirstBlockNumber.Load() + var fromTxSig, toTxSig solana.Signature + + // fromTxSig = last signature of the nearest non-skipped block below fromBlock (the `until` bound). + for n := fromBlock - 1; fromBlock > 0 && n >= first; n-- { + blk, gerr := c.GetBlock(ctx, n) + if gerr != nil { + return nil, gerr + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + fromTxSig = blk.Signatures[len(blk.Signatures)-1] + break + } + if n == 0 { + break + } + } + // toTxSig = first signature of the nearest non-skipped block above toBlock (the `before` bound). + for n := toBlock + 1; n <= latest; n++ { + blk, gerr := c.GetBlock(ctx, n) + if gerr != nil { + return nil, gerr + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + toTxSig = blk.Signatures[0] + break + } + } + + limit := findSignaturesPageSize + page, err := c.getSignaturesForAddress(ctx, address, fromTxSig, toTxSig, limit) + if err != nil { + return nil, errors.Wrapf(err, "get signatures for %s in [%d,%d] failed", address, fromBlock, toBlock) + } + // page is DESC. fromTxSig/toTxSig may be empty (open borders), so clip to the block range. + var finished bool + for _, sig := range page { + if sig.Slot > toBlock { + continue + } + if sig.Slot < fromBlock { + finished = true + continue + } + result = append(result, sig) + } + if len(page) >= limit && !finished { + return nil, errors.Errorf("too many signatures for %s in [%d,%d] (page limit %d)", address, fromBlock, toBlock, limit) + } + return result, nil +} + +func (c *nativeClient) getSignaturesForAddress( + ctx context.Context, + address solana.PublicKey, + until, before solana.Signature, + limit int, +) ([]*rpc.TransactionSignature, error) { + var sigs []*rpc.TransactionSignature + if err := c.callContext(ctx, &sigs, 0, "sol_getSignaturesForAddress", address, until, before, limit); err != nil { + return nil, err + } + return sigs, nil +} + +func (c *nativeClient) FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, +) ([]solcore.BlockTransactions, error) { + // Collect the matched transaction signatures per slot, across all programs. + matched := make(map[uint64]map[solana.Signature]struct{}) + for _, p := range programs { + sigs, err := c.findProgramSignatures(ctx, from, to, p) + if err != nil { + return nil, err + } + for _, s := range sigs { + if matched[s.Slot] == nil { + matched[s.Slot] = make(map[solana.Signature]struct{}) + } + matched[s.Slot][s.Signature] = struct{}{} + } + } + if len(matched) == 0 { + return nil, nil + } + slots := make([]uint64, 0, len(matched)) + for s := range matched { + slots = append(slots, s) + } + sort.Slice(slots, func(i, j int) bool { return slots[i] < slots[j] }) + + out := make([]solcore.BlockTransactions, 0, len(slots)) + for _, slot := range slots { + full, err := c.getFullBlock(ctx, slot) + if err != nil { + return nil, err + } + want := matched[slot] + var wts []solcore.WrappedTransaction + for i, tx := range full.Transactions { + if tx.Transaction == nil || len(tx.Transaction.Signatures) == 0 { + continue + } + sig := tx.Transaction.Signatures[0] + if _, ok := want[sig]; !ok { + continue + } + wts = append(wts, solcore.WrappedTransaction{ + TransactionIndex: uint32(i), + Signature: sig, + // Native getBlock(jsonParsed) does not surface the version cheaply; default legacy + // (same as the BigQuery store). Only the version label is affected. + Version: rpc.LegacyTransactionVersion, + Transaction: tx.Transaction, + Meta: tx.Meta, + }) + } + if len(wts) == 0 { + continue + } + out = append(out, solcore.BlockTransactions{ + Slot: slot, + Blockhash: full.Blockhash, + PreviousBlockhash: full.PreviousBlockhash, + BlockTime: full.BlockTime, + Transactions: wts, + }) + } + return out, nil +} + +func (c *nativeClient) GetContractStartBlock( + ctx context.Context, + address solana.PublicKey, + start, latest uint64, +) (uint64, bool, error) { + return data.BinarySearchContractStart(ctx, start, latest, func(ctx context.Context, bn uint64) (bool, error) { + // Does address appear at or after slot bn? Page back from the top border until a signature at + // slot <= bn is found (present) or signatures run out (absent). + var toTxSig solana.Signature + for n := bn + 1; n <= latest; n++ { + blk, err := c.GetBlock(ctx, n) + if err != nil { + return false, err + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + toTxSig = blk.Signatures[0] + break + } + } + for { + sigs, err := c.getSignaturesForAddress(ctx, address, solana.Signature{}, toTxSig, 1) + if err != nil { + return false, err + } + if len(sigs) == 0 { + return false, nil + } + if sigs[0].Slot <= bn { + return true, nil + } + toTxSig = sigs[0].Signature + } + }) +} + +func (c *nativeClient) GetPreviousUnskippedBlock( + ctx context.Context, + beforeSlot uint64, +) (solcore.PreviousUnskippedBlock, error) { + if beforeSlot == 0 { + return solcore.PreviousUnskippedBlock{}, nil + } + first := c.savedFirstBlockNumber.Load() + const chunk = 1000 + hi := beforeSlot - 1 + for { + lo := first + if hi >= chunk && hi-chunk+1 > first { + lo = hi - chunk + 1 + } + slots, err := c.getBlocks(ctx, lo, hi) + if err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + if len(slots) > 0 { + s := slots[len(slots)-1] // nearest below beforeSlot + blk, err := c.GetBlock(ctx, s) + if err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + res := solcore.PreviousUnskippedBlock{Slot: s, Found: true} + if !blk.Skipped() { + res.BlockTime = blk.BlockTime + } + return res, nil + } + if lo <= first { + return solcore.PreviousUnskippedBlock{}, nil // none in [first, beforeSlot) + } + hi = lo - 1 + } +} + +func (c *nativeClient) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *nativeClient) Snapshot() any { + return map[string]any{ + "mode": "native", + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + "getLatestTimeout": c.getLatestTimeout.String(), + }, + "savedFirstBlockNumber": c.savedFirstBlockNumber.Load(), + "savedLatestBlockNumber": c.savedLatestBlockNumber.Load(), + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, func(block Block) string { + if block.Skipped() { + return fmt.Sprintf("%d/", block.Slot) + } + return controller.GetBlockFullText(block) + }), + }, + } +} diff --git a/driver/controller/data/sol/transaction.go b/driver/controller/data/sol/transaction.go new file mode 100644 index 0000000..51e1ac8 --- /dev/null +++ b/driver/controller/data/sol/transaction.go @@ -0,0 +1,108 @@ +package sol + +import ( + "context" + "time" + + "github.com/gagliardetto/solana-go" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// TransactionRequirement is one instruction handler's demand: the programs it indexes over a block +// range. +type TransactionRequirement struct { + controller.BlockRange + + Programs []solana.PublicKey +} + +func (r TransactionRequirement) Snapshot() any { + programs := make([]string, len(r.Programs)) + for i, p := range r.Programs { + programs[i] = p.String() + } + return map[string]any{ + "programs": programs, + "range": r.BlockRange.String(), + } +} + +// MergeTxRequirements cuts the requirements (from currentBlockNumber onward) into disjoint ranges, +// each carrying only the programs whose range covers it. This avoids fetching a program's +// transactions over sub-ranges it does not need (e.g. a program whose start is far ahead of the +// current progress is not queried over the gap before its start). +func MergeTxRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + seen := set.New[solana.PublicKey]() + for _, req := range reqs { + if req.BlockRange.Include(r) { + seen.Add(req.Programs...) + } + } + programs := seen.DumpValues() + if len(programs) == 0 { + continue + } + result = append(result, TransactionRequirement{BlockRange: r, Programs: programs}) + } + return result +} + +// BuildTxFetcher builds the fetcher for one disjoint range and its program set, fetching the full +// matching transactions per block via sol_findTransactions. +func BuildTxFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + // minQuerySize 1: the super node errors when a multi-block range exceeds its transaction cap, + // so the fetcher must be able to shrink to a single block (where the cap no longer applies). + 1, + 10000, + targetKeepBytes, + targetQueryBytes, + time.Second*15, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + // The super node caps the result and errors when exceeded (except for a single block); + // that error propagates and the fetcher halves the range, so no client-side check here. + blocks, err := client.FindTransactions(ctx, start, end, req.Programs) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData, len(blocks)) + for _, b := range blocks { + result[b.Slot] = BlockMainData{ + Slot: b.Slot, + Blockhash: b.Blockhash.String(), + PreviousBlockhash: b.PreviousBlockhash.String(), + BlockTime: b.BlockTime, + Transactions: b.Transactions, + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/statistics.go b/driver/controller/data/statistics.go new file mode 100644 index 0000000..d6bf6be --- /dev/null +++ b/driver/controller/data/statistics.go @@ -0,0 +1,164 @@ +package data + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/queue" + "sentioxyz/sentio-core/common/timehist" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" +) + +func getCaller() string { + return fmt.Sprintf("%+v", errors.Errorf("")) +} + +type call struct { + Caller string + Method string + Params string + Err error + WaitUsed time.Duration + Used time.Duration + EndAt time.Time +} + +func (c call) Snapshot() any { + r := map[string]any{ + "endAt": c.EndAt.String(), + "waitUsed": c.WaitUsed.String(), + "used": c.Used.String(), + "method": c.Method, + "caller": c.Caller, + } + if len(c.Params) > 0 { + r["params"] = c.Params + } + if c.Err != nil { + r["err"] = c.Err.Error() + } + return r +} + +type statWindow struct { + StartAt time.Time + Count map[string]int + FailedCount map[string]int + TotalUsed map[string]time.Duration + TotalWaitUsed map[string]time.Duration + Used map[string]timehist.Histogram + WaitUsed map[string]timehist.Histogram +} + +func newStatWindow(c call) *statWindow { + w := &statWindow{ + StartAt: c.EndAt, + Count: make(map[string]int), + FailedCount: make(map[string]int), + TotalUsed: make(map[string]time.Duration), + TotalWaitUsed: make(map[string]time.Duration), + Used: make(map[string]timehist.Histogram), + WaitUsed: make(map[string]timehist.Histogram), + } + w.Count[c.Method] += 1 + if c.Err != nil { + w.FailedCount[c.Method] += 1 + } + w.TotalUsed[c.Method] += c.Used + w.Used[c.Method] = w.Used[c.Method].Incr(c.Used) + w.TotalWaitUsed[c.Method] += c.WaitUsed + w.WaitUsed[c.Method] = w.WaitUsed[c.Method].Incr(c.WaitUsed) + return w +} + +func (w *statWindow) GetStartAt() time.Time { + return w.StartAt +} + +func (w *statWindow) Merge(a *statWindow) { + for method, v := range a.Count { + w.Count[method] += v + } + for method, v := range a.FailedCount { + w.FailedCount[method] += v + } + for method, v := range a.TotalUsed { + w.TotalUsed[method] += v + } + for method, v := range a.TotalWaitUsed { + w.TotalWaitUsed[method] += v + } + for method, th := range a.Used { + w.Used[method] = w.Used[method].Add(th) + } + for method, th := range a.WaitUsed { + w.WaitUsed[method] = w.WaitUsed[method].Add(th) + } +} + +func (w *statWindow) Snapshot(endAt time.Time) any { + return map[string]any{ + "startAt": w.StartAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(w.StartAt).String(), + "count": w.Count, + "failed": w.FailedCount, + "totalUsed": utils.MapMapNoError(w.TotalUsed, time.Duration.String), + "used": utils.MapMapNoError(w.Used, timehist.Histogram.String), + // waitUsed is the time spent waiting for a resource-manager concurrency token before + // the actual RPC; (totalUsed - totalWaitUsed) is therefore the real on-the-wire latency. + // Surfacing it lets us tell network-bound calls apart from token-contention-bound ones. + "totalWaitUsed": utils.MapMapNoError(w.TotalWaitUsed, time.Duration.String), + "waitUsed": utils.MapMapNoError(w.WaitUsed, timehist.Histogram.String), + } +} + +type CallStatistics struct { + latest queue.Circular[call] + stat *timewin.TimeWindowsManager[*statWindow] +} + +var ( + clientKeepRecentRequestCount = envconf.LoadUInt64("SENTIO_CLIENT_KEEP_RECENT_REQUEST_COUNT", + 1000, envconf.WithMin(10)) + clientStatTimeWindowWidth = envconf.LoadDuration("SENTIO_CLIENT_STAT_TIME_WINDOW_WIDTH", + time.Minute, envconf.WithMinDuration(time.Second*30)) +) + +func NewDefaultCallStatistics() *CallStatistics { + return NewCallStatistics(int(clientKeepRecentRequestCount), clientStatTimeWindowWidth) +} + +func NewCallStatistics(latestNum int, winWidth time.Duration) *CallStatistics { + return &CallStatistics{ + latest: queue.NewSafeCircular[call](latestNum), + stat: timewin.NewTimeWindowsManager[*statWindow](winWidth), + } +} + +func (s *CallStatistics) Called(method string, args []any, err error, startAt, waitEndAt time.Time) { + c := call{ + Caller: getCaller(), + Method: method, + Err: err, + EndAt: time.Now(), + } + c.Used = c.EndAt.Sub(startAt) + c.WaitUsed = waitEndAt.Sub(startAt) + if len(args) > 0 { + c.Params = utils.MustJSONMarshal(args) + } + s.latest.Push(c) + s.stat.Append(newStatWindow(c)) +} + +func (s *CallStatistics) Snapshot() any { + return map[string]any{ + "recent": utils.MapSliceNoError(s.latest.Dump(true), call.Snapshot), + "statistics": s.stat.Snapshot(), + } +} diff --git a/driver/controller/data/statistics_test.go b/driver/controller/data/statistics_test.go new file mode 100644 index 0000000..b6c8cf8 --- /dev/null +++ b/driver/controller/data/statistics_test.go @@ -0,0 +1,31 @@ +package data + +import ( + "encoding/json" + "math/rand" + "sync" + "testing" + "time" +) + +func Test_CallStatistics(t *testing.T) { + cs := NewCallStatistics(5, time.Millisecond*100) + + var w sync.WaitGroup + for i := 0; i < 1000; i++ { + time.Sleep(time.Millisecond) + w.Add(1) + go func(i int) { + defer w.Done() + startAt := time.Now() + time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10))) + waitEndAt := time.Now() + time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10))) + cs.Called("foo", []any{i}, nil, startAt, waitEndAt) + }(i) + } + w.Wait() + + b, _ := json.MarshalIndent(cs.Snapshot(), "", " ") + t.Logf(string(b)) +} diff --git a/driver/controller/data/subscribe.go b/driver/controller/data/subscribe.go new file mode 100644 index 0000000..983116a --- /dev/null +++ b/driver/controller/data/subscribe.go @@ -0,0 +1,114 @@ +package data + +import ( + "context" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/cenkalti/backoff/v4" + "github.com/pkg/errors" +) + +func SubscribeUsingPolling( + ctx context.Context, + minWatchInterval time.Duration, + timeout time.Duration, + from controller.BlockHeader, + latestGetter func(context.Context) (controller.BlockHeader, error), + callback func(controller.BlockHeader, error), +) { + const maxWaiting = time.Minute * 3 + _, logger := log.FromContext(ctx) + watchInterval := minWatchInterval // estimated block interval + waiting := watchInterval + for { + getCtx, cancel := context.WithTimeout(ctx, timeout) + latest, err := latestGetter(getCtx) + cancel() + if err != nil { + waiting = min(waiting*2, maxWaiting) + logger.Warnfe(err, "get latest for subscribe failed, will retry after %s", waiting.String()) + } else if latest.GetBlockNumber() < from.GetBlockNumber() { + waiting = min(waiting*2, maxWaiting) + logger.Warnf("latest from %s back to %s, will be ignored, and will retry after %s", + controller.GetBlockSummary(from), controller.GetBlockSummary(latest), waiting.String()) + } else if latest.GetBlockNumber() == from.GetBlockNumber() { + passed := time.Since(from.GetBlockTime()) + if passed < time.Minute { + waiting = watchInterval + logger.Debugf("latest stay at %s, and passed %s, will retry after %s", + controller.GetBlockSummary(from), passed, waiting.String()) + } else { + waiting = min(waiting*2, maxWaiting) + logger.Warnf("latest stay at %s, and passed %s, will retry after %s", + controller.GetBlockSummary(from), passed, waiting.String()) + } + } else { + if latest.GetBlockTime().After(from.GetBlockTime()) { + timeDelta := latest.GetBlockTime().Sub(from.GetBlockTime()) + blockDelta := latest.GetBlockNumber() - from.GetBlockNumber() + watchInterval = max(timeDelta/time.Duration(blockDelta), minWatchInterval) + } else { + watchInterval = minWatchInterval + } + waiting = watchInterval + logger.Debugf("latest growth from %s to %s, will get latest again after %s", + controller.GetBlockSummary(from), controller.GetBlockSummary(latest), waiting.String()) + callback(latest, nil) + from = latest + } + select { + case <-time.After(waiting): + case <-ctx.Done(): + return + } + } +} + +func SubscribeUsingWaiting( + ctx context.Context, + queryInterval time.Duration, + from controller.BlockHeader, + waitLatest func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error), + callback func(controller.BlockHeader, error), +) { + _, logger := log.FromContext(ctx) + var broken error + for broken == nil { + if queryInterval > 0 { + select { + case <-time.After(queryInterval): + case <-ctx.Done(): + return + } + } + fromText := controller.GetBlockSummary(from) + broken = backoff.RetryNotify( + func() error { + callCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + latest, brokenErr, err := waitLatest(callCtx, from.GetBlockNumber()) + if err != nil { + return err + } else if brokenErr != nil { + logger.Errorfe(broken, "wait latest from %s for subscribe failed", fromText) + callback(latest, brokenErr) + return backoff.Permanent(brokenErr) + } else if latest.GetBlockNumber() < from.GetBlockNumber() { + return errors.Errorf("latest from %s back to %s", fromText, controller.GetBlockSummary(latest)) + } else if latest.GetBlockNumber() == from.GetBlockNumber() { + return errors.Errorf("latest stay at %s", fromText) + } + logger.Debugf("latest growth from %s to %s", fromText, controller.GetBlockSummary(latest)) + callback(latest, nil) + from = latest + return nil + }, + backoff.WithContext(backoff.NewExponentialBackOff(backoff.WithMaxElapsedTime(0)), ctx), + func(err error, duration time.Duration) { + logger.Warnfe(err, "wait latest from %s for subscribe failed, will retry after %s", fromText, duration.String()) + }) + } +} diff --git a/driver/controller/data/sui/BUILD.bazel b/driver/controller/data/sui/BUILD.bazel new file mode 100644 index 0000000..178026e --- /dev/null +++ b/driver/controller/data/sui/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sui", + srcs = [ + "block.go", + "client.go", + "object_change.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sui", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//chain/sui/types", + "//common/concurrency", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + ], +) + +go_test( + name = "sui_test", + srcs = ["block_test.go"], + embed = [":sui"], + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/driver/controller/data/sui/block.go b/driver/controller/data/sui/block.go new file mode 100644 index 0000000..f3fc76a --- /dev/null +++ b/driver/controller/data/sui/block.go @@ -0,0 +1,198 @@ +package sui + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type SimpleBlock sui.SimpleCheckpoint + +func (b SimpleBlock) GetBlockNumber() uint64 { + return b.Checkpoint +} + +func (b SimpleBlock) GetBlockParentHash() string { + return "" +} + +func (b SimpleBlock) GetBlockHash() string { + return b.Digest +} + +func (b SimpleBlock) GetBlockTime() time.Time { + return time.UnixMilli(int64(b.TimestampMS)) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetSimpleBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + // timeGetter above already warmed GetSimpleBlock for these blocks, so this mostly hits cache. + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} + +type BlockMainData struct { + Txs []types.TransactionResponseV1 + ObjectChanges []types.ObjectChangeExtend + Intervals []data.IntervalConfig + // SimpleBlock is the checkpoint header, prefetched concurrently by the data fetchers alongside the + // block's data. The strictly block-ordered transfer step (handler.go) needs the header for every + // non-empty block; fetching it here (off the serial path) keeps sui_getSimpleCheckpoint — which is + // order-independent — from becoming the throughput bottleneck. nil means it wasn't prefetched, in + // which case the transfer step falls back to a (serial) Client.GetSimpleBlock. + SimpleBlock *SimpleBlock +} + +func (b BlockMainData) IsEmpty() bool { + return b.Size() == 0 +} + +// Size intentionally ignores SimpleBlock: it is header metadata that only ever rides along with real +// data, and must not on its own make an otherwise-empty block look non-empty. +func (b BlockMainData) Size() int { + return len(b.Intervals) + len(b.ObjectChanges) + len(b.Txs)*10 +} + +// attachSimpleBlocks fetches the checkpoint header for every block that already carries data and stores +// it on the BlockMainData, so the downstream block-ordered transfer can avoid a serial RPC. It runs in +// the (concurrent, read-ahead) fetcher goroutines, and GetSimpleBlock is cached + singleflighted, so +// overlapping fetchers requesting the same block don't produce duplicate sui_getSimpleCheckpoint calls. +func attachSimpleBlocks(ctx context.Context, client Client, result map[uint64]BlockMainData) error { + if len(result) == 0 { + return nil + } + // Snapshot the keys first: the worker goroutines start immediately on Go(), so ranging the map here + // while they write back into it would be a concurrent map iteration + write (a runtime panic). Each + // worker writes its own headers[i] (distinct index, no lock needed); the map is mutated only after + // Wait(), back on this goroutine. + bns := make([]uint64, 0, len(result)) + for bn := range result { + bns = append(bns, bn) + } + headers := make([]SimpleBlock, len(bns)) + // No concurrency limit here: GetSimpleBlock goes through the client's resource manager, which + // already bounds in-flight RPCs. + g, gctx := errgroup.WithContext(ctx) + for i, bn := range bns { + i, bn := i, bn // nogo's loopclosure analyzer predates Go 1.22 per-iteration loop vars + g.Go(func() error { + sb, err := client.GetSimpleBlock(gctx, bn) + if err != nil { + return err + } + headers[i] = sb + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + for i, bn := range bns { + d := result[bn] + d.SimpleBlock = &headers[i] + result[bn] = d + } + return nil +} + +type DataRequirement struct { + Interval []data.IntervalRequirement + ObjectChanges []ObjectChangeRequirement + Txn []TransactionRequirement +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.ObjectChanges = MergeObjectChangeRequirements(currentBlockNumber, req.ObjectChanges) + req.Txn = MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.ObjectChanges { + fetchers = append(fetchers, BuildObjectChangeFetcher( + namePrefix+fmt.Sprintf("ObjectChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // ObjectChanges and Txn never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Txs = append(data.Txs, box.Txs...) + data.ObjectChanges = append(data.ObjectChanges, box.ObjectChanges...) + data.Intervals = append(data.Intervals, box.Intervals...) + if data.SimpleBlock == nil && box.SimpleBlock != nil { + data.SimpleBlock = box.SimpleBlock + } + } + return + }) +} diff --git a/driver/controller/data/sui/block_test.go b/driver/controller/data/sui/block_test.go new file mode 100644 index 0000000..3fd0a26 --- /dev/null +++ b/driver/controller/data/sui/block_test.go @@ -0,0 +1,48 @@ +package sui + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeSimpleBlockClient implements just the one method attachSimpleBlocks needs. The embedded nil +// Client interface makes any other call panic, keeping the fake honest about what it exercises. +type fakeSimpleBlockClient struct { + Client + calls atomic.Int64 +} + +func (f *fakeSimpleBlockClient) GetSimpleBlock(_ context.Context, bn uint64) (SimpleBlock, error) { + f.calls.Add(1) + return SimpleBlock{Checkpoint: bn}, nil +} + +// TestAttachSimpleBlocks guards the concurrency contract: it must fetch every block's header exactly +// once and pair each header with the right block, with no concurrent access to the result map. Run +// under -race, the many worker goroutines would trip the detector if the map were touched off the +// main goroutine, or the per-block pairing would break if the index bookkeeping were wrong. +func TestAttachSimpleBlocks(t *testing.T) { + const n = 1000 + result := make(map[uint64]BlockMainData, n) + for i := uint64(1); i <= n; i++ { + result[i] = BlockMainData{} + } + client := &fakeSimpleBlockClient{} + + require.NoError(t, attachSimpleBlocks(context.Background(), client, result)) + + require.EqualValues(t, n, client.calls.Load(), "each block fetched exactly once") + for bn, d := range result { + require.NotNilf(t, d.SimpleBlock, "block %d missing prefetched header", bn) + require.Equalf(t, bn, d.SimpleBlock.Checkpoint, "block %d paired with wrong header", bn) + } +} + +func TestAttachSimpleBlocksEmpty(t *testing.T) { + client := &fakeSimpleBlockClient{} + require.NoError(t, attachSimpleBlocks(context.Background(), client, map[uint64]BlockMainData{})) + require.Zero(t, client.calls.Load(), "no fetch for an empty result set") +} diff --git a/driver/controller/data/sui/client.go b/driver/controller/data/sui/client.go new file mode 100644 index 0000000..1979360 --- /dev/null +++ b/driver/controller/data/sui/client.go @@ -0,0 +1,496 @@ +package sui + +import ( + "context" + "math" + "strings" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + lru "github.com/sentioxyz/golang-lru" + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetSimpleBlock(ctx context.Context, blockNumber uint64) (SimpleBlock, error) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, + ) (map[uint64][]types.ObjectChangeExtend, error) + GetTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, + ) (map[uint64][]types.TransactionResponseV1, error) + + // grpc-format counterparts (super node DriverVersion[2]); used by the suigrpc handler path. + GetGrpcTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, + ) (map[uint64][]*sui.ExtendedGrpcTransaction, error) + GetGrpcObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, + ) (map[uint64][]*sui.ExtendedGrpcChangedObject, error) + GetGrpcObjects( + ctx context.Context, + reqs []*rpcv2.GetObjectRequest, + concurrency, batchSize int, + ) ([]*rpcv2.GetObjectResult, error) + + TryMultiGetPastObjects( + ctx context.Context, + requests []types.SuiGetPastObjectRequest, + options types.SuiObjectDataOptions, + ) ([]types.SuiPastObjectResponse, error) + MultiGetTransactionBlocks( + ctx context.Context, + txDigests []string, + options map[string]any, + ) ([]types.TransactionResponseV1, error) + GetObjectStat(ctx context.Context, fromBlock, toBlock uint64, objectID string) (sui.ObjectStat, error) + GetObjectsStat(ctx context.Context, fromBlock, toBlock uint64, objectIDList []string) ([]sui.ObjectStat, error) + GetObjectVersionHistory(ctx context.Context, objectID string) ([]types.ObjectChangeExtend, error) + + GetPackageHistory(ctx context.Context, pkgID string) ([]string, error) + GetObjectCreation(ctx context.Context, objectID string, start uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedSimpleBlock *data.BlockCache[SimpleBlock] + cachedPackageHistory *lru.Cache[string, []string] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedSimpleBlock, _ = data.NewBlockCache[SimpleBlock](100000) + cli.cachedPackageHistory, _ = lru.New[string, []string](10000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + if err = c.cli.CallContext(ctx, &result, method, args...); err != nil { + return errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return nil +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp sui.GetLatestSimpleCheckpointResponse + err = c.callContext(ctx, &resp, 0, "sui_getLatestSimpleCheckpoint", blockNumberGt) + if err == nil { + latest, broken = SimpleBlock(resp.Checkpoint), resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + var resp sui.GetLatestSimpleCheckpointResponse + if err = c.callContext(ctx, &resp, 0, "sui_getLatestSimpleCheckpoint", 0); err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest = SimpleBlock(resp.Checkpoint) + return latest, data.GetFirst(c.firstBlockNumber, latest.GetBlockNumber()), err +} + +func (c *client) fetchSimpleBlock(ctx context.Context, blockNumber uint64) (sc SimpleBlock, err error) { + err = c.callContext(ctx, &sc, blockNumber, "sui_getSimpleCheckpoint", blockNumber) + return +} + +func (c *client) getSimpleBlockIgnoreCache(ctx context.Context, blockNumber uint64) (SimpleBlock, error) { + sc, err := c.fetchSimpleBlock(ctx, blockNumber) + if err == nil { + c.cachedSimpleBlock.Add(blockNumber, sc) + } + return sc, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getSimpleBlockIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetSimpleBlock(ctx context.Context, blockNumber uint64) (SimpleBlock, error) { + // The cache + singleflight (in BlockCache) collapses the concurrent prefetch requests for the same + // checkpoint — made by the object-change / txn / interval fetchers — into a single RPC. + return c.cachedSimpleBlock.GetOrFetch(blockNumber, func() (SimpleBlock, error) { + return c.fetchSimpleBlock(ctx, blockNumber) + }) +} + +func (c *client) GetObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, +) (map[uint64][]types.ObjectChangeExtend, error) { + var result []types.ObjectChangeExtend + err := c.callContext(ctx, &result, 0, "sui_filterObjectChangesV2", fromBlock, toBlock, filter) + return utils.Group(result, func(oc types.ObjectChangeExtend) uint64 { + return oc.Checkpoint.Uint64() + }), err +} + +func (c *client) GetTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, +) (map[uint64][]types.TransactionResponseV1, error) { + var result []types.TransactionResponseV1 + err := c.callContext(ctx, &result, 0, "sui_getTransactionsV2", fromBlock, toBlock, filter, fetchConfig) + return utils.Group(result, func(oc types.TransactionResponseV1) uint64 { + return oc.Checkpoint.Uint64() + }), err +} + +func (c *client) GetGrpcTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, +) (map[uint64][]*sui.ExtendedGrpcTransaction, error) { + var result []*sui.ExtendedGrpcTransaction + err := c.callContext(ctx, &result, 0, "sui_getGrpcTransactions", fromBlock, toBlock, filter, fetchConfig) + return utils.Group(result, func(tx *sui.ExtendedGrpcTransaction) uint64 { + return tx.Checkpoint + }), err +} + +func (c *client) GetGrpcObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, +) (map[uint64][]*sui.ExtendedGrpcChangedObject, error) { + var result []*sui.ExtendedGrpcChangedObject + err := c.callContext(ctx, &result, 0, "sui_filterGrpcChangedObjects", fromBlock, toBlock, filter) + return utils.Group(result, func(oc *sui.ExtendedGrpcChangedObject) uint64 { + return oc.Checkpoint + }), err +} + +// GetGrpcObjects fetches objects by id+version in grpc format. The super node batches/parallelizes +// server-side per the concurrency/batchSize args, so no client-side paging is needed. +func (c *client) GetGrpcObjects( + ctx context.Context, + reqs []*rpcv2.GetObjectRequest, + concurrency, batchSize int, +) ([]*rpcv2.GetObjectResult, error) { + // GetObjectResult carries a protobuf oneof, which the JSON-RPC transport's + // encoding/json can't round-trip; decode into the protojson-backed wrapper + // (sui.GrpcObjectResult) and unwrap back to the raw proto for callers. + var wrapped []*sui.GrpcObjectResult + if err := c.callContext(ctx, &wrapped, 0, "sui_getGrpcObjects", reqs, concurrency, batchSize); err != nil { + return nil, err + } + return sui.UnwrapGrpcObjectResults(wrapped), nil +} + +const QueryObjectsPageSize = 50 // Max number of objects to request in a single call + +func (c *client) TryMultiGetPastObjects( + ctx context.Context, + requests []types.SuiGetPastObjectRequest, + options types.SuiObjectDataOptions, +) ([]types.SuiPastObjectResponse, error) { + var result []types.SuiPastObjectResponse + for len(requests) > 0 { + query := requests + if len(requests) > QueryObjectsPageSize { + query = requests[:QueryObjectsPageSize] + requests = requests[QueryObjectsPageSize:] + } else { + requests = requests[:0] + } + var pageResult []types.SuiPastObjectResponse + err := c.callContext(ctx, &pageResult, 0, "sui_tryMultiGetPastObjects", query, options) + if err != nil { + return nil, err + } + if len(pageResult) != len(query) { + return nil, errors.Errorf("call sui_tryMultiGetPastObjects failed: unexpected number of results: %d", len(pageResult)) + } + result = append(result, pageResult...) + } + return result, nil +} + +const QueryTxPageSize = 50 // Max number of txs to request in a single call + +func (c *client) MultiGetTransactionBlocks( + ctx context.Context, + txDigests []string, + options map[string]any, +) ([]types.TransactionResponseV1, error) { + txList := make([]types.TransactionResponseV1, 0, len(txDigests)) + for len(txDigests) > 0 { + query := txDigests + if len(txDigests) > QueryTxPageSize { + query = txDigests[:QueryTxPageSize] + txDigests = txDigests[QueryTxPageSize:] + } else { + txDigests = txDigests[:0] + } + var result []types.TransactionResponseV1 + err := c.callContext(ctx, &result, 0, "sui_multiGetTransactionBlocks", query, options) + if err != nil { + return nil, err + } + txList = append(txList, result...) + } + return txList, nil +} + +func (c *client) GetObjectStat(ctx context.Context, fromBlock, toBlock uint64, objectID string) (sui.ObjectStat, error) { + var result sui.ObjectStat + err := c.callContext(ctx, &result, fromBlock, "sui_getObjectStat", fromBlock, toBlock, objectID) + return result, err +} + +func (c *client) getObjectsStat( + ctx context.Context, + fromBlock, toBlock uint64, + objectIDList []string, +) ([]sui.ObjectStat, error) { + var dict map[string]sui.ObjectStat + err := c.callContext(ctx, &dict, fromBlock, "sui_getObjectsStat", fromBlock, toBlock, objectIDList) + result := make([]sui.ObjectStat, len(objectIDList)) + for i, id := range objectIDList { + result[i] = dict[id] + } + return result, err +} + +func (c *client) GetObjectsStat( + ctx context.Context, + fromBlock, toBlock uint64, + objectIDList []string, +) ([]sui.ObjectStat, error) { + const maxPageSize = 200 + const maxConcurrency = 20 + return concurrency.TraverseByPage(ctx, maxConcurrency, maxPageSize, objectIDList, + func(ctx context.Context, page concurrency.Page, ids []string) ([]sui.ObjectStat, error) { + return c.getObjectsStat(ctx, fromBlock, toBlock, ids) + }, + ) +} + +func (c *client) GetObjectVersionHistory(ctx context.Context, objectID string) ([]types.ObjectChangeExtend, error) { + stat, err := c.GetObjectStat(ctx, 0, math.MaxUint64, objectID) + if err != nil { + return nil, err + } + if stat.Count == 0 { + return nil, nil + } + filter := sui.ObjectChangeFilter{ + ObjectIDIn: set.New(objectID), + } + var changes map[uint64][]types.ObjectChangeExtend + if changes, err = c.GetObjectChanges(ctx, stat.MinCheckpoint, stat.MaxCheckpoint, filter); err != nil { + return nil, err + } + return utils.MergeArr(utils.GetMapValuesOrderByKey(changes)...), nil +} + +func (c *client) GetPackageHistory(ctx context.Context, pkgID string) (history []string, err error) { + var has bool + if history, has = c.cachedPackageHistory.Get(pkgID); has { + return history, nil + } + defer func() { + if err == nil { + c.cachedPackageHistory.Add(pkgID, history) + } + }() + // step-1: package object is immutable, so only have one version, just use sui_getObject to fetch it + type getObjectResponse struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + Data struct { + PreviousTransaction string `json:"previousTransaction"` + } `json:"data"` + } + var getObjectResp getObjectResponse + getObjectOpt := types.SuiObjectDataOptions{ShowPreviousTransaction: true} + if err = c.callContext(ctx, &getObjectResp, 0, "sui_getObject", pkgID, getObjectOpt); err != nil { + return nil, errors.Wrapf(err, "get package object %s failed", pkgID) + } else if getObjectResp.Error.Code != "" { + return nil, errors.Errorf("get package object %s failed: %s", pkgID, getObjectResp.Error.Code) + } + // step-2: the tx getObjectResp.Data.PreviousTransaction is the creating tx of the package, + // which contains the creating or update record of the upgrade cap object, + // use sui_getTransactionBlock to fetch the object changes in this tx + var pkgCreatingTxDigest = types.StrToDigestMust(getObjectResp.Data.PreviousTransaction) + var pkgCreateTx *types.TransactionResponseV1 + var getTxOpt = map[string]any{"showObjectChanges": true} + if err = c.callContext(ctx, &pkgCreateTx, 0, "sui_getTransactionBlock", pkgCreatingTxDigest, getTxOpt); err != nil { + err = errors.Wrapf(err, "get creation trransaction %s for package %s failed", pkgCreatingTxDigest.String(), pkgID) + return nil, err + } + const upgradeCapObjectType = "0x2::package::UpgradeCap" + var upgradeCapID string + for _, objectChange := range pkgCreateTx.ObjectChanges { + if utils.EmptyStringIfNil(utils.NullOrToString(objectChange.ObjectType)) == upgradeCapObjectType { + upgradeCapID = objectChange.GetObjectID() + break + } + } + if upgradeCapID == "" { + return []string{pkgID}, nil + } + // step-3: now have the upgrade cap object id, we should find the change history of it, + // which contains all upgraded package creating record. + // use super node to get all change records of the upgrade cap object, + // which contains all package creating tx digest. + var upgradeCapHistory []types.ObjectChangeExtend + upgradeCapHistory, err = c.GetObjectVersionHistory(ctx, upgradeCapID) + if err != nil { + err = errors.Wrapf(err, "get the first tx of the upgrade cap object %s for package %s failed", upgradeCapID, pkgID) + return nil, err + } + if len(upgradeCapHistory) == 0 { + err = errors.Errorf("the history of the upgrade cap object %s for package %s is not found", upgradeCapID, pkgID) + return nil, err + } + historyTxDigest := utils.MapSliceNoError(upgradeCapHistory, func(oc types.ObjectChangeExtend) string { + return oc.TxDigest.String() + }) + // step-4: now get the object changes of all history tx, all the package id is in it + var historyTxList []types.TransactionResponseV1 + historyTxList, err = c.MultiGetTransactionBlocks(ctx, historyTxDigest, getTxOpt) + if err != nil { + err = errors.Wrapf(err, "get history tx of the upgrade cap object %s for package %s failed", upgradeCapID, pkgID) + return nil, err + } + for _, tx := range historyTxList { + for _, change := range tx.ObjectChanges { + if change.Type == types.ObjectChangeTypePublished { + history = append(history, change.GetObjectID()) + } + } + } + if utils.IndexOf(history, pkgID) < 0 { + history = append(history, pkgID) + } + return history, nil +} + +func (c *client) GetObjectCreation(ctx context.Context, objectID string, start uint64) (uint64, bool, error) { + var creation *sui.ObjectCreation + err := c.callContext(ctx, &creation, start, "sui_getObjectCreation", objectID) + if err != nil { + return 0, false, err + } + if creation == nil { + return 0, false, nil + } + return max(creation.Checkpoint, start), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedSimpleBlock.Keys() { + if r.Contains(bn) { + c.cachedSimpleBlock.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedSimpleBlock": c.cachedSimpleBlock.Snapshot(10, controller.GetBlockFullText[SimpleBlock]), + "cachedPackageHistory": utils.CacheSnapshot(c.cachedPackageHistory, 100, func(history []string) string { + return strings.Join(history, ",") + }), + }, + } +} diff --git a/driver/controller/data/sui/grpc/BUILD.bazel b/driver/controller/data/sui/grpc/BUILD.bazel new file mode 100644 index 0000000..368b46b --- /dev/null +++ b/driver/controller/data/sui/grpc/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "grpc", + srcs = [ + "block.go", + "object_change.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sui/grpc", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//common/errgroup", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/fetcher", + ], +) diff --git a/driver/controller/data/sui/grpc/block.go b/driver/controller/data/sui/grpc/block.go new file mode 100644 index 0000000..ab0187f --- /dev/null +++ b/driver/controller/data/sui/grpc/block.go @@ -0,0 +1,169 @@ +// Package suigrpc is the grpc-data twin of data/sui: same fetch/requirement +// machinery, but the block data carries grpc-format transactions / object +// changes (sentio-core ExtendedGrpc*), fetched from the super node's +// sui_getGrpcTransactions / sui_filterGrpcChangedObjects methods. The +// format-agnostic pieces (Client, SimpleBlock, the *Requirement types and their +// Merge helpers, DataRequirement) are reused from data/sui. +package grpc + +import ( + "context" + "fmt" + "time" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// BlockMainData is the grpc-format counterpart of suidata.BlockMainData. +type BlockMainData struct { + Txs []*chainsui.ExtendedGrpcTransaction + ObjectChanges []*chainsui.ExtendedGrpcChangedObject + Intervals []data.IntervalConfig + // SimpleBlock is the checkpoint header, prefetched concurrently alongside the block's data so the + // strictly block-ordered transfer step doesn't need a serial RPC. nil means not prefetched. + SimpleBlock *suidata.SimpleBlock +} + +func (b BlockMainData) IsEmpty() bool { + return b.Size() == 0 +} + +// Size intentionally ignores SimpleBlock (header metadata only rides along with real data). +func (b BlockMainData) Size() int { + return len(b.Intervals) + len(b.ObjectChanges) + len(b.Txs)*10 +} + +// attachSimpleBlocks fetches the checkpoint header for every block and stores it on the BlockMainData. +// Only the interval fetcher needs it: those blocks carry no tx / object-change data to derive the header +// from (the txn and object-change fetchers read the header off the grpc data via GetSimpleCheckpoint). +// GetSimpleBlock is cached + singleflighted. +func attachSimpleBlocks(ctx context.Context, client suidata.Client, result map[uint64]BlockMainData) error { + if len(result) == 0 { + return nil + } + bns := make([]uint64, 0, len(result)) + for bn := range result { + bns = append(bns, bn) + } + headers := make([]suidata.SimpleBlock, len(bns)) + g, gctx := errgroup.WithContext(ctx) + for i, bn := range bns { + i, bn := i, bn + g.Go(func() error { + sb, err := client.GetSimpleBlock(gctx, bn) + if err != nil { + return err + } + headers[i] = sb + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + for i, bn := range bns { + d := result[bn] + d.SimpleBlock = &headers[i] + result[bn] = d + } + return nil +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetSimpleBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{Intervals: []data.IntervalConfig{req.IntervalConfig}} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} + +// BuildBlockMainDataFetcher mirrors suidata.BuildBlockMainDataFetcher but builds grpc fetchers. It +// reuses suidata.DataRequirement and the Merge* helpers (all format-agnostic). +func BuildBlockMainDataFetcher( + namePrefix string, + req suidata.DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + req.ObjectChanges = suidata.MergeObjectChangeRequirements(currentBlockNumber, req.ObjectChanges) + req.Txn = suidata.MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.ObjectChanges { + fetchers = append(fetchers, BuildObjectChangeFetcher( + namePrefix+fmt.Sprintf("ObjectChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (out BlockMainData, has bool, _ error) { + has = len(from) > 0 + for _, box := range from { + out.Txs = append(out.Txs, box.Txs...) + out.ObjectChanges = append(out.ObjectChanges, box.ObjectChanges...) + out.Intervals = append(out.Intervals, box.Intervals...) + if out.SimpleBlock == nil && box.SimpleBlock != nil { + out.SimpleBlock = box.SimpleBlock + } + } + return + }) +} diff --git a/driver/controller/data/sui/grpc/object_change.go b/driver/controller/data/sui/grpc/object_change.go new file mode 100644 index 0000000..da95745 --- /dev/null +++ b/driver/controller/data/sui/grpc/object_change.go @@ -0,0 +1,53 @@ +package grpc + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/controller" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +func BuildObjectChangeFetcher( + name string, + req suidata.ObjectChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 10000, + 100000, + 10000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + changes, err := client.GetGrpcObjectChanges(ctx, start, end, req.Filter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, cs := range changes { + if len(cs) > 0 { + // each grpc object change already carries its checkpoint header — no extra RPC needed + result[bn] = BlockMainData{ + ObjectChanges: cs, + SimpleBlock: new(suidata.SimpleBlock(cs[0].GetSimpleCheckpoint())), + } + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/grpc/transaction.go b/driver/controller/data/sui/grpc/transaction.go new file mode 100644 index 0000000..516a139 --- /dev/null +++ b/driver/controller/data/sui/grpc/transaction.go @@ -0,0 +1,54 @@ +package grpc + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/controller" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +func BuildTxnFetcher( + name string, + req suidata.TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 10000, + 10000, + 5000, + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txGroups, err := client.GetGrpcTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, txs := range txGroups { + + if len(txs) > 0 { + // each grpc tx already carries its checkpoint header — no extra GetSimpleBlock RPC needed + result[bn] = BlockMainData{ + Txs: txs, + SimpleBlock: new(suidata.SimpleBlock(txs[0].GetSimpleCheckpoint())), + } + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/object_change.go b/driver/controller/data/sui/object_change.go new file mode 100644 index 0000000..da1fee0 --- /dev/null +++ b/driver/controller/data/sui/object_change.go @@ -0,0 +1,89 @@ +package sui + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type ObjectChangeRequirement struct { + controller.BlockRange + + Filter sui.ObjectChangeFilter +} + +func (r ObjectChangeRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "range": r.BlockRange.String(), + } +} + +func MergeObjectChangeRequirements(current uint64, reqs []ObjectChangeRequirement) (result []ObjectChangeRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r ObjectChangeRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + var filters []sui.ObjectChangeFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.Filter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, ObjectChangeRequirement{ + Filter: utils.Reduce(filters, sui.ObjectChangeFilter.Merge), + BlockRange: r, + }) + } + return result +} + +func BuildObjectChangeFetcher( + name string, + req ObjectChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 10000, + 100000, + 10000, // the target is that each query got no more than 1000 object change records + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + changes, err := client.GetObjectChanges(ctx, start, end, req.Filter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, cs := range changes { + result[bn] = BlockMainData{ObjectChanges: cs} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/transaction.go b/driver/controller/data/sui/transaction.go new file mode 100644 index 0000000..b8e7b48 --- /dev/null +++ b/driver/controller/data/sui/transaction.go @@ -0,0 +1,97 @@ +package sui + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type TransactionRequirement struct { + controller.BlockRange + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "fetchConfig": r.FetchConfig, + "range": r.BlockRange.String(), + } +} + +func MergeTxnRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + rr := TransactionRequirement{BlockRange: r} + first := true + for _, req := range reqs { + if !req.BlockRange.Include(r) { + continue + } + if first { + rr.Filter = req.Filter + rr.FetchConfig = req.FetchConfig + first = false + } else { + rr.Filter = rr.Filter.Merge(req.Filter) + rr.FetchConfig = rr.FetchConfig.Merge(req.FetchConfig) + } + } + if first { + continue + } + result = append(result, rr) + } + return result +} + +func BuildTxnFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 10000, + 10000, // size of transaction is 10, so will cache 1000 transactions + 5000, // the target is that each query got no more than 500 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txGroups, err := client.GetTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, txs := range txGroups { + result[bn] = BlockMainData{Txs: txs} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} diff --git a/driver/controller/entity.go b/driver/controller/entity.go new file mode 100644 index 0000000..ea90449 --- /dev/null +++ b/driver/controller/entity.go @@ -0,0 +1,125 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" +) + +type EntityController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, + ) (created, updated map[string]int, err *ExternalError) + + GetEntityOrInterfaceType(entity string) schema.EntityOrInterface + GetEntityType(entity string) *schema.Entity + GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, + ) (boxes []*persistent.EntityBox, next *string, err *ExternalError) + ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, + ) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) + SetEntity(ctx context.Context, entityType *schema.Entity, box persistent.UncommittedEntityBox) *ExternalError + + Snapshot() any +} + +type EmptyEntityController struct{} + +func (c EmptyEntityController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyEntityController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyEntityController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, map[string]int, *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) GetEntityOrInterfaceType(entity string) schema.EntityOrInterface { + return nil +} + +func (c EmptyEntityController) GetEntityType(entity string) *schema.Entity { + return nil // all are unknown entity type +} + +func (c EmptyEntityController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + return nil, nil +} + +func (c EmptyEntityController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + return nil, nil +} + +func (c EmptyEntityController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) (boxes []*persistent.EntityBox, next *string, err *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) SetEntity(context.Context, *schema.Entity, persistent.UncommittedEntityBox) *ExternalError { + return nil +} + +func (c EmptyEntityController) Snapshot() any { + return nil +} diff --git a/driver/controller/errors.go b/driver/controller/errors.go new file mode 100644 index 0000000..9214600 --- /dev/null +++ b/driver/controller/errors.go @@ -0,0 +1,152 @@ +package controller + +import ( + "fmt" + "io" + + "github.com/pkg/errors" +) + +var ( + ErrInternalNeedUpgrade = errors.New("need upgrade") + ErrInternalHasNewTemplate = errors.New("has new template") + ErrInternalReorgDetected = errors.New("reorg detected") +) + +func NewExternalError(code int, err error) *ExternalError { + if err == nil { + panic(errors.Errorf("NewExternalError with code %d and nil err", code)) + } + return &ExternalError{code: code, error: err} +} + +type ExternalError struct { + code int + error +} + +func (e *ExternalError) Code() int { + return e.code +} + +func (e *ExternalError) Wrapped() error { + return e.error +} + +func (e *ExternalError) Wrapf(fmt string, args ...any) *ExternalError { + return NewExternalError(e.code, errors.Wrapf(e.error, fmt, args...)) +} + +func (e *ExternalError) Error() string { + return fmt.Sprintf("ERR%03d: %s", e.code, e.error.Error()) +} + +func (e *ExternalError) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + _, _ = fmt.Fprintf(s, "ERR%03d: %+v\n", e.code, e.error) + return + } + fallthrough + case 's', 'q': + _, _ = io.WriteString(s, e.Error()) + } +} + +func (e *ExternalError) IsUserError() bool { + return e.code >= 300 +} + +func (e *ExternalError) IsUserRuntimeError() bool { + switch e.code { + case ErrCodeProcessFailed: + return true + default: + return false + } +} + +func (e *ExternalError) IsSystemError() bool { + return e.code < 200 +} + +func (e *ExternalError) IsDriverError() bool { + return e.code >= 200 && e.code < 300 +} + +func (e *ExternalError) IsBillingError() bool { + return e.code >= 400 +} + +// other error +const ( + ErrCodeSystem = iota + 100 + ErrCodeNeedUpgrade + ErrCodeOOM +) + +// driver error +const ( + ErrCodeCallProcessorFailed = iota + 200 + ErrCodeResetWasmInstanceFailed + ErrCodeWasmError + ErrCodeGetContractStartBlockFailed + ErrCodeFetchDataFailed + ErrCodeSubgraphEthCallFailed + ErrCodeSubgraphIpfsCatFailed + + ErrCodeInvalidCheckpointData + ErrCodeSaveCheckpointFailed + ErrCodeQuotaServiceError + + ErrCodeCleanTimeSeriesDataFailed + ErrCodeSaveTimeSeriesDataFailed + + ErrCodeSendWebhookDataFailed + + ErrCodeInitEntityFailed + ErrCodeCleanEntityDataFailed + ErrCodeSaveEntityDataFailed + ErrCodeGetEntityFromDBFailed + ErrCodeListEntityFromDBFailed + ErrCodeInvalidEntityData +) + +// processor error +const ( + ErrCodeUnexpectedProcessorConfig = iota + 300 + ErrCodeInvalidEntitySchema + ErrCodeProcessorConfigsHasDiff + ErrCodeProcessFailed + ErrCodeCallWasmExportFunctionFailed + ErrCodeWasmInitFailed + ErrCodeWasmStackOverFlow + ErrCodeCreateTemplateFailed + ErrCodeInvalidSubgraphManifest + + ErrCodeGetUnknownEntity + ErrCodeListUnknownEntity + ErrCodeListRelatedEntityWithInvalidField + ErrCodeInvalidListEntityFilter + ErrCodeInvalidUpsertEntityRequest + ErrCodeUpsertUnknownEntity + ErrCodeInvalidUpdateEntityRequest + ErrCodeUpdateUnknownEntity + ErrCodeInvalidDeleteEntityRequest + ErrCodeDeleteUnknownEntity + ErrCodeUpdateImmutableEntity + ErrCodeInvalidEntityFieldValue + + ErrCodeInvalidTimeSeriesData + ErrCodeTimeSeriesDataSchemaChanged + + ErrCodeTooManyWebhookMsgEntity + + ErrCodeSubgraphEthCallWithInvalidParam +) + +// billing error +const ( + ErrCodeOverQuota = iota + 400 +) diff --git a/driver/controller/errors_test.go b/driver/controller/errors_test.go new file mode 100644 index 0000000..a3bc948 --- /dev/null +++ b/driver/controller/errors_test.go @@ -0,0 +1,87 @@ +package controller + +import ( + "context" + "testing" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func Test_externalErrorWrap(t *testing.T) { + err := errors.Errorf("level1") + extErr1 := NewExternalError(ErrCodeSystem, err) + extErr2 := extErr1.Wrapf("level2") + assert.Equal(t, "level1", err.Error()) + assert.Equal(t, "ERR100: level1", extErr1.Error()) + assert.Equal(t, "ERR100: level2: level1", extErr2.Error()) + + var extErr *ExternalError + assert.False(t, errors.As(err, &extErr)) + assert.True(t, errors.As(extErr1, &extErr)) + assert.Equal(t, extErr1, extErr) + assert.True(t, errors.As(extErr2, &extErr)) + assert.Equal(t, extErr2, extErr) + + log.Errorfe(err, "err") + log.Errorfe(extErr1, "extErr1") + log.Errorfe(extErr2, "extErr2") + log.Errorf("err: %+v", err) + log.Errorf("extErr1: %+v", extErr1) + log.Errorf("extErr2: %+v", extErr2) +} + +func Test_errgroup(t *testing.T) { + g, gctx := errgroup.WithContext(context.Background()) + fn := func(ctx context.Context, wait time.Duration, code int) *ExternalError { + select { + case <-ctx.Done(): + return NewExternalError(0, ctx.Err()) + case <-time.After(wait): + if code == 0 { + return nil + } + return NewExternalError(code, errors.Errorf("err")) + } + } + g.Go(func() error { + if err := fn(gctx, time.Millisecond*100, 0); err != nil { // got nil + return err + } + return nil + }) + g.Go(func() error { + if err := fn(gctx, time.Millisecond*200, 2); err != nil { // got code:2 + return err + } + return nil + }) + g.Go(func() error { + if err := fn(gctx, time.Millisecond*300, 3); err != nil { // got code:0 and ignored + return err + } + return nil + }) + + err := g.Wait() + var extErr *ExternalError + assert.True(t, errors.As(err, &extErr)) + assert.Equal(t, 2, extErr.code) +} + +func Test_panic(t *testing.T) { + fn := func() (err error) { + defer func() { + if panicErr := recover(); panicErr != nil { + err = errors.Errorf("%v", panicErr) // row2 + } + }() + panic("panic here") // row1 + } + + log.Infof("err: %+v", fn()) // row3, log include all row1-3 +} diff --git a/driver/controller/fetcher/BUILD.bazel b/driver/controller/fetcher/BUILD.bazel new file mode 100644 index 0000000..93295e6 --- /dev/null +++ b/driver/controller/fetcher/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "fetcher", + srcs = [ + "fetcher.go", + "merge.go", + "retry.go", + "stat.go", + "transfer.go", + "utils.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/fetcher", + visibility = ["//visibility:public"], + deps = [ + "//common/log", + "//common/timehist", + "//common/timewin", + "//common/utils", + "//driver/controller", + "@com_github_pkg_errors//:errors", + "@org_golang_x_exp//constraints", + ], +) + +go_test( + name = "fetcher_test", + srcs = [ + "fetcher_test.go", + "transfer_test.go", + ], + embed = [":fetcher"], + deps = [ + "//common/log", + "//driver/controller", + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + "@org_uber_go_zap//:zap", + ], +) diff --git a/driver/controller/fetcher/fetcher.go b/driver/controller/fetcher/fetcher.go new file mode 100644 index 0000000..0f4ba7b --- /dev/null +++ b/driver/controller/fetcher/fetcher.go @@ -0,0 +1,385 @@ +package fetcher + +import ( + "context" + "fmt" + "math" + "sync" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +type Requirement interface { + Snapshot() any +} + +type fetcher[T controller.FetchTarget] struct { + name string + queryFunc func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]T, error) + + requirement Requirement + minQuerySize uint64 + maxQuerySize uint64 + targetKeepDataSize int + targetQueryDataSize int + maxQueryTime time.Duration + maxRetry int + queryRetryInterval time.Duration + querySizeMultiplier float64 + + // protect the data below + mu sync.Mutex + + full controller.BlockRange // full is the whole data range + fetchingStart uint64 // [fetchingStart,fetchingEnd] is the range where data is fetching + fetchingEnd uint64 // so [full.StartBlock, fetchingStart) is the range where data is ready + fetchingDone chan struct{} // if the fetching task has a result, this chan will be closed + fetchingFailed error // has error, the fetching process will be aborted + + brokenErr error + + latest controller.BlockHeader + + data map[uint64]T // the ready data in [full.StartBlock, fetchingStart) will be here，key is blockNumber + totalSize int // is the total size in data，every time data is changed, it will be changed automatically + + // when [full.StartBlock,latest] changed, this chan will be closed to trigger growth, + // and after growth is completed, this chan will be re-build + winChanged chan struct{} + + stat *timewin.TimeWindowsManager[*processStat] +} + +func NewFetcher[T controller.FetchTarget]( + name string, + requirement Requirement, + full controller.BlockRange, + latest controller.BlockHeader, + minQuerySize uint64, + maxQuerySize uint64, + targetKeepDataSize int, + targetQueryDataSize int, + maxQueryTime time.Duration, + maxRetry int, + queryRetryInterval time.Duration, + querySizeMultiplier float64, + queryFunc func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]T, error), +) controller.Fetcher[T] { + if querySizeMultiplier < 1 { + panic("querySizeMultiplier cannot less than 1") + } + return &fetcher[T]{ + name: name, + requirement: requirement, + queryFunc: queryFunc, + minQuerySize: minQuerySize, + maxQuerySize: maxQuerySize, + targetKeepDataSize: targetKeepDataSize, + targetQueryDataSize: targetQueryDataSize, + maxQueryTime: maxQueryTime, + maxRetry: maxRetry, + queryRetryInterval: queryRetryInterval, + querySizeMultiplier: querySizeMultiplier, + full: full, + fetchingStart: full.StartBlock, + fetchingEnd: _min(min(full.StartBlock+minQuerySize-1, latest.GetBlockNumber()), full.EndBlock), + fetchingDone: make(chan struct{}), + latest: latest, + data: make(map[uint64]T), + winChanged: make(chan struct{}), + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +func (f *fetcher[T]) GetName() string { + return f.name +} + +func (f *fetcher[T]) Snapshot() any { + f.mu.Lock() + defer f.mu.Unlock() + return map[string]any{ + "name": f.name, + "type": fmt.Sprintf("%T", f), + "config": map[string]any{ + "requirement": f.requirement.Snapshot(), + "minQuerySize": f.minQuerySize, + "maxQuerySize": f.maxQuerySize, + "targetKeepDataSize": f.targetKeepDataSize, + "targetQueryDataSize": f.targetQueryDataSize, + "maxQueryTime": f.maxQueryTime.String(), + "maxRetry": f.maxRetry, + "queryRetryInterval": f.queryRetryInterval.String(), + "querySizeMultiplier": f.querySizeMultiplier, + }, + "fullRange": f.full.String(), + "readyRange": controller.BlockRange{ + StartBlock: f.full.StartBlock, + EndBlock: utils.WrapPointer(f.fetchingStart - 1), + }.String(), + "fetchingRange": controller.BlockRange{ + StartBlock: f.fetchingStart, + EndBlock: utils.WrapPointer(f.fetchingEnd), + }.String(), + "fetchingFailed": f.fetchingFailed, + "brokenErr": fmt.Sprintf("%+v", f.brokenErr), + "latest": controller.GetBlockFullText(f.latest), + "dataSize": f.totalSize, + "dataBlockCount": len(f.data), + "statistics": f.stat.Snapshot(), + } +} + +func (f *fetcher[T]) GetFullRange() controller.BlockRange { + f.mu.Lock() + defer f.mu.Unlock() + return f.full +} + +func (f *fetcher[T]) nextFetchSize(current uint64, got int, used time.Duration) uint64 { + if f.targetQueryDataSize > 0 && int(float64(got)*f.querySizeMultiplier) >= f.targetQueryDataSize { + return current // data got is big enough + } + if f.maxQueryTime > 0 && time.Duration(float64(used)*f.querySizeMultiplier) > f.maxQueryTime { + return current // time used is big enough + } + // increase the fetch size, use ceil to make sure newFetchSize not always equal to oldFetchSize + next := uint64(math.Ceil(float64(current) * f.querySizeMultiplier)) + return min(max(next, f.minQuerySize), f.maxQuerySize) +} + +func (f *fetcher[T]) growth(ctx context.Context) (pause bool, reject bool, changed chan struct{}) { + growthStartAt := time.Now() + _, logger := log.FromContext(ctx) + f.mu.Lock() + defer f.mu.Unlock() + if f.fetchingFailed != nil { + // already failed + return true, true, nil + } + if f.full.EndBlock != nil && *f.full.EndBlock < f.fetchingStart { + // no more data need to fetch + return true, true, nil + } + if f.totalSize >= f.targetKeepDataSize { + // total size of f.data is more than the limit, continue pause to wait consume + f.winChanged = make(chan struct{}) + return true, false, f.winChanged + } + start, end, latest := f.fetchingStart, f.fetchingEnd, f.latest + if end < start { + // last growth reached the right side + if start <= latest.GetBlockNumber() { + // latest may be increased, so now we can extend fetching range + end = _min(min(start+f.minQuerySize-1, latest.GetBlockNumber()), f.full.EndBlock) + } else { + // latest not increased, just rebuild f.winChanged + f.winChanged = make(chan struct{}) + return true, false, f.winChanged + } + } + done := f.fetchingDone + f.mu.Unlock() + + var result map[uint64]T + var err error + for retry := f.maxRetry; retry >= 0; { + if f.queryRetryInterval > 0 && err != nil { + select { + case <-ctx.Done(): + f.mu.Lock() + return true, true, nil + case <-time.After(f.queryRetryInterval): + } + } + startAt := time.Now() + if f.maxQueryTime > 0 { + queryCtx, cancel := context.WithTimeout(ctx, f.maxQueryTime) + result, err = f.queryFunc(queryCtx, start, end, latest) + cancel() + } else { + result, err = f.queryFunc(ctx, start, end, latest) + } + used := time.Since(startAt) + var size int + if err == nil { + size = sumSize(result) + } + st := &processStat{startAt: startAt} + st.fetchComplete(used, err == nil, end-start+1, size) + f.stat.Append(st) + tryLogger := logger.With( + "start", start, + "end", end, + "latest", controller.GetBlockSummary(latest), + "used", used.String(), + "retry", retry) + if err != nil { + var pe *PermanentError + if errors.As(err, &pe) { + err = pe.Err + retry = -1 + } + if errors.Is(err, context.Canceled) { + tryLogger.Debug("fetch canceled") + f.mu.Lock() + return true, true, nil + } + tryLogger.Warnfe(err, "fetch failed") + if fetchSize := end - start + 1; fetchSize > f.minQuerySize { + fetchSize = max(f.minQuerySize, fetchSize/2) + end = start + fetchSize - 1 + } else { + retry-- + } + } else { + // got the data in [start,end] + tryLogger.Debugw("fetch succeed", "size", size) + f.mu.Lock() + f.totalSize += size + f.data = utils.MergeMap(f.data, result) + f.fetchingStart = end + 1 + newFetchSize := f.nextFetchSize(end-start+1, size, used) + f.fetchingEnd = _min(min(f.fetchingStart+newFetchSize-1, f.latest.GetBlockNumber()), f.full.EndBlock) + f.fetchingDone = make(chan struct{}) + close(done) + f.winChanged = make(chan struct{}) + logger.Debugw("growth succeed", "used", time.Since(growthStartAt).String(), "current", f.current()) + return f.totalSize >= f.targetKeepDataSize, false, f.winChanged + } + } + // fetch the data in [start,end], keep reducing the range size, and keep retrying, + // but it still fails and can no longer continue + f.mu.Lock() + f.fetchingEnd = end + f.fetchingFailed = err + close(done) + logger.With("current", f.current()).Errore(err, "growth failed") + return true, true, nil +} + +func (f *fetcher[T]) current() string { + var ready string + if f.fetchingStart <= f.full.StartBlock { + ready = "[empty]" + } else { + ready = fmt.Sprintf("[%d,%d]", f.full.StartBlock, f.fetchingStart-1) + } + return fmt.Sprintf("FULL%s,READY%s,LATEST:%d,DATA:%d/%d", + f.full.String(), ready, f.latest.GetBlockNumber(), len(f.data), f.totalSize) +} + +func (f *fetcher[T]) KeepFetch(ctx context.Context) { + _, logger := log.FromContext(ctx, "fetcher", f.name, "requirement", f.requirement) + logger.Info("keep fetch start") + defer logger.Info("keep fetch end") + for round := 0; ; round++ { + roundCtx, _ := log.FromContext(ctx, "fetcher", f.name, "requirement", f.requirement, "fetchRound", round) + if pause, reject, changed := f.growth(roundCtx); !pause && !reject { + continue + } else if reject { + return + } else { + select { + case <-changed: + case <-ctx.Done(): + return + } + } + } +} + +func (f *fetcher[T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + defer func() { + st := &processStat{startAt: time.Now()} + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + }() + f.mu.Lock() + latest = f.latest + if !f.full.Contains(blockNumber) { + f.mu.Unlock() + return + } + for { + latest = f.latest + if f.brokenErr != nil { + // broken, just return the broken error + err = f.brokenErr + f.mu.Unlock() + return + } else if f.fetchingStart <= blockNumber { + // the required data is not yet ready + if f.fetchingFailed != nil { + // the fetch has failed and cannot continue, an error is returned directly + err = f.fetchingFailed + f.mu.Unlock() + return + } + done := f.fetchingDone + f.mu.Unlock() + + // waiting fetch completed + select { + case <-done: + case <-ctx.Done(): + err = ctx.Err() + return + } + f.mu.Lock() + } else { + // the needed data is ready，we can return data now + data, has = f.data[blockNumber] + f.mu.Unlock() + return + } + } +} + +func (f *fetcher[T]) UpdateLatest(latest controller.BlockHeader) { + f.mu.Lock() + defer f.mu.Unlock() + if latest.GetBlockNumber() < f.latest.GetBlockNumber() { + panic(errors.Errorf("try update latest from %d back to %d", f.latest.GetBlockNumber(), latest.GetBlockNumber())) + } + f.latest = latest + utils.TryCloseChan(f.winChanged) +} + +func (f *fetcher[T]) Broken(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.brokenErr = err + utils.TryCloseChan(f.winChanged) +} + +func (f *fetcher[T]) MoveStart(start uint64) { + f.mu.Lock() + defer f.mu.Unlock() + if start < f.full.StartBlock { + return + } + if start > f.fetchingStart { + start = f.fetchingStart + } + for n := f.full.StartBlock; n < start; n++ { + if item, has := f.data[n]; has { + f.totalSize -= item.Size() + delete(f.data, n) + } + } + f.full.StartBlock = start + utils.TryCloseChan(f.winChanged) +} diff --git a/driver/controller/fetcher/fetcher_test.go b/driver/controller/fetcher/fetcher_test.go new file mode 100644 index 0000000..1aa30ba --- /dev/null +++ b/driver/controller/fetcher/fetcher_test.go @@ -0,0 +1,261 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +type testBlockHeader struct { + BlockNumber uint64 + BlockTime time.Time +} + +func (b testBlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b testBlockHeader) GetBlockParentHash() string { + return "" +} + +func (b testBlockHeader) GetBlockHash() string { + return "" +} + +func (b testBlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} + +func newTestBlockHeader(blockNumber uint64) testBlockHeader { + zeroTime, _ := time.Parse(time.DateTime, "2025-07-01 00:00:00") + return testBlockHeader{ + BlockNumber: blockNumber, + BlockTime: zeroTime.Add(time.Second * time.Duration(blockNumber)), + } +} + +type testData []string + +func (t testData) Size() int { return len(t) } + +func buildTestData(bn uint64) (r testData) { + for j := uint64(0); j < bn%3; j++ { + r = append(r, fmt.Sprintf("%d-%d", bn, j)) + } + return +} + +func Test_Fetcher(t *testing.T) { + log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + fr := NewFetcher[testData]( + "testFetcher", + nil, + controller.BlockRange{StartBlock: 10}, + newTestBlockHeader(38), + 3, + 10, + 20, + 20, + time.Second, + 3, + time.Second, + 1.2, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]testData, error) { + r := make(map[uint64]testData) + for i := start; i <= end; i++ { + br := buildTestData(i) + if len(br) > 0 { + r[i] = br + } + } + return r, nil + }, + ) + f := fr.(*fetcher[testData]) + + var g sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + g.Wait() + }() + + // ============================================================================================================== + // ### staging 0, init state + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S L + // -------------------------------------------------------------------------------------------------------------- + // FS FE + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + // ============================================================================================================== + // ### staging 1, after growth 6 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S L + // -------------------------------------------------------------------------------------------------------------- + // R0 FS FE + // R1 FS FE + // R2 FS FE + // R3 FS FE + // R4 FS FE + // R5 *FS *FE + // -------------------------------------------------------------------------------------------------------------- + // count = 27 + // ============================================================================================================== + // ### staging 2, after pop 7 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *S L + // -------------------------------------------------------------------------------------------------------------- + // FS FE + // -------------------------------------------------------------------------------------------------------------- + // count = 20 + // ============================================================================================================== + // ### staging 3, after pop 4 time and growth 1 time + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *S L + // -------------------------------------------------------------------------------------------------------------- + // R0 FS FE + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 18 + // ============================================================================================================== + // ### staging 4, after update latest + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S *L + // -------------------------------------------------------------------------------------------------------------- + // R0 FE FS + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 19 + // ============================================================================================================== + // ### staging 5, after pop 20 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // L *S + // -------------------------------------------------------------------------------------------------------------- + // FE FS + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + // ============================================================================================================== + // ### staging 6, after update latest and growth 1 time and pop 1 time + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *L *S + // -------------------------------------------------------------------------------------------------------------- + // R0 FE FS + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + + // ### staging 0 + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(10), f.fetchingStart) + assert.Equal(t, uint64(12), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + + g.Add(1) + go func() { + defer g.Done() + f.KeepFetch(ctx) + }() + + // ### staging 1 + time.Sleep(time.Second) + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(36), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 27, f.totalSize) + + // ### staging 2 + for n := uint64(10); n <= 16; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(17) + time.Sleep(time.Second) + assert.Equal(t, uint64(17), f.full.StartBlock) + assert.Equal(t, uint64(36), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 20, f.totalSize) + + // ### staging 3 + for n := uint64(17); n <= 20; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(21) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(39), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 18, f.totalSize) + + // ### staging 4 + f.UpdateLatest(newTestBlockHeader(40)) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(41), f.fetchingStart) + assert.Equal(t, uint64(40), f.fetchingEnd) + assert.Equal(t, 19, f.totalSize) + + // ### staging 5 + for n := uint64(21); n <= 40; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(42) // will be automatically reduced to 41 because fetchingStart is still at 41 + time.Sleep(time.Second) + assert.Equal(t, uint64(41), f.full.StartBlock) + assert.Equal(t, uint64(41), f.fetchingStart) + assert.Equal(t, uint64(40), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + + // ### staging 6 + go func() { + time.Sleep(time.Second) + f.UpdateLatest(newTestBlockHeader(41)) + }() + r, _, _, err := f.Get(ctx, 41) + assert.Equal(t, buildTestData(41), r) + assert.Nil(t, err) + f.MoveStart(42) + time.Sleep(time.Second) + assert.Equal(t, uint64(42), f.full.StartBlock) + assert.Equal(t, uint64(42), f.fetchingStart) + assert.Equal(t, uint64(41), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + +} diff --git a/driver/controller/fetcher/merge.go b/driver/controller/fetcher/merge.go new file mode 100644 index 0000000..65fc5db --- /dev/null +++ b/driver/controller/fetcher/merge.go @@ -0,0 +1,125 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "time" + + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" +) + +func MergeIsomorphicFetchers[F controller.FetchTarget, T controller.FetchTarget]( + name string, + config any, + fetchers []controller.Fetcher[F], + mergeFunc func(blockNumber uint64, from []F) (data T, has bool, err error), +) controller.Fetcher[T] { + fetchers = utils.FilterArr(fetchers, func(f controller.Fetcher[F]) bool { + return !f.GetFullRange().IsEmpty() + }) + return &mergedIsomorphicFetchers[F, T]{ + name: name, + config: config, + fetchers: fetchers, + mergeFunc: mergeFunc, + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +type mergedIsomorphicFetchers[F controller.FetchTarget, T controller.FetchTarget] struct { + name string + config any + fetchers []controller.Fetcher[F] + mergeFunc func(blockNumber uint64, from []F) (data T, has bool, err error) + stat *timewin.TimeWindowsManager[*processStat] +} + +func (f *mergedIsomorphicFetchers[F, T]) GetName() string { + return f.name +} + +func (f *mergedIsomorphicFetchers[F, T]) GetFullRange() controller.BlockRange { + full := controller.EmptyBlockRange + for _, up := range f.fetchers { + full = full.Cover(up.GetFullRange()) + } + return full +} + +func (f *mergedIsomorphicFetchers[F, T]) Snapshot() any { + upstreams := make([]any, len(f.fetchers)) + for i, up := range f.fetchers { + upstreams[i] = up.Snapshot() + } + return map[string]any{ + "name": f.name, + "config": f.config, + "type": fmt.Sprintf("%T", f), + "upstreams": upstreams, + "statistics": f.stat.Snapshot(), + } +} + +func (f *mergedIsomorphicFetchers[F, T]) KeepFetch(ctx context.Context) { + var g sync.WaitGroup + g.Add(len(f.fetchers)) + for _, up := range f.fetchers { + go func(up controller.Fetcher[F]) { + defer g.Done() + up.KeepFetch(ctx) + }(up) + } + g.Wait() +} + +func (f *mergedIsomorphicFetchers[F, T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + var from []F + for _, up := range f.fetchers { + var d F + d, has, latest, err = up.Get(ctx, blockNumber) + if err != nil { + return + } + if has { + from = append(from, d) + } + } + mergeStartAt := time.Now() + data, has, err = f.mergeFunc(blockNumber, from) + var dataSize int + if err == nil && has { + dataSize = data.Size() + } + st := &processStat{startAt: time.Now()} + st.fetchComplete(time.Since(mergeStartAt), err == nil, 1, dataSize) + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + return +} + +func (f *mergedIsomorphicFetchers[F, T]) Broken(err error) { + for _, up := range f.fetchers { + up.Broken(err) + } +} + +func (f *mergedIsomorphicFetchers[F, T]) UpdateLatest(latest controller.BlockHeader) { + for _, up := range f.fetchers { + up.UpdateLatest(latest) + } +} + +func (f *mergedIsomorphicFetchers[F, T]) MoveStart(start uint64) { + for _, up := range f.fetchers { + up.MoveStart(start) + } +} diff --git a/driver/controller/fetcher/retry.go b/driver/controller/fetcher/retry.go new file mode 100644 index 0000000..677806b --- /dev/null +++ b/driver/controller/fetcher/retry.go @@ -0,0 +1,15 @@ +package fetcher + +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +func Permanent(err error) error { + return &PermanentError{ + Err: err, + } +} diff --git a/driver/controller/fetcher/stat.go b/driver/controller/fetcher/stat.go new file mode 100644 index 0000000..32fcf4e --- /dev/null +++ b/driver/controller/fetcher/stat.go @@ -0,0 +1,87 @@ +package fetcher + +import ( + "time" + + "sentioxyz/sentio-core/common/timehist" +) + +type processStat struct { + startAt time.Time + + fetchUsed timehist.Histogram + fetchTotalUsed time.Duration + fetchFailedCount int + fetchQueryBlockRange uint64 + fetchGotDataSize int + + getUsed timehist.Histogram + getTotalUsed time.Duration + getEmptyCount int +} + +func (s *processStat) GetStartAt() time.Time { + return s.startAt +} + +func (s *processStat) Merge(a *processStat) { + s.fetchUsed = s.fetchUsed.Add(a.fetchUsed) + s.fetchTotalUsed += a.fetchTotalUsed + s.fetchFailedCount += a.fetchFailedCount + s.fetchQueryBlockRange += a.fetchQueryBlockRange + s.fetchGotDataSize += a.fetchGotDataSize + + s.getEmptyCount += a.getEmptyCount + s.getUsed = s.getUsed.Add(a.getUsed) + s.getTotalUsed += a.getTotalUsed +} + +func (s *processStat) Snapshot(endAt time.Time) any { + sn := map[string]any{ + "startAt": s.startAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(s.startAt).String(), + } + if fetchCount := s.fetchUsed.Sum(); fetchCount > 0 { + sn["fetch"] = map[string]any{ + "count": fetchCount, + "failedCount": s.fetchFailedCount, + "totalUsed": s.fetchTotalUsed.String(), + "used": s.fetchUsed.String(), + "avgUsed": (s.fetchTotalUsed / time.Duration(fetchCount)).String(), + "totalGotDataSize": s.fetchGotDataSize, + "avgGotDataSize": s.fetchGotDataSize / fetchCount, + "totalQueryBlockRange": s.fetchQueryBlockRange, + "avgQueryBlockRange": s.fetchQueryBlockRange / uint64(fetchCount), + "pressure": float64(s.fetchTotalUsed) / float64(endAt.Sub(s.startAt)), + } + } + if getCount := s.getUsed.Sum(); getCount > 0 { + sn["get"] = map[string]any{ + "count": getCount, + "emptyCount": s.getEmptyCount, + "totalUsed": s.getTotalUsed, + "used": s.getUsed.String(), + "avgUsed": (s.getTotalUsed / time.Duration(getCount)).String(), + } + } + return sn +} + +func (s *processStat) fetchComplete(used time.Duration, succeed bool, queryRange uint64, dataSize int) { + s.fetchUsed = timehist.Histogram{}.Incr(used) + s.fetchTotalUsed = used + if !succeed { + s.fetchFailedCount = 1 + } + s.fetchQueryBlockRange = queryRange + s.fetchGotDataSize = dataSize +} + +func (s *processStat) getComplete(used time.Duration, has bool) { + s.getUsed = timehist.Histogram{}.Incr(used) + s.getTotalUsed = used + if !has { + s.getEmptyCount = 1 + } +} diff --git a/driver/controller/fetcher/transfer.go b/driver/controller/fetcher/transfer.go new file mode 100644 index 0000000..5636883 --- /dev/null +++ b/driver/controller/fetcher/transfer.go @@ -0,0 +1,418 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +func TransferFetcher[F controller.FetchTarget, T controller.FetchTarget]( + name string, + upstream controller.Fetcher[F], + latest controller.BlockHeader, + maxConcurrency uint64, + maxKeepDataSize int, + maxKeepDataNum int, + transferTimeout time.Duration, + maxRetry int, + retryInterval time.Duration, + transferFunc func(ctx context.Context, blockNumber uint64, from F) (T, bool, error), +) controller.Fetcher[T] { + full := upstream.GetFullRange() + return &transferFetcher[F, T]{ + name: name, + transferFunc: transferFunc, + maxConcurrency: maxConcurrency, + maxKeepDataSize: maxKeepDataSize, + maxKeepDataNum: maxKeepDataNum, + transferTimeout: transferTimeout, + maxRetry: maxRetry, + retryInterval: retryInterval, + upstream: upstream, + + full: full, + readyEnd: full.StartBlock, + fetchStart: full.StartBlock, + fetched: make(map[uint64]struct{}), + readyEndMoved: make(chan struct{}), + latest: latest, + data: make(map[uint64]T), + tryGrowth: make(chan struct{}), + broken: make(chan struct{}), + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +type transferFetcher[F controller.FetchTarget, T controller.FetchTarget] struct { + name string + transferFunc func(ctx context.Context, blockNumber uint64, from F) (T, bool, error) + + maxConcurrency uint64 + maxKeepDataSize int + maxKeepDataNum int + transferTimeout time.Duration + maxRetry int + retryInterval time.Duration + + upstream controller.Fetcher[F] + + // protect the data below + mu sync.Mutex + g sync.WaitGroup + + full controller.BlockRange // full is the whole data range + readyEnd uint64 // [full.StartBlock,readyEnd) is the range where data is ready + fetchStart uint64 // [readyEnd,fetchStart) is the range where data is fetching + fetched map[uint64]struct{} + readyEndMoved chan struct{} + fetchFailed error + fetchFailedAt uint64 + + latest controller.BlockHeader + + data map[uint64]T // the ready data in [full.StartBlock, fetchingStart) will be here，key is blockNumber + totalSize int // is the total size in data，every time data is changed, it will be changed automatically + + // when [full.StartBlock,readyEnd) changed or latest changed, this chan will be closed to trigger growth, + // and after growth is completed, this chan will be re-build + tryGrowth chan struct{} + + // will be closed when Broken called + broken chan struct{} + brokenErr error + + stat *timewin.TimeWindowsManager[*processStat] +} + +func (f *transferFetcher[F, T]) GetName() string { + return f.name +} + +func (f *transferFetcher[F, T]) GetFullRange() controller.BlockRange { + f.mu.Lock() + defer f.mu.Unlock() + return f.full +} + +func (f *transferFetcher[F, T]) Snapshot() any { + f.mu.Lock() + defer f.mu.Unlock() + + return map[string]any{ + "name": f.name, + "type": fmt.Sprintf("%T", f), + "config": map[string]any{ + "maxConcurrency": f.maxConcurrency, + "targetKeepDataSize": f.maxKeepDataSize, + "targetKeepDataNum": f.maxKeepDataNum, + "transferTimeout": f.transferTimeout.String(), + "maxRetry": f.maxRetry, + "retryInterval": f.retryInterval.String(), + }, + "fullRange": f.full.String(), + "readyRange": controller.BlockRange{ + StartBlock: f.full.StartBlock, + EndBlock: utils.WrapPointer(f.readyEnd - 1), + }.String(), + "fetchingRange": controller.BlockRange{ + StartBlock: f.readyEnd, + EndBlock: utils.WrapPointer(f.fetchStart - 1), + }.String(), + "fetchFailed": f.fetchFailed, + "fetchFailedAt": f.fetchFailedAt, + "dataSize": f.totalSize, + // dataBlockCount only counts ready blocks that carried data; empty blocks advance readyEnd + // without entering f.data, so fetchedAhead (the ready-but-unconsumed prefetch depth) can be larger. + "dataBlockCount": len(f.data), + // fetchedAhead is the ready prefetch depth: blocks already prepared and waiting to be consumed. + // (MoveStart clamps full.StartBlock to readyEnd, so this never underflows.) + "fetchedAhead": f.readyEnd - f.full.StartBlock, + // fetchedUnreadyCount is the number of out-of-order-completed blocks held back because an + // earlier block in [readyEnd, fetchStart) is not done yet. Always 0 when maxConcurrency==1. + "fetchedUnreadyCount": len(f.fetched), + "upstream": f.upstream.Snapshot(), + "statistics": f.stat.Snapshot(), + } +} + +// current is used as log argument, if not print debug log, current.String will not be called +type current struct { + full controller.BlockRange + readyEnd uint64 + fetchStart uint64 + dataBlockNum int + dataTotalSize int +} + +func (c current) String() string { + return fmt.Sprintf("FULL%s,READY[%d,%d),FETCHING[%d,%d),DATA:%d/%d", + c.full.String(), c.full.StartBlock, c.readyEnd, c.readyEnd, c.fetchStart, c.dataBlockNum, c.dataTotalSize) +} + +func (f *transferFetcher[F, T]) current() current { + return current{ + full: f.full, + readyEnd: f.readyEnd, + fetchStart: f.fetchStart, + dataBlockNum: len(f.data), + dataTotalSize: f.totalSize, + } +} + +func (f *transferFetcher[F, T]) setFetchFailed(blockNumber uint64, err error) { + if f.fetchFailed == nil || f.fetchFailedAt > blockNumber { + f.fetchFailed = err + f.fetchFailedAt = blockNumber + } +} + +func (f *transferFetcher[F, T]) transfer(ctx context.Context, blockNumber uint64) { + _, logger := log.FromContext(ctx) + from, has, _, err := f.upstream.Get(ctx, blockNumber) + var result T + var stage string + if err != nil { + stage = "fetch from upstream" + } else if has { + for retry := f.maxRetry; retry >= 0; { + if f.retryInterval > 0 && err != nil { + select { + case <-ctx.Done(): + return + case <-time.After(f.retryInterval): + } + } + startAt := time.Now() + if f.transferTimeout > 0 { + transferCtx, cancel := context.WithTimeout(ctx, f.transferTimeout) + result, has, err = f.transferFunc(transferCtx, blockNumber, from) + cancel() + } else { + result, has, err = f.transferFunc(ctx, blockNumber, from) + } + used := time.Since(startAt) + var dataSize int + if err == nil && has { + dataSize = result.Size() + } + st := &processStat{startAt: time.Now()} + st.fetchComplete(used, err == nil, 1, dataSize) + f.stat.Append(st) + if err == nil { + break + } + var pe *PermanentError + if errors.As(err, &pe) { + err = pe.Err + retry = -1 + } + tryLogger := logger.With("blockNumber", blockNumber, "used", used.String(), "retry", retry) + if errors.Is(err, context.Canceled) { + tryLogger.Debug("fetch canceled") + return + } + tryLogger.Warnfe(err, "fetch failed") + retry-- + } + if err != nil { + stage = "transfer" + } + } + + f.mu.Lock() + f.fetched[blockNumber] = struct{}{} + if err != nil { + f.setFetchFailed(blockNumber, err) + if errors.Is(err, context.Canceled) { + logger.With("blockNumber", blockNumber).Warnfe(err, "fetch failed because %s canceled", stage) + } else { + logger.With("blockNumber", blockNumber).Errorfe(err, "fetch failed because %s failed", stage) + } + } else if has { + f.data[blockNumber] = result + f.totalSize += result.Size() + logger.Debugw("fetch succeed", "blockNumber", blockNumber) + } else { + logger.Debugw("fetch succeed and no data", "blockNumber", blockNumber) + } + // move f.readyEnd + var readyEnd uint64 + for readyEnd = f.readyEnd; readyEnd < f.fetchStart; readyEnd++ { + if _, ready := f.fetched[readyEnd]; ready { + delete(f.fetched, readyEnd) + } else { + break + } + } + if f.readyEnd < readyEnd { + f.readyEnd = readyEnd + logger.Debugw("growth succeed", "current", f.current()) + utils.TryCloseChan(f.tryGrowth) + utils.TryCloseChan(f.readyEndMoved) + f.readyEndMoved = make(chan struct{}) + } + f.mu.Unlock() +} + +func (f *transferFetcher[F, T]) growth(ctx context.Context) (reject bool, changed chan struct{}) { + f.mu.Lock() + defer f.mu.Unlock() + if f.fetchFailed != nil { + // already failed + return true, nil + } + if f.full.EndBlock != nil && *f.full.EndBlock < f.fetchStart { + // no more data need to fetch + return true, nil + } + if f.totalSize < f.maxKeepDataSize && len(f.data) < f.maxKeepDataNum { + // fetch more + maxFetchBlock := min(f.readyEnd+f.maxConcurrency-1, f.latest.GetBlockNumber()) + if f.full.EndBlock != nil { + maxFetchBlock = min(maxFetchBlock, *f.full.EndBlock) + } + for ; f.fetchStart <= maxFetchBlock; f.fetchStart++ { + f.g.Add(1) + go func(bn uint64) { + f.transfer(ctx, bn) + f.g.Done() + }(f.fetchStart) + } + } + f.tryGrowth = make(chan struct{}) + return false, f.tryGrowth +} + +func (f *transferFetcher[F, T]) keepFetch(ctx context.Context) { + _, logger := log.FromContext(ctx, "fetcher", f.name) + logger.Info("keep fetch start") + defer logger.Info("keep fetch end") + defer func() { + f.g.Wait() + }() + for round := 0; ; round++ { + roundCtx, _ := log.FromContext(ctx, "fetcher", f.name, "round", round) + if reject, changed := f.growth(roundCtx); reject { + return + } else { + select { + case <-changed: + case <-ctx.Done(): + return + } + } + } +} + +func (f *transferFetcher[F, T]) KeepFetch(ctx context.Context) { + var g sync.WaitGroup + g.Add(2) + go func() { + defer g.Done() + f.upstream.KeepFetch(ctx) + }() + go func() { + defer g.Done() + f.keepFetch(ctx) + }() + g.Wait() +} + +func (f *transferFetcher[F, T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + defer func() { + st := &processStat{startAt: time.Now()} + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + }() + f.mu.Lock() + for { + latest = f.latest + if !f.full.Contains(blockNumber) { + // not in full, no data + f.mu.Unlock() + return + } + if f.brokenErr != nil { + err = f.brokenErr + f.mu.Unlock() + return + } + if f.readyEnd <= blockNumber { + // data not ready, wait readyEnd moved + readyEndMoved, broken := f.readyEndMoved, f.broken + f.mu.Unlock() + select { + case <-readyEndMoved: + case <-broken: + case <-ctx.Done(): + err = ctx.Err() + return + } + f.mu.Lock() + continue + } + // data is ready + if f.fetchFailed != nil && f.fetchFailedAt <= blockNumber { + err = f.fetchFailed + } else { + data, has = f.data[blockNumber] + } + f.mu.Unlock() + return + } +} + +func (f *transferFetcher[F, T]) UpdateLatest(latest controller.BlockHeader) { + f.upstream.UpdateLatest(latest) + + f.mu.Lock() + f.latest = latest + utils.TryCloseChan(f.tryGrowth) + f.mu.Unlock() +} + +func (f *transferFetcher[F, T]) Broken(err error) { + f.upstream.Broken(err) + + f.mu.Lock() + defer f.mu.Unlock() + if f.brokenErr == nil { + f.brokenErr = err + utils.TryCloseChan(f.broken) + } +} + +func (f *transferFetcher[F, T]) MoveStart(start uint64) { + f.upstream.MoveStart(start) + + f.mu.Lock() + defer f.mu.Unlock() + if start < f.full.StartBlock { + return + } + if start > f.readyEnd { + start = f.readyEnd + } + for n := f.full.StartBlock; n < start; n++ { + if item, has := f.data[n]; has { + f.totalSize -= item.Size() + delete(f.data, n) + } + } + f.full.StartBlock = start + utils.TryCloseChan(f.tryGrowth) +} diff --git a/driver/controller/fetcher/transfer_test.go b/driver/controller/fetcher/transfer_test.go new file mode 100644 index 0000000..43c1999 --- /dev/null +++ b/driver/controller/fetcher/transfer_test.go @@ -0,0 +1,183 @@ +package fetcher + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func Test_transfer(t *testing.T) { + log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + upstream := NewFetcher[testData]( + "testFetcher", + nil, + controller.BlockRange{StartBlock: 10}, + newTestBlockHeader(25), + 3, + 10, + 20, + 20, + time.Second, + 3, + time.Second, + 1.2, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]testData, error) { + r := make(map[uint64]testData) + for i := start; i <= end; i++ { + br := buildTestData(i) + if len(br) > 0 { + r[i] = br + } + } + return r, nil + }, + ) + fr := TransferFetcher[testData]( + "transferFetcher", + upstream, + newTestBlockHeader(25), + 2, + 5, + 5, + 0, + 10, + 0, + func(ctx context.Context, blockNumber uint64, from testData) (testData, bool, error) { + if len(from) == 0 { + return testData{}, false, nil + } + select { + case <-ctx.Done(): + return testData{}, false, ctx.Err() + case <-time.After(time.Millisecond * 100): + //if rand.Int()%2 == 0 { + // return testData{}, false, errors.Errorf("transfer failed sadly") + //} + return testData{strings.Join(from, ",")}, true, nil + } + }) + f := fr.(*transferFetcher[testData, testData]) + + var g sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + g.Wait() + }() + + g.Add(1) + go func() { + defer g.Done() + f.KeepFetch(ctx) + }() + + mergedTestData := func(n uint64) testData { + src := buildTestData(n) + if len(src) == 0 { + return nil + } + return testData{strings.Join(src, ",")} + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS RE LA + time.Sleep(time.Second) + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(18), f.readyEnd) + assert.Equal(t, uint64(18), f.fetchStart) + assert.Equal(t, 6, f.totalSize) + for n := uint64(10); n < 18; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS RE LA + f.MoveStart(12) + time.Sleep(time.Second) + assert.Equal(t, uint64(12), f.full.StartBlock) + assert.Equal(t, uint64(21), f.readyEnd) + assert.Equal(t, uint64(21), f.fetchStart) + assert.Equal(t, 6, f.totalSize) + for n := uint64(12); n < 21; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS LA RE + f.MoveStart(21) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(26), f.readyEnd) + assert.Equal(t, uint64(26), f.fetchStart) // latest is 25, so no block is transferring + assert.Equal(t, 3, f.totalSize) + for n := uint64(21); n < 25; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS LA RE + f.UpdateLatest(newTestBlockHeader(26)) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(27), f.readyEnd) + assert.Equal(t, uint64(27), f.fetchStart) // latest is 26, so no block is transferring + assert.Equal(t, 4, f.totalSize) + for n := uint64(21); n < 26; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(26), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // LA RE + // FA + f.MoveStart(27) + time.Sleep(time.Second) + assert.Equal(t, uint64(27), f.full.StartBlock) + assert.Equal(t, uint64(27), f.readyEnd) + assert.Equal(t, uint64(27), f.fetchStart) + assert.Equal(t, 0, f.totalSize) + brokenErr := errors.New("broken") + go func() { + time.Sleep(time.Second) + f.Broken(brokenErr) + }() + { + d, has, latest, err := f.Get(ctx, 27) + assert.ErrorIs(t, err, brokenErr) + assert.Equal(t, uint64(26), latest.GetBlockNumber()) + assert.False(t, has) + assert.Nil(t, d) + } + +} diff --git a/driver/controller/fetcher/utils.go b/driver/controller/fetcher/utils.go new file mode 100644 index 0000000..4d57f6d --- /dev/null +++ b/driver/controller/fetcher/utils.go @@ -0,0 +1,21 @@ +package fetcher + +import ( + "golang.org/x/exp/constraints" + + "sentioxyz/sentio-core/driver/controller" +) + +func sumSize[T controller.FetchTarget](dict map[uint64]T) (size int) { + for _, item := range dict { + size += item.Size() + } + return +} + +func _min[V constraints.Integer](a V, b *V) V { + if b == nil { + return a + } + return min(a, *b) +} diff --git a/driver/controller/handler.go b/driver/controller/handler.go new file mode 100644 index 0000000..a192a1e --- /dev/null +++ b/driver/controller/handler.go @@ -0,0 +1,102 @@ +package controller + +import ( + "context" + "fmt" +) + +type HandlerID struct { + DataSource string + DataSourceID int + Type string + Name string + ID int32 +} + +func (h HandlerID) String() string { + return fmt.Sprintf("%d#%s/%s/%s/%d", h.DataSourceID, h.DataSource, h.Type, h.Name, h.ID) +} + +type HandlerConfig interface { + GetHandlerName() string + GetHandlerId() int32 +} + +type SimpleHandlerConfig struct { + Name string + ID int32 +} + +func (h SimpleHandlerConfig) GetHandlerName() string { + return h.Name +} + +func (h SimpleHandlerConfig) GetHandlerId() int32 { + return h.ID +} + +func BuildHandlerID(dataSource string, dataSourceID int, handlerType string, handlerConfig HandlerConfig) HandlerID { + return HandlerID{ + DataSource: dataSource, + DataSourceID: dataSourceID, + Type: handlerType, + Name: handlerConfig.GetHandlerName(), + ID: handlerConfig.GetHandlerId(), + } +} + +type HandlerAgent interface { + GetHandlerID() HandlerID + GetRange() BlockRange + Snapshot() any +} + +func GetHandleAgentsBlockRange[HA HandlerAgent](agents []HA) BlockRange { + r := EmptyBlockRange + for _, ag := range agents { + r = r.Cover(ag.GetRange()) + } + return r +} + +type BaseHandlerAgent struct { + HandlerID HandlerID + Range BlockRange +} + +func (h BaseHandlerAgent) GetHandlerID() HandlerID { + return h.HandlerID +} + +func (h BaseHandlerAgent) GetRange() BlockRange { + return h.Range +} + +func NewBaseHandlerAgent( + dataSource string, + dataSourceID int, + handlerType string, + handlerConfig HandlerConfig, + blockRange BlockRange, +) BaseHandlerAgent { + return BaseHandlerAgent{ + HandlerID: BuildHandlerID(dataSource, dataSourceID, handlerType, handlerConfig), + Range: blockRange, + } +} + +type HandlerController interface { + Prologue( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + first uint64, + latest BlockHeader, + ) *ExternalError + GetBlockRange() BlockRange + GetAgentStat() map[string]int + BuildBlockDataFetcher(firstBlockNumber uint64, currentBlockNumber uint64, latest BlockHeader) Fetcher[BlockData] + Epilogue() + + Snapshot() any +} diff --git a/driver/controller/main.go b/driver/controller/main.go new file mode 100644 index 0000000..c32a00a --- /dev/null +++ b/driver/controller/main.go @@ -0,0 +1,354 @@ +package controller + +import ( + "context" + "sync/atomic" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type MainController struct { + seqMode bool + + blockBuilder BlockBuilder + checkpointCtrl CheckpointController + processor *models.Processor + chainID string + + bindingIndex atomic.Uint64 + + analyser +} + +func NewMainController( + blockBuilder BlockBuilder, + checkpointCtrl CheckpointController, + seqMode bool, + processor *models.Processor, + chainID string, +) *MainController { + N.DriverCreated(processor, chainID, func() (int64, bool) { + if cc := checkpointCtrl.GetSavedLatestCheckpoint(); cc != nil { + return int64(cc.BlockNumber), true + } + return 0, false + }) + return &MainController{ + seqMode: seqMode, + blockBuilder: blockBuilder, + checkpointCtrl: checkpointCtrl, + processor: processor, + chainID: chainID, + analyser: newAnalyser(), + } +} + +func (c *MainController) Main(ctx context.Context) error { + const maxDupErrRetryTimes = 10 + lastExtErrCode, lastExtErrRound := 0, 0 + for round := 0; ; round++ { + runCtx, logger := log.FromContext(ctx, "runID", round) + err := c.run(runCtx) + if err == nil { + logger.UserVisible().Info("chain is done") + return nil + } + switch { + case errors.Is(err, ErrInternalReorgDetected): + continue + case errors.Is(err, ErrInternalHasNewTemplate): + continue + } + var extErr *ExternalError + if errors.As(err, &extErr) { + logger.Errorf("run got external error: %+v", extErr) + saveErr := c.checkpointCtrl.SaveError(ctx, extErr) + if saveErr != nil { + logger.Errore(saveErr, "save chain error failed") + } + if extErr.IsDriverError() || extErr.IsUserRuntimeError() || saveErr != nil { + if saveErr == nil && extErr.Code() == lastExtErrCode { + if dupTimes := round - lastExtErrRound; dupTimes >= maxDupErrRetryTimes { + logger.Warnf("same error dupped %d times, will exit now", dupTimes) + return err + } else { + logger.Warnf("same error dupped %d times", round-lastExtErrRound) + } + } else { + lastExtErrCode, lastExtErrRound = extErr.Code(), round + } + logger.Infof("will retry after %s", RunWaiting.String()) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(RunWaiting): + continue + } + } + } else { + logger.Errorf("run got error: %+v", err) + } + return err + } +} + +func (c *MainController) Snapshot() any { + // Mirror the taskProcessConcurrency computed in run(): in sequential mode tasks run on a single + // worker, otherwise on ProcessConcurrency workers. Surfacing both (plus seqMode) lets a reader tell + // "handlers are saturated" apart from "handlers are starved" without reverse-engineering the config. + taskProcessConcurrency := ProcessConcurrency + if c.seqMode { + taskProcessConcurrency = 1 + } + return map[string]any{ + "seqMode": c.seqMode, + "taskProcessConcurrency": taskProcessConcurrency, + "bindingIndex": c.bindingIndex.Load(), + "blockBuilder": c.blockBuilder.Snapshot(), + "checkpointController": c.checkpointCtrl.Snapshot(), + "statistics": c.analyser.Snapshot(), + } +} + +type BlockDataSummary struct { + BlockNumber uint64 + BlockParentHash string + BlockHash string + BlockTime time.Time + + TaskCount int + CheckpointData map[string]string +} + +func (s BlockDataSummary) GetBlockNumber() uint64 { + return s.BlockNumber +} + +func (s BlockDataSummary) GetBlockParentHash() string { + return s.BlockParentHash +} + +func (s BlockDataSummary) GetBlockHash() string { + return s.BlockHash +} + +func (s BlockDataSummary) GetBlockTime() time.Time { + return s.BlockTime +} + +type BlockPanel struct { + BlockNumber uint64 + DataSummary *BlockDataSummary + TaskCount int + + ProgressBar +} + +type ProgressMessage struct { + BlockStart *BlockPanel + BlockTaskDone uint64 + BlockAllDone *uint64 +} + +func (c *MainController) run(ctx context.Context) error { + _, logger := log.FromContext(ctx) + logger.Info("run started") + defer func() { + logger.Info("run finished") + }() + + checkpoint, templates := c.checkpointCtrl.GetLatestCheckpoint(), c.checkpointCtrl.GetTemplates() + agentStat, extErr := c.blockBuilder.Start(ctx, checkpoint, templates) + if extErr != nil { + return extErr + } + defer c.blockBuilder.Finish() + if extErr = c.checkpointCtrl.Ready(ctx, agentStat); extErr != nil { + return extErr + } + taskProcessConcurrency := utils.Select(c.seqMode, 1, int(ProcessConcurrency)) + logger.Infow("main stream is ready", + "checkpoint", utils.NullOrToString(checkpoint), + "templates", utils.CountMap(templates), + "taskProcessConcurrency", taskProcessConcurrency) + + N.DriverStarted(ctx, c.processor, c.chainID, CountTemplatesByID(templates)) + + g, gctx := errgroup.WithContext(ctx) + // More capacity to ensure the goroutines that execute tasks and build blockData are less likely to be blocked by this + progressNotice := make(chan ProgressMessage, 10000) + // Once all the data for a block is ready, an element will be push to this chan. + // When a block's checkpoint is constructed, an element will be pop from this chan. + // So its capacity is the number of blocks that can be processed simultaneously. + waitingBlocks := make(chan struct{}, 1000) + concurrency.RunWithProducer( + g, + gctx, + taskProcessConcurrency, + func(ctx context.Context, taskChan chan<- Task) error { + logger.Info("keep build block data started") + defer func() { + logger.Info("keep build block data finished") + }() + for { + // fetch BlockData, may be waiting some fetcher, may be waiting latest block + fetchStartAt := time.Now() + blockNumber, blockData, progressBar, reorg, getErr := c.blockBuilder.Next(ctx) + c.analyser.fetchWait(time.Since(fetchStartAt)) + if getErr != nil { + if errors.Is(getErr, ErrInternalNeedUpgrade) { + return NewExternalError(ErrCodeNeedUpgrade, getErr) + } + // even the endpoint was override by user, the fetch data failed error should also be driver error + return NewExternalError(ErrCodeFetchDataFailed, getErr) + } + if reorg != nil { + if cleanErr := c.checkpointCtrl.CleanCheckpoint(ctx, blockNumber, *reorg); cleanErr != nil { + return cleanErr + } + N.ReorgDetected(ctx, c.processor, c.chainID) + return ErrInternalReorgDetected + } + if !progressBar.FullBlockRange.Contains(blockNumber) { + // no more data, build block data can finish now, + // blockData and progressBar.LatestBlock will always be nil here. + logger.Infow("no more block data", "blockNumber", blockNumber, "full", progressBar.FullBlockRange.String()) + select { + case progressNotice <- ProgressMessage{BlockAllDone: &blockNumber}: + case <-ctx.Done(): + } + return nil + } + startAt := time.Now() + // Using `dataSummary` instead of `blockData` is to release the reference to `blockData`, allowing its memory + // to be released promptly and preventing subsequent tasks from having their references to `blockData` + // continuously attached due to a task running for too long. + var dataSummary *BlockDataSummary + var taskList []Task + if blockData != nil { + taskList = blockData.GetTaskList() + dataSummary = &BlockDataSummary{ + BlockNumber: blockData.GetBlockNumber(), + BlockParentHash: blockData.GetBlockParentHash(), + BlockHash: blockData.GetBlockHash(), + BlockTime: blockData.GetBlockTime(), + TaskCount: len(taskList), + CheckpointData: blockData.CheckpointData(), + } + } + // If waitingBlocks is full, it means that the checkpoint goroutine is stuck. + // This could be because making checkpoint is too time-consuming, or because a task is taking too long. + select { + case waitingBlocks <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + select { + case progressNotice <- ProgressMessage{ + BlockStart: &BlockPanel{ + BlockNumber: blockNumber, + DataSummary: dataSummary, + ProgressBar: progressBar, + TaskCount: len(taskList), + }, + }: + case <-ctx.Done(): + return ctx.Err() + } + for i, task := range taskList { + index := TaskIndex{ + Global: c.bindingIndex.Add(1), + InBlock: i, + TotalInBlock: len(taskList), + } + task.Init(ctx, index, progressBar) + select { + case taskChan <- task: + case <-ctx.Done(): + return ctx.Err() + } + } + c.analyser.taskSent(time.Since(startAt)) + } + }, + func(ctx context.Context, task Task) error { + startAt := time.Now() + if taskErr := task.Exec(ctx, c.checkpointCtrl); taskErr != nil { + return taskErr + } + completeAt := time.Now() + select { + case progressNotice <- ProgressMessage{BlockTaskDone: task.GetBlockNumber()}: + case <-ctx.Done(): + } + c.analyser.taskComplete(task.GetHandlerID().String(), time.Since(startAt), time.Since(completeAt)) + return nil + }) + + makeCheckpointDone := make(chan struct{}) + g.Go(func() error { + logger.Info("keep make checkpoint started") + defer func() { + logger.Info("keep make checkpoint finished") + }() + waiting := make(map[uint64]*BlockPanel) // size will always less than cap(waitingBlocks) + for { + var bn uint64 + select { + case <-gctx.Done(): + return gctx.Err() + case pm := <-progressNotice: + if pm.BlockStart != nil { + bn = pm.BlockStart.BlockNumber + waiting[bn] = pm.BlockStart + } else if pm.BlockAllDone != nil { + bn = *pm.BlockAllDone + waiting[bn] = nil // bn is the finalize block, will out of full block range + } else { + bn = pm.BlockTaskDone + waiting[bn].TaskCount -= 1 + } + } + if bn > 0 && waiting[bn-1] != nil { + continue + } + startAt := time.Now() + for { + r, has := waiting[bn] + if has && r == nil { + // no more checkpoint need to be make now + close(makeCheckpointDone) + return nil + } + if !has || r.TaskCount > 0 { + break + } + if r.DataSummary != nil { + hasNewTpl, makeErr := c.checkpointCtrl.MakeCheckpoint(gctx, *r.DataSummary, r.ProgressBar) + if makeErr != nil { + return makeErr + } else if hasNewTpl { + return ErrInternalHasNewTemplate + } + } + delete(waiting, bn) + <-waitingBlocks + bn++ + } + c.analyser.makeCheckpoint(time.Since(startAt)) + } + }) + + g.Go(func() error { + return c.checkpointCtrl.KeepSave(gctx, makeCheckpointDone) + }) + + return g.Wait() +} diff --git a/driver/controller/main_test.go b/driver/controller/main_test.go new file mode 100644 index 0000000..50cdf01 --- /dev/null +++ b/driver/controller/main_test.go @@ -0,0 +1,659 @@ +package controller + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +type testBlockHeader struct { + BlockNumber uint64 + BlockHash string + BlockParentHash string + BlockTime time.Time +} + +func (b testBlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b testBlockHeader) GetBlockParentHash() string { + return b.BlockParentHash +} + +func (b testBlockHeader) GetBlockHash() string { + return b.BlockHash +} + +func (b testBlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} + +func newTestBlockHeader(blockNumber uint64, blockHash, blockParentHash string) testBlockHeader { + zeroTime, _ := time.Parse(time.DateTime, "2025-07-01 00:00:00") + return testBlockHeader{ + BlockNumber: blockNumber, + BlockHash: blockHash, + BlockParentHash: blockParentHash, + BlockTime: zeroTime.Add(time.Second * time.Duration(blockNumber)), + } +} + +type testClient struct { + mu sync.Mutex + + first uint64 + latest uint64 + subscribeCallBack func(latest BlockHeader, broken error) + + fork []uint64 + + broken error +} + +func newTestClient(first uint64, latest uint64) *testClient { + return &testClient{first: first, latest: latest} +} + +func (c *testClient) GetLatest(ctx context.Context) (latest BlockHeader, first uint64, err error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.buildHeader(c.latest), c.first, nil +} + +func (c *testClient) Subscribe(ctx context.Context, from BlockHeader, callback func(latest BlockHeader, broken error)) { + c.mu.Lock() + c.subscribeCallBack = callback + c.mu.Unlock() + <-ctx.Done() +} + +func (c *testClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.buildHeader(blockNumber), nil +} + +func (c *testClient) ResetCache(r BlockRange) { +} + +func (c *testClient) Snapshot() any { + return nil +} + +func (c *testClient) buildBlockHash(bn uint64) string { + var ver int + for _, fk := range c.fork { + if fk <= bn { + ver++ + } + } + return fmt.Sprintf("h%d-%d", bn, ver) +} + +func (c *testClient) buildHeader(bn uint64) testBlockHeader { + if bn == 0 { + return newTestBlockHeader(bn, c.buildBlockHash(0), "") + } + return newTestBlockHeader(bn, c.buildBlockHash(bn), c.buildBlockHash(bn-1)) +} + +func (c *testClient) Fork(bn uint64) { + c.mu.Lock() + defer c.mu.Unlock() + log.Warnf("fork at %d", bn) + c.fork = append(c.fork, bn) +} + +func (c *testClient) UpdateLatest(bn uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if bn <= c.latest { + return + } + if c.latest < bn { + c.latest = bn + } + c.subscribeCallBack(c.buildHeader(bn), c.broken) +} + +func (c *testClient) Broken(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.broken = err +} + +type testBlockDataFetcher struct { + client *testClient + taskSleep time.Duration + errTaskIndex uint64 + newTplIndex map[uint64]TemplateInstance + + mu sync.Mutex + latest BlockHeader + latestChanged chan struct{} +} + +func (f *testBlockDataFetcher) GetName() string { + return "testBlockDataFetcher" +} + +func (f *testBlockDataFetcher) GetFullRange() BlockRange { + return BlockRange{} +} + +func (f *testBlockDataFetcher) Snapshot() any { + return nil +} + +func (f *testBlockDataFetcher) KeepFetch(ctx context.Context) { + <-ctx.Done() +} + +func (f *testBlockDataFetcher) Get(ctx context.Context, bn uint64) ( + data BlockData, + has bool, + latest BlockHeader, + err error, +) { + f.mu.Lock() + defer f.mu.Unlock() + for bn > f.latest.GetBlockNumber() { + changed := f.latestChanged + f.mu.Unlock() + select { + case <-changed: + case <-ctx.Done(): + err = ctx.Err() + f.mu.Lock() + return + } + f.mu.Lock() + } + latest = f.latest + + taskLen := bn % 5 + // no data in block 5 15 25 ... + if taskLen == 0 && bn%2 != 0 { + return + } + + header, _ := f.client.GetHeaderIgnoreCache(context.Background(), bn) + tasks := make([]Task, taskLen) + for i := uint64(0); i < taskLen; i++ { + tasks[i] = &testTask{ + BlockHeader: header, + errIndex: f.errTaskIndex, + newTplIndex: f.newTplIndex, + sleep: f.taskSleep, + } + } + data = newTestBlockData(header, tasks...) + has = true + return +} + +func (f *testBlockDataFetcher) UpdateLatest(latest BlockHeader) { + f.mu.Lock() + f.latest = latest + close(f.latestChanged) + f.latestChanged = make(chan struct{}) + f.mu.Unlock() +} + +func (f *testBlockDataFetcher) Broken(err error) { +} + +func (f *testBlockDataFetcher) MoveStart(start uint64) { +} + +type testHandlerController struct { + Client *testClient + TaskSleep time.Duration + ErrTaskIndex uint64 + NewTplIndex map[uint64]TemplateInstance + EndBlock *uint64 + + BlockRange +} + +func (c *testHandlerController) Prologue( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + first uint64, + latest BlockHeader, +) *ExternalError { + c.BlockRange = BlockRange{StartBlock: first, EndBlock: c.EndBlock} + return nil +} + +func (c *testHandlerController) Epilogue() { +} + +func (c *testHandlerController) GetBlockRange() BlockRange { + return c.BlockRange +} + +func (c *testHandlerController) GetAgentStat() map[string]int { + return map[string]int{"placeholder": 1} +} + +func (c *testHandlerController) BuildBlockDataFetcher(_, _ uint64, latest BlockHeader) Fetcher[BlockData] { + return &testBlockDataFetcher{ + client: c.Client, + taskSleep: c.TaskSleep, + errTaskIndex: c.ErrTaskIndex, + newTplIndex: c.NewTplIndex, + latest: latest, + latestChanged: make(chan struct{}), + } +} + +func (c *testHandlerController) Snapshot() any { + return nil +} + +type testTask struct { + BlockHeader + index TaskIndex + errIndex uint64 + newTplIndex map[uint64]TemplateInstance + sleep time.Duration +} + +func (t *testTask) GetHandlerID() HandlerID { + return HandlerID{} +} + +func (t *testTask) Init(ctx context.Context, index TaskIndex, progressbar ProgressBar) { + t.index = index +} + +func (t *testTask) Summary() string { + return fmt.Sprintf("#%d binding data %d/%d in block %s", + t.index.Global, t.index.InBlock, t.index.TotalInBlock, GetBlockSummary(t)) +} + +func (t *testTask) Exec(ctx context.Context, checkpointCtrl CheckpointController) *ExternalError { + _, logger := log.FromContext(ctx, "block", t.GetBlockNumber(), "index", t.index) + logger.Debug("task start") + if t.errIndex == t.index.Global { + time.Sleep(t.sleep / 2) + logger.Warnf("task failed") + return NewExternalError(ErrCodeCallProcessorFailed, errors.Errorf("task %#v fail", t.index)) + } + if newTpl, has := t.newTplIndex[t.GetBlockNumber()]; has && t.index.InBlock == 0 { + time.Sleep(t.sleep / 2) + logger.Warnf("task has new template %s", newTpl) + return checkpointCtrl.NewTemplateInstance(ctx, t, []TemplateInstance{newTpl}) + } + select { + case <-ctx.Done(): + logger.Warnf("task canceled") + return NewExternalError(ErrCodeCallProcessorFailed, errors.Wrapf(ctx.Err(), "task %#v canceled", t.index)) + case <-time.After(t.sleep): + logger.Debug("task end") + return nil + } +} + +func Test_main_succeed(t *testing.T) { + //log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} + +func Test_main_succeed_and_end(t *testing.T) { + //log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: make(map[uint64]TemplateInstance), + EndBlock: utils.WrapPointer[uint64](80), // 80 will build a empty BlockData and make checkpoint, 75 will not + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + assert.NoError(t, err) + + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(80), last.BlockNumber) + assert.True(t, last.AllDone()) +} + +func Test_main_failed(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + ErrTaskIndex: 50, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + + var extErr *ExternalError + assert.True(t, errors.As(err, &extErr)) + assert.Equal(t, ErrCodeCallProcessorFailed, extErr.code) + assert.Equal(t, "task controller.TaskIndex{Global:0x32, InBlock:3, TotalInBlock:4} fail", extErr.error.Error()) +} + +func Test_main_reorg(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 30) + hc := &testHandlerController{ + Client: cli, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, true) + + go func() { + time.Sleep(time.Second) + cli.Fork(26) + cli.UpdateLatest(60) + time.Sleep(time.Second * 3) + cli.Fork(58) + cli.UpdateLatest(100) + }() + + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalReorgDetected) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalReorgDetected) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} + +func Test_main_newTpl(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: map[uint64]TemplateInstance{ + 49: { + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }, + 51: { + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }, + 52: { // dup create, will be ignored + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + }, + }, + } + bb := NewBlockBuilder(hc, cli, true) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + ccc := cc.(*checkpointController) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(48), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(51), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.Equal(t, uint64(100), cs.checkpoints[len(cs.checkpoints)-1].BlockNumber) +} + +func Test_main_removeTpl(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: map[uint64]TemplateInstance{ + 49: { + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }, + 51: { + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }, + 52: { // dup create, will be ignored + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + }, + 53: { // remove 0::0x1111 + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + Removed: true, + }, + }, + } + bb := NewBlockBuilder(hc, cli, true) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + ccc := cc.(*checkpointController) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(48), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(51), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + 53: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + Removed: true, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(53), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 3) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} diff --git a/driver/controller/notifier.go b/driver/controller/notifier.go new file mode 100644 index 0000000..7c71123 --- /dev/null +++ b/driver/controller/notifier.go @@ -0,0 +1,102 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/service/processor/models" +) + +// TaskInfo identifies the handler task an event is attributed to. +type TaskInfo struct { + Processor *models.Processor + ChainID string + Handler string + Category string + DataSource string +} + +// Notifier is the controller's outbound side channel. Each method describes +// something that happened in the controller; what to do with it (record an +// OpenTelemetry metric, call the processor service, ...) is up to the +// implementation, which lives in the driver binary. This keeps the controller +// free of the binary's metric and service-client packages so it can be hosted +// in sentio-core. +// +// No method returns an error: the controller treats notifications as +// best-effort, so the implementation logs and swallows any failure. +// +// The implementation is installed once at startup via SetNotifier. +type Notifier interface { + // Chain lifecycle. + + // DriverCreated reports that a chain's main controller has been created (once + // per chain). get returns the latest processed block number and whether it is + // available yet, for continuous observation. + DriverCreated(processor *models.Processor, chainID string, get func() (int64, bool)) + // DriverStarted reports that a chain's main stream has (re)started, carrying + // the current instance count of each template keyed by template id. + DriverStarted(ctx context.Context, processor *models.Processor, chainID string, templateInstanceCounts map[int32]int) + // ReorgDetected reports that a chain reorg was detected while fetching blocks. + ReorgDetected(ctx context.Context, processor *models.Processor, chainID string) + // ReorgDone reports that a detected reorg has been applied: the chain state was + // rolled back and persisted. reorgBlocks is the number of blocks rolled back; + // reduceToBlock is the processed block number afterwards (-1 if the rollback + // went below the start of the processed range). + ReorgDone(ctx context.Context, processor *models.Processor, chainID string, reorgBlocks uint64, reduceToBlock int64) + + // Per task. + + // BeforeEntityOperation returns a context tagged with the task, so the entity + // store can attribute the events it reports while serving the task. + BeforeEntityOperation(ctx context.Context, task TaskInfo) context.Context + // TaskDone reports that a handler task finished. + TaskDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration) + // SubgraphTaskDone reports that a subgraph handler task finished, together with + // the extra resource usage a subgraph task exposes: the time spent in each + // import function and the wasm memory used. + SubgraphTaskDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration, + importFuncUsed map[string]time.Duration, memoryUsed uint32) + // SubgraphRPCDone reports that an RPC call issued by a subgraph handler finished. + SubgraphRPCDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration) + // DataEmitted reports data a task produced. dataType/subtype/name describe the + // data point, e.g. ("event", "", name), ("metric", "gauge", name) or + // ("entity", op, name). + DataEmitted(ctx context.Context, task TaskInfo, dataType, subtype, name string, count int64) + + // On commit. + + // DataSaved reports data persisted for a processor on a chain. dataType/subtype/name + // describe the data point as in DataEmitted. + DataSaved(ctx context.Context, processor *models.Processor, chainID, dataType, subtype, name string, count int64) +} + +// N is the process-wide notifier. It defaults to a no-op so that tests and tools +// that build controllers without a driver binary do not panic; the driver binary +// installs the real implementation via SetNotifier before any controller runs. +var N Notifier = noopNotifier{} + +// SetNotifier installs the process-wide notifier. Called once by the driver +// binary at startup. +func SetNotifier(n Notifier) { + if n != nil { + N = n + } +} + +type noopNotifier struct{} + +func (noopNotifier) DriverCreated(*models.Processor, string, func() (int64, bool)) {} +func (noopNotifier) DriverStarted(context.Context, *models.Processor, string, map[int32]int) {} +func (noopNotifier) ReorgDetected(context.Context, *models.Processor, string) {} +func (noopNotifier) ReorgDone(context.Context, *models.Processor, string, uint64, int64) {} +func (noopNotifier) BeforeEntityOperation(ctx context.Context, _ TaskInfo) context.Context { + return ctx +} +func (noopNotifier) TaskDone(context.Context, TaskInfo, bool, time.Duration) {} +func (noopNotifier) SubgraphTaskDone(context.Context, TaskInfo, bool, time.Duration, map[string]time.Duration, uint32) { +} +func (noopNotifier) SubgraphRPCDone(context.Context, TaskInfo, bool, time.Duration) {} +func (noopNotifier) DataEmitted(context.Context, TaskInfo, string, string, string, int64) {} +func (noopNotifier) DataSaved(context.Context, *models.Processor, string, string, string, string, int64) { +} diff --git a/driver/controller/quota.go b/driver/controller/quota.go new file mode 100644 index 0000000..023aec9 --- /dev/null +++ b/driver/controller/quota.go @@ -0,0 +1,60 @@ +package controller + +import ( + "context" + "fmt" + "strings" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/timeseries" +) + +type OverQuota struct { + Msg string + Detail string +} + +type Usage struct { + TimeSeries map[timeseries.MetaType]map[string]int + Export map[string]int + EntityCreated map[string]int + EntityUpdated map[string]int +} + +func (u Usage) String() string { + var parts []string + metric := utils.SumMap(u.TimeSeries[timeseries.MetaTypeCounter]) + + utils.SumMap(u.TimeSeries[timeseries.MetaTypeGauge]) + if metric > 0 { + parts = append(parts, fmt.Sprintf("%d metric points", metric)) + } + if event := utils.SumMap(u.TimeSeries[timeseries.MetaTypeEvent]); event > 0 { + parts = append(parts, fmt.Sprintf("%d events", event)) + } + if entity := utils.SumMap(u.EntityCreated) + utils.SumMap(u.EntityUpdated); entity > 0 { + parts = append(parts, fmt.Sprintf("%d entity upserts", entity)) + } + if export := utils.SumMap(u.Export); export > 0 { + parts = append(parts, fmt.Sprintf("%d export messages", export)) + } + if len(parts) == 0 { + return "0 data" + } + return strings.Join(parts, " ") +} + +type QuotaService interface { + CheckOverQuota(ctx context.Context) (*OverQuota, error) + SaveUsage(ctx context.Context, usage Usage, inWatching bool) error +} + +type EmptyQuotaService struct { +} + +func (u EmptyQuotaService) CheckOverQuota(ctx context.Context) (*OverQuota, error) { + return nil, nil +} + +func (u EmptyQuotaService) SaveUsage(ctx context.Context, usage Usage, inWatching bool) error { + return nil +} diff --git a/driver/controller/range.go b/driver/controller/range.go new file mode 100644 index 0000000..c30f36a --- /dev/null +++ b/driver/controller/range.go @@ -0,0 +1,582 @@ +package controller + +import ( + "bytes" + "fmt" + "math" + + "sentioxyz/sentio-core/common/utils" +) + +type BlockRange struct { + StartBlock uint64 + EndBlock *uint64 +} + +var EmptyBlockRange = BlockRange{ + StartBlock: math.MaxUint64, + EndBlock: utils.WrapPointer[uint64](0), +} + +func EqualNilAsInf(a, b *uint64) bool { + if a == nil && b == nil { + return true + } + if a != nil && b != nil { + return *a == *b + } + return false +} + +func MinNilAsInf(ns ...*uint64) *uint64 { + var r uint64 + var has bool + for _, n := range ns { + if n == nil { + continue + } + if !has { + r, has = *n, true + } else { + r = min(r, *n) + } + } + if !has { + return nil + } + return &r +} + +func MaxNilAsInf(ns ...*uint64) *uint64 { + var r uint64 + for _, n := range ns { + if n == nil { + return nil + } + r = max(r, *n) + } + return &r +} + +func LessNilAsInf(a, b *uint64) bool { + if a == nil { + return false + } + if b == nil { + return true + } + return *a < *b +} + +func LessEqualNilAsInf(a, b *uint64) bool { + if a == nil { + return b == nil + } + if b == nil { + return true + } + return *a <= *b +} + +func (r BlockRange) EndOrZero() uint64 { + if r.EndBlock == nil { + return 0 + } + return *r.EndBlock +} + +func (r BlockRange) Equal(a BlockRange) bool { + return r.Include(a) && a.Include(r) +} + +func (r BlockRange) Contains(n uint64) bool { + if r.IsEmpty() { + return false + } + return r.StartBlock <= n && LessEqualNilAsInf(&n, r.EndBlock) +} + +func (r BlockRange) Include(a BlockRange) bool { + if a.IsEmpty() { + return true + } + if r.IsEmpty() { + return false + } + return r.StartBlock <= a.StartBlock && LessEqualNilAsInf(a.EndBlock, r.EndBlock) +} + +func (r BlockRange) IsEmpty() bool { + return r.EndBlock != nil && *r.EndBlock < r.StartBlock +} + +func (r BlockRange) String() string { + if r.EndBlock == nil { + return fmt.Sprintf("[%d,INF]", r.StartBlock) + } else if r.StartBlock > *r.EndBlock { + return fmt.Sprintf("[%d,%d/EMPTY]", r.StartBlock, *r.EndBlock) + } else { + return fmt.Sprintf("[%d,%d/%d]", r.StartBlock, *r.EndBlock, *r.EndBlock+1-r.StartBlock) + } +} + +func (r BlockRange) Intersection(a BlockRange) BlockRange { + if a.IsEmpty() || r.IsEmpty() { + return EmptyBlockRange + } + return BlockRange{ + StartBlock: max(r.StartBlock, a.StartBlock), + EndBlock: MinNilAsInf(r.EndBlock, a.EndBlock), + } +} + +func (r BlockRange) Remove(a BlockRange) BlockRangeSet { + if r.IsEmpty() { + return EmptyBlockRangeSet + } + if a.IsEmpty() { + return BlockRangeSet{BlockRange: r} + } + if a.StartBlock <= r.StartBlock { + if LessNilAsInf(a.EndBlock, &r.StartBlock) { + return BlockRangeSet{BlockRange: r} // no intersection, a is to the left of r + } + if LessNilAsInf(a.EndBlock, r.EndBlock) { + return BlockRangeSet{BlockRange: BlockRange{ // left part of r removed + StartBlock: *a.EndBlock + 1, + EndBlock: r.EndBlock, + }} + } + return EmptyBlockRangeSet // all removed + } + if LessNilAsInf(r.EndBlock, &a.StartBlock) { + return BlockRangeSet{BlockRange: r} // no intersection, a is to the right of r + } + // now r.StartBlock < a.StartBlock <= r.EndBlock + if LessNilAsInf(a.EndBlock, r.EndBlock) { + return BlockRangeSet{ // a middle part removed, remains are two separate part + BlockRange: r, + Holes: [][2]uint64{{a.StartBlock, *a.EndBlock}}, + } + } + return BlockRangeSet{BlockRange: BlockRange{ // right part of r removed + StartBlock: r.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }} +} + +// Cover return a minimal BlockRange that both include r and a +func (r BlockRange) Cover(a BlockRange) BlockRange { + if r.IsEmpty() { + return a + } + if a.IsEmpty() { + return r + } + return BlockRange{ + StartBlock: min(r.StartBlock, a.StartBlock), + EndBlock: MaxNilAsInf(r.EndBlock, a.EndBlock), + } +} + +type BlockRangeSet struct { + BlockRange + + // All the holes must be arranged strictly in ascending order, + // with no two holes overlapping or adjacent to each other. + // The left side of the leftmost hole and the right side of the rightmost hole are definitely not empty. + Holes [][2]uint64 +} + +var EmptyBlockRangeSet = BlockRangeSet{ + BlockRange: EmptyBlockRange, +} + +func (rs BlockRangeSet) Equal(a BlockRangeSet) bool { + if !rs.BlockRange.Equal(a.BlockRange) { + return false + } + if len(rs.Holes) != len(a.Holes) { + return false + } + for i := 0; i < len(rs.Holes); i++ { + if rs.Holes[i] != a.Holes[i] { + return false + } + } + return true +} + +func (rs BlockRangeSet) Contains(n uint64) bool { + if !rs.BlockRange.Contains(n) { + return false + } + for i := range rs.Holes { + if rs.Holes[i][0] <= n && n <= rs.Holes[i][1] { + return false + } + } + return true +} + +func (rs BlockRangeSet) Include(a BlockRange) bool { + if !rs.BlockRange.Include(a) { + return false + } + for i := range rs.Holes { + if !a.Intersection(BlockRange{StartBlock: rs.Holes[i][0], EndBlock: &rs.Holes[i][1]}).IsEmpty() { + return false + } + } + return true +} + +func (rs BlockRangeSet) Last() BlockRange { + if rs.IsEmpty() { + return EmptyBlockRange + } + if len(rs.Holes) == 0 { + return rs.BlockRange + } + return BlockRange{ + StartBlock: rs.Holes[len(rs.Holes)-1][1] + 1, + EndBlock: rs.EndBlock, + } +} + +func (rs BlockRangeSet) String() string { + var b bytes.Buffer + total, s := uint64(0), rs.StartBlock + b.WriteString(fmt.Sprintf("[%d,", rs.StartBlock)) + for _, hole := range rs.Holes { + leftLen := hole[0] - s + total += leftLen + s = hole[1] + 1 + b.WriteString(fmt.Sprintf("%d/%d]+[%d,", hole[0]-1, leftLen, s)) + } + if rs.EndBlock == nil { + b.WriteString("INF]") + } else if s > *rs.EndBlock { + b.WriteString(fmt.Sprintf("%d/EMPTY]", *rs.EndBlock)) + } else { + lastLen := *rs.EndBlock + 1 - s + b.WriteString(fmt.Sprintf("%d/%d]", *rs.EndBlock, lastLen)) + if len(rs.Holes) > 0 { + b.WriteString(fmt.Sprintf("/%d", total+lastLen)) + } + } + return b.String() +} + +func (rs BlockRangeSet) Intersection(a BlockRange) BlockRangeSet { + r := BlockRangeSet{ + BlockRange: rs.BlockRange.Intersection(a), + Holes: rs.Holes, + } + if r.IsEmpty() { + return EmptyBlockRangeSet + } + // remove invalid holes to the left + // pl: * x + // r.Holes: ... { } { } ... + // r.BlockRange: [ ... + // r.BlockRange: [ ... + pl := 0 + for pl < len(r.Holes) && r.Holes[pl][1] < r.StartBlock { + pl++ + } + if pl == len(r.Holes) { + r.Holes = nil // all holes are to the left of r + } else if r.StartBlock < r.Holes[pl][0] { + r.Holes = r.Holes[pl:] + } else { + r.StartBlock = r.Holes[pl][1] + 1 + r.Holes = r.Holes[pl+1:] + } + // remove invalid holes to the right + // pr: x * + // r.Holes: ... { } { } ... + // r.BlockRange: ... ] + // r.BlockRange: ... ] + pr := len(r.Holes) - 1 + for pr >= 0 && LessNilAsInf(r.EndBlock, &r.Holes[pr][0]) { + pr-- + } + if pr < 0 { + r.Holes = nil // all holes are to the right of r + } else if LessNilAsInf(&r.Holes[pr][1], r.EndBlock) { + r.Holes = r.Holes[:pr+1] + } else { + r.EndBlock = utils.WrapPointer(r.Holes[pr][0] - 1) + r.Holes = r.Holes[:pr] + } + + if r.IsEmpty() { + return EmptyBlockRangeSet + } + if len(r.Holes) == 0 { + r.Holes = nil + } + return r +} + +func (rs BlockRangeSet) Remove(a BlockRange) (result BlockRangeSet) { + if a.IsEmpty() || rs.IsEmpty() { + return rs + } + if LessNilAsInf(a.EndBlock, &rs.StartBlock) { + // rs: [ ] + // a: [ ] + // no intersection, a is to the left of rs + return rs + } + if LessNilAsInf(rs.EndBlock, &a.StartBlock) { + // rs: [ ] + // a: [ ] + // no intersection, a is to the right of rs + return rs + } + // rs: [ ] + // a: [ ] + // ^ ^ + // left right + left := EmptyBlockRangeSet + if rs.StartBlock < a.StartBlock { + // rs: [ ] + // a: [ ... + // always have left part + for i := 0; i < len(rs.Holes); i++ { + if a.StartBlock < rs.Holes[i][0] { + // rs: ... [ ] (rs.Holes[i]) [ ] [ ] + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }, + Holes: rs.Holes[:i], + } + break + } + if a.StartBlock <= rs.Holes[i][1]+1 { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(rs.Holes[i][0] - 1), + }, + Holes: rs.Holes[:i], + } + break + } + } + if left.IsEmpty() { + // rs: ... [ ] [ ] [ ] + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }, + Holes: rs.Holes, + } + } + } + right := EmptyBlockRangeSet + if LessNilAsInf(a.EndBlock, rs.EndBlock) { + // rs: [ ] + // a: ... ] + // always have right part + for i := 0; i < len(rs.Holes); i++ { + if *a.EndBlock < rs.Holes[i][0]-1 { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: *a.EndBlock + 1, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes[i:], + } + break + } + if *a.EndBlock <= rs.Holes[i][1] { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.Holes[i][1] + 1, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes[i+1:], + } + break + } + } + if right.IsEmpty() { + // rs: ... [ ] [ ] [ ] + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: *a.EndBlock + 1, + EndBlock: rs.EndBlock, + }, + } + } + } + if !left.IsEmpty() && !right.IsEmpty() { + result = BlockRangeSet{ + BlockRange: rs.BlockRange, + } + result.Holes = append(result.Holes, left.Holes...) + result.Holes = append(result.Holes, [2]uint64{ + *left.EndBlock + 1, + right.StartBlock - 1, + }) + result.Holes = append(result.Holes, right.Holes...) + } else if left.IsEmpty() { + result = right + } else { + result = left + } + if result.IsEmpty() { + return EmptyBlockRangeSet + } + if len(result.Holes) == 0 { + result.Holes = nil + } + return result +} + +func (rs BlockRangeSet) Union(a BlockRange) BlockRangeSet { + if a.IsEmpty() { + return rs + } + if rs.IsEmpty() { + return BlockRangeSet{BlockRange: a} + } + if LessNilAsInf(rs.EndBlock, &a.StartBlock) { + if *rs.EndBlock+1 == a.StartBlock { + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: a.EndBlock, + }, + Holes: rs.Holes, + } + } + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: a.EndBlock, + }, + Holes: append(rs.Holes, [2]uint64{ + *rs.EndBlock + 1, + a.StartBlock - 1, + }), + } + } + if LessNilAsInf(a.EndBlock, &rs.StartBlock) { + if *a.EndBlock+1 == rs.StartBlock { + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: a.StartBlock, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes, + } + } + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: a.StartBlock, + EndBlock: rs.EndBlock, + }, + Holes: utils.Prepend(rs.Holes, [2]uint64{ + *a.EndBlock + 1, + rs.StartBlock - 1, + }), + } + } + result := BlockRangeSet{BlockRange: a.Cover(rs.BlockRange)} + for i := 0; i < len(rs.Holes); i++ { + remain := BlockRange{StartBlock: rs.Holes[i][0], EndBlock: &rs.Holes[i][1]}.Remove(a) + if remain.IsEmpty() { + continue // The hole was completely filled. + } + if len(remain.Holes) == 1 { + // The middle part of the hole was filled in, turned into two holes. + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: [ ] + // remain: [ ] [ ] + // result: ... [ ] [ ] [ ] ... + result.Holes = append(result.Holes, [2]uint64{ + rs.Holes[i][0], + remain.Holes[0][0] - 1, + }, [2]uint64{ + remain.Holes[0][1] + 1, + rs.Holes[i][1], + }) + // append all remain rs.Holes and then break + result.Holes = append(result.Holes, rs.Holes[i+1:]...) + break + } + // len(remain.Holes) cannot be greater than 1, so here it must be 0, + // it means the hole was either partially filled on the left or right side, or left completely as it was. + result.Holes = append(result.Holes, [2]uint64{ + remain.StartBlock, + *remain.EndBlock, + }) + } + return result +} + +// CutRangeSet divide the entire range into multiple non-overlapping ranges +// using the endpoints of multiple potentially intersecting ranges. +func CutRangeSet(start uint64, rs []BlockRange) []BlockRange { + if len(rs) == 0 { + return nil + } + sbn := make(map[uint64]bool) + var inf bool + for _, r := range rs { + if r.EndBlock != nil && *r.EndBlock < start { + continue + } + sbn[max(r.StartBlock, start)] = true + if r.EndBlock != nil { + sbn[*r.EndBlock+1] = true + } else { + inf = true + } + } + ns := utils.GetOrderedMapKeys(sbn) + var result []BlockRange + for i := 0; i+1 < len(ns); i++ { + end := ns[i+1] - 1 + result = append(result, BlockRange{ + StartBlock: ns[i], + EndBlock: &end, + }) + } + if inf { + result = append(result, BlockRange{ + StartBlock: ns[len(ns)-1], + }) + } + return result +} diff --git a/driver/controller/range_test.go b/driver/controller/range_test.go new file mode 100644 index 0000000..6bd4658 --- /dev/null +++ b/driver/controller/range_test.go @@ -0,0 +1,552 @@ +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/common/utils" +) + +func newBlockRange(a, b uint64) BlockRange { + return BlockRange{StartBlock: a, EndBlock: &b} +} + +func Test_cmpNilAsInf(t *testing.T) { + one := uint64(1) + two := uint64(2) + testcases := []struct { + a, b, max, min *uint64 + eq, lt, le bool + }{ + {a: nil, b: nil, max: nil, min: nil, eq: true, lt: false, le: true}, + {a: nil, b: &one, max: nil, min: &one, eq: false, lt: false, le: false}, + {a: &one, b: nil, max: nil, min: &one, eq: false, lt: true, le: true}, + {a: &one, b: &two, max: &two, min: &one, eq: false, lt: true, le: true}, + {a: &two, b: &one, max: &two, min: &one, eq: false, lt: false, le: false}, + {a: &two, b: &two, max: &two, min: &two, eq: true, lt: false, le: true}, + } + for i, tc := range testcases { + assert.Equalf(t, tc.max, MaxNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.min, MinNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.eq, EqualNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.lt, LessNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.le, LessEqualNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + } +} + +func Test_remove(t *testing.T) { + for si := uint64(0); si <= 10; si++ { + for ei := si - 1; ei <= 10; ei++ { + for sj := uint64(0); sj <= 10; sj++ { + for ej := sj - 1; ej <= 10; ej++ { + ri := newBlockRange(si, ei) + rj := newBlockRange(sj, ej) + rk := ri.Remove(rj) + for x := uint64(0); x <= 11; x++ { + ok := si <= x && x <= ei && !(sj <= x && x <= ej) + //log.Debugf("!!! ri:%s, rj:%s, ri.Remove(rj):%s, x:%d, ok:%v\n", ri, rj, rk, x, ok) + assert.Equalf(t, ok, rk.Contains(x), "invalid result ri:%s, rj:%s, ri.Remove(rj):%s, x:%d, ok:%v", rj, rj, rk, x, ok) + } + } + } + } + } +} + +func Test_equal(t *testing.T) { + assert.Equal(t, true, EmptyBlockRange.Equal(newBlockRange(1, 0))) + assert.Equal(t, true, newBlockRange(1, 0).Equal(EmptyBlockRange)) + + for si := uint64(0); si <= 10; si++ { + for ei := si; ei <= 10; ei++ { + for sj := uint64(0); sj <= 10; sj++ { + for ej := sj; ej <= 10; ej++ { + ri := newBlockRange(si, ei) + rj := newBlockRange(sj, ej) + eq := si == sj && ei == ej + assert.Equalf(t, eq, ri.Equal(rj), "invalid result ri:%s, rj:%s, eq:%v", ri, rj, eq) + } + } + } + } +} + +func Test_include(t *testing.T) { + assert.Equal(t, true, EmptyBlockRange.Include(newBlockRange(1, 0))) + assert.Equal(t, true, newBlockRange(1, 0).Include(EmptyBlockRange)) + assert.Equal(t, false, EmptyBlockRange.Include(BlockRange{StartBlock: 0})) + assert.Equal(t, true, BlockRange{StartBlock: 0}.Include(EmptyBlockRange)) +} + +func Test_setContains(t *testing.T) { + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + contains := []uint64{1, 4, 5, 6, 10} + for x := uint64(0); x <= 11; x++ { + ok := utils.IndexOf(contains, x) >= 0 + assert.Equalf(t, ok, set.Contains(x), "invalid result contains:%v", x) + } +} + +func Test_setInclude(t *testing.T) { + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + okPairs := [][2]uint64{ + {1, 1}, + {4, 4}, + {4, 5}, + {4, 6}, + {5, 5}, + {5, 6}, + {6, 6}, + {10, 10}, + } + for s := uint64(1); s <= 10; s++ { + for e := s; e <= 10; e++ { + ok := utils.IndexOf(okPairs, [2]uint64{s, e}) >= 0 + assert.Equalf(t, ok, set.Include(newBlockRange(s, e)), "invalid testcase: [%s,%s], ok: %v", s, e, ok) + } + } +} + +func Test_setLast(t *testing.T) { + assert.Equal(t, EmptyBlockRange, EmptyBlockRangeSet.Last()) + + set := BlockRangeSet{ + BlockRange: newBlockRange(1, 10), + } + assert.Equal(t, newBlockRange(1, 10), set.Last()) + + set.Holes = [][2]uint64{ + {2, 3}, + {7, 9}, + } + assert.Equal(t, newBlockRange(10, 10), set.Last()) +} + +func Test_setIntersection(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Intersection(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Intersection(BlockRange{StartBlock: 1})) + + // [1][4-6][10] + set := BlockRangeSet{ + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{{2, 3}, {7, 9}}, + } + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(EmptyBlockRange)) + assert.Equal(t, set, set.Intersection(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 9))) + + assert.Equal(t, set, set.Intersection(newBlockRange(1, 10))) + assert.Equal(t, set, set.Intersection(newBlockRange(1, 11))) + assert.Equal(t, set, set.Intersection(newBlockRange(0, 10))) + assert.Equal(t, set, set.Intersection(newBlockRange(0, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(2, 2))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(2, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(3, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(4, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 5)}, set.Intersection(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(5, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(6, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 7))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 8))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(7, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(8, 8))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(8, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(9, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(10, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(11, 11))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(0, 0))) +} + +func Test_setRemove(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Remove(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Remove(BlockRange{StartBlock: 1})) + + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + assert.Equal(t, set, set.Remove(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(0, 0))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 9))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(0, 10))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(0, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 9))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(1, 10))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(1, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(2, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(2, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(3, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(4, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 5}, {7, 9}}}, set.Remove(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(5, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(6, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(7, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(8, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(8, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(9, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(10, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(11, 11))) +} + +func Test_setUnion(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Union(EmptyBlockRange)) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}.Union(EmptyBlockRange)) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}, EmptyBlockRangeSet.Union(BlockRange{StartBlock: 1})) + + set := BlockRangeSet{ // [2][5-7][11] + BlockRange: newBlockRange(2, 11), + Holes: [][2]uint64{ + {3, 4}, + {8, 10}, + }, + } + assert.Equal(t, set, set.Union(EmptyBlockRange)) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{1, 1}, {3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 0))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(0, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(0, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(0, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11)}, set.Union(newBlockRange(0, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11)}, set.Union(newBlockRange(0, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 12)}, set.Union(newBlockRange(0, 12))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 0}}, set.Union(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(1, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11)}, set.Union(newBlockRange(1, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11)}, set.Union(newBlockRange(1, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 12)}, set.Union(newBlockRange(1, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(2, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(2, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12)}, set.Union(newBlockRange(2, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(3, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12)}, set.Union(newBlockRange(3, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {9, 10}}}, set.Union(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {10, 10}}}, set.Union(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(7, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(7, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(8, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}, {10, 10}}}, set.Union(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(11, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(11, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(12, 12))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 2}, Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(BlockRange{StartBlock: 12})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 13), Holes: [][2]uint64{{3, 4}, {8, 10}, {12, 12}}}, set.Union(newBlockRange(13, 13))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 2}, Holes: [][2]uint64{{3, 4}, {8, 10}, {12, 12}}}, set.Union(BlockRange{StartBlock: 13})) +} + +func Test_CutRangeSet(t *testing.T) { + assert.Nil(t, CutRangeSet(0, nil)) + assert.Nil(t, CutRangeSet(0, []BlockRange{})) + + src := []BlockRange{ + {StartBlock: 100}, + {StartBlock: 100}, + {StartBlock: 100}, + } + assert.Equal(t, []BlockRange{{StartBlock: 100}}, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{{StartBlock: 100}}, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{{StartBlock: 110}}, CutRangeSet(110, src)) + + src = []BlockRange{ + {StartBlock: 100}, + {StartBlock: 200}, + {StartBlock: 300}, + } + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 150, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(150, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 199, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(199, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(200, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 250, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(250, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 299, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(299, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300}}, CutRangeSet(300, src)) + assert.Equal(t, []BlockRange{{StartBlock: 301}}, CutRangeSet(301, src)) + + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](150)}, + {StartBlock: 151, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](350)}, + {StartBlock: 351, EndBlock: utils.WrapPointer[uint64](550)}, + }, CutRangeSet(0, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](150)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](350)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](550)}, + })) + + src = []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + } + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 199, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(199, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(200, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(299, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(300, src)) + assert.Equal(t, []BlockRange{{StartBlock: 350, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(350, src)) + assert.Equal(t, []BlockRange{{StartBlock: 399, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(399, src)) + assert.Equal(t, []BlockRange(nil), CutRangeSet(400, src)) + assert.Equal(t, []BlockRange(nil), CutRangeSet(401, src)) +} diff --git a/driver/controller/standard/BUILD.bazel b/driver/controller/standard/BUILD.bazel new file mode 100644 index 0000000..643502b --- /dev/null +++ b/driver/controller/standard/BUILD.bazel @@ -0,0 +1,50 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "standard", + srcs = [ + "binding_data.go", + "config.go", + "convert.go", + "handler.go", + "helper.go", + "interval.go", + "task.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard", + visibility = ["//visibility:public"], + deps = [ + "//common/concurrency", + "//common/envconf", + "//common/log", + "//common/protojson", + "//common/timer", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/timeseries", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//encoding/gzip", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "standard_test", + srcs = ["convert_test.go"], + embed = [":standard"], + deps = [ + "//driver/controller", + "//processor/protos", + "@com_github_stretchr_testify//assert", + "@org_golang_google_protobuf//proto", + "@org_golang_google_protobuf//types/known/structpb", + ], +) diff --git a/driver/controller/standard/aptos/BUILD.bazel b/driver/controller/standard/aptos/BUILD.bazel new file mode 100644 index 0000000..59bd18c --- /dev/null +++ b/driver/controller/standard/aptos/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "aptos", + srcs = [ + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/aptos", + visibility = ["//visibility:public"], + deps = [ + "//chain/aptos", + "//chain/move", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/aptos", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_aptos_labs_aptos_go_sdk//:aptos-go-sdk", + "@com_github_pkg_errors//:errors", + ], +) diff --git a/driver/controller/standard/aptos/block_data.go b/driver/controller/standard/aptos/block_data.go new file mode 100644 index 0000000..b46ee23 --- /dev/null +++ b/driver/controller/standard/aptos/block_data.go @@ -0,0 +1,81 @@ +package aptos + +import ( + "encoding/json" + "strings" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" +) + +type BlockData struct { + controller.BlockHeader + + mainData aptosdata.BlockMainData + accountResources []aptosdata.AccountResource + + cachedTxn map[string]string + cachedEvent map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getRawTxn( + fetchConfig aptos.TransactionFetchConfig, + eventFilters []aptos.EventFilter, +) (string, error) { + key := fetchConfig.String() + "@" + strings.Join(utils.MapSliceNoError(eventFilters, aptos.EventFilter.String), "|") + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if rawTxn, has := b.cachedTxn[key]; has { + return rawTxn, nil + } + txn := fetchConfig.PruneTransaction(*b.mainData.Txn, eventFilters) + raw, err := json.Marshal(txn) + if err != nil { + return "", err + } + rawTxn := string(raw) + b.cachedTxn[key] = rawTxn + return rawTxn, nil +} + +func (b *BlockData) getRawEvent(index int) (string, error) { + if b.cachedEvent == nil { + b.cachedEvent = make(map[int]string) + } + if rawEvent, has := b.cachedEvent[index]; has { + return rawEvent, nil + } + ev := b.mainData.Txn.Events[index] + raw, err := json.Marshal(ev) + if err != nil { + return "", err + } + rawEvent := string(raw) + b.cachedEvent[index] = rawEvent + return rawEvent, nil +} diff --git a/driver/controller/standard/aptos/handler.go b/driver/controller/standard/aptos/handler.go new file mode 100644 index 0000000..708bf7d --- /dev/null +++ b/driver/controller/standard/aptos/handler.go @@ -0,0 +1,431 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + aptossdk "github.com/aptos-labs/aptos-go-sdk" + "github.com/pkg/errors" + + chainaptos "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/chain/move" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type AptosHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[aptos.Client, *BlockData, AptosHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client aptos.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[aptos.Client, *BlockData, AptosHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetAddressStartBlock(ctx, address, start, latest) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, accountConfig := range c.Config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + normalizedAccountAddr := accountAddress + var accountAddr aptossdk.AccountAddress + if accountAddress != "" { + if err = accountAddr.ParseStringRelaxed(accountAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, + errors.Wrapf(err, "invalid account address %q", accountAddress)) + } + normalizedAccountAddr = accountAddr.String() + } + dataSource := fmt.Sprintf("APTOS:%s/Account:%s", c.ChainConfig.ChainID, accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + blockRange.StartBlock, err = c.getAddressStart(ctx, normalizedAccountAddr, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, intervalConfig := range accountConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + } + if accountAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("account address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig = aptos.AccountResourceFilter{ + Address: normalizedAccountAddr, + ResourceType: nil, // default need all resources of the account + } + if resourceType := intervalConfig.GetType(); resourceType != "" { + agent.FetchConfig.ResourceType = []string{resourceType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range accountConfig.GetMoveResourceChangeConfigs() { + resTypes := changeConfig.GetTypes() + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + Filter: chainaptos.ChangeFilter{ + Address: set.New[aptossdk.AccountAddress](), + ResourceTypes: nil, + }, + } + agent.Filter.ResourceTypes, err = utils.MapSlice(resTypes, move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + if accountAddress != "" { + agent.Filter.Address.Add(accountAddr) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + normalizedContractAddr := contractAddress + if contractAddress != "" { + var contractAddr aptossdk.AccountAddress + if err = contractAddr.ParseStringRelaxed(contractAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, + errors.Wrapf(err, "invalid contract address %q", contractAddress)) + } + normalizedContractAddr = contractAddr.String() + } + dataSource := fmt.Sprintf("APTOS:%s/Contract:%s", c.ChainConfig.ChainID, contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + blockRange.StartBlock, err = c.getAddressStart(ctx, normalizedContractAddr, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, moveCallConfig := range contractConfig.GetMoveCallConfigs() { + agent := HandlerAgentFunction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "call", moveCallConfig, blockRange), + Filter: chainaptos.TransactionFilter{ + FailedIsOK: moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction(), + MultiSigTxnIsOK: moveCallConfig.GetFetchConfig().GetSupportMultisigFunc(), + }, + } + agent.Filter.FunctionFilters, err = utils.MapSlice( + moveCallConfig.GetFilters(), + func(f *protos.MoveCallFilter) (ff chainaptos.FunctionFilter, err error) { + var funcName string + if f.GetFunction() != "" { + funcName = contractAddress + "::" + f.GetFunction() + } else if contractAddress != "" { + funcName = contractAddress + "::" + contractConfig.GetContract().GetName() + } + ff.FunctionPattern, err = move.BuildType(funcName) + if err != nil { + return ff, errors.Wrapf(err, "invalid call function %q", funcName) + } + if f.GetWithTypeArguments() { + ff.CheckTypeArguments, ff.TypedArguments = true, f.GetTypeArguments() + } + if sender := f.GetFromAndToAddress().GetFrom(); sender != "" { + var senderAddr aptossdk.AccountAddress + if err = senderAddr.ParseStringRelaxed(sender); err != nil { + return ff, errors.Wrapf(err, "invalid sender address %q", sender) + } + ff.Sender = &senderAddr + } + return + }, + ) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "handler %s have invalid filter", agent.GetHandlerID().String())) + } + if len(agent.Filter.FunctionFilters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter in handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig.NeedAllEvents = moveCallConfig.GetFetchConfig().GetAllEvents() + if moveCallConfig.GetFetchConfig().GetResourceChanges() { + var resType move.Type + resType, err = move.BuildType(moveCallConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid resouce type prefix %q", + agent.GetHandlerID().String(), + moveCallConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix())) + } + agent.FetchConfig.ChangeResourceTypes = move.TypeSet{resType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, eventConfig := range contractConfig.GetMoveEventConfigs() { + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "event", eventConfig, blockRange), + Filter: chainaptos.TransactionFilter{ + FailedIsOK: eventConfig.GetFetchConfig().GetIncludeFailedTransaction(), + MultiSigTxnIsOK: eventConfig.GetFetchConfig().GetSupportMultisigFunc(), + }, + } + if contractAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.Filter.EventFilters, err = utils.MapSlice( + eventConfig.GetFilters(), + func(f *protos.MoveEventFilter) (ff chainaptos.EventFilter, err error) { + eventType := contractAddress + "::" + f.GetType() + ff.Type, err = move.BuildType(eventType) + if err != nil { + return ff, errors.Wrapf(err, "invalid event type %q", eventType) + } + if f.GetEventAccount() != "" { + var addr aptossdk.AccountAddress + if err = addr.ParseStringRelaxed(f.GetEventAccount()); err != nil { + return ff, errors.Wrapf(err, "invalid event account %q", f.GetEventAccount()) + } + ff.GuiAccountAddress = &addr + } + if ff.IsEmpty() { + return ff, errors.New("event filter is empty") + } + return ff, nil + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "handler %s have invalid filter", agent.GetHandlerID().String())) + } + if len(agent.Filter.EventFilters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter in handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig.NeedAllEvents = eventConfig.GetFetchConfig().GetAllEvents() + if eventConfig.GetFetchConfig().GetResourceChanges() { + var resType move.Type + resType, err = move.BuildType(eventConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid resouce type prefix %q", + agent.GetHandlerID().String(), + eventConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix())) + } + agent.FetchConfig.ChangeResourceTypes = move.TypeSet{resType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, intervalConfig := range contractConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + if contractAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig = aptos.AccountResourceFilter{ + Address: normalizedContractAddr, + ResourceType: make([]string, 0), // do not need any resource + } + if resourceType := intervalConfig.GetType(); resourceType != "" { + logger.Warnf("type %q in contract move interval config %s will be ignored", resourceType, agent.HandlerID) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("APTOS::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + aptos.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 1000, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from aptos.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // take the main data and ask the agents what account resources are needed + var accountResourceFilters []aptos.AccountResourceFilter + var needFullTx bool + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(from.Intervals, ag.IntervalConfig) { + continue + } + if ag.FetchConfig.NeedNothing() { + needFullTx = true + } else { + accountResourceFilters = append(accountResourceFilters, ag.FetchConfig) + } + } + // actually get the extended data + result.accountResources, err = c.Client.GetAccountResources( + ctx, blockNumber, aptos.MergeAccountResourceFilters(accountResourceFilters)) + if err != nil { + return nil, false, err + } + // fetch the transaction + if needFullTx { + // Although result.mainData.Txn may already exist, it might be missing some events or changes. + // So need to re-get the tx here + var tx chainaptos.Transaction + if tx, err = c.Client.GetTransaction(ctx, blockNumber); err != nil { + return nil, false, err + } + result.mainData.Txn = &tx + } + // always need header, fill result.BlockHeader here + if result.mainData.Txn != nil { + result.BlockHeader = (*aptos.Transaction)(result.mainData.Txn) + } else if result.mainData.SimpleTxn != nil { + result.BlockHeader = (*aptos.MinimalistTransaction)(result.mainData.SimpleTxn) + } else if result.BlockHeader, err = c.Client.GetMinimalistTransaction(ctx, blockNumber); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) + +} + +func (c *HandlerController) getDataRequirement() (dr aptos.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentChange: + dr.Changes = append(dr.Changes, aptos.ChangeRequirement{ + BlockRange: ag.Range, + ChangeFilter: ag.Filter, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, aptos.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, aptos.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} diff --git a/driver/controller/standard/aptos/handler_change.go b/driver/controller/standard/aptos/handler_change.go new file mode 100644 index 0000000..0609134 --- /dev/null +++ b/driver/controller/standard/aptos/handler_change.go @@ -0,0 +1,53 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +// HandlerAgentChange only need txn changes, do not need the txn self, so no aptos.FetchTxnConfig in it +type HandlerAgentChange struct { + controller.BaseHandlerAgent + + Filter aptos.ChangeFilter +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + changes := utils.FilterArr(bd.mainData.Changes, func(c aptosdata.Change) bool { + return a.Filter.Check(c.WriteSetChange) + }) + resources := utils.MapSliceNoError(changes, func(c aptosdata.Change) string { return c.Raw }) + if len(resources) == 0 { + return nil, nil + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_RESOURCE, + Data: &protos.Data{ + Value: &protos.Data_AptResource_{ + AptResource: &protos.Data_AptResource{ + Version: int64(bd.GetBlockNumber()), + RawResources: resources, + TimestampMicros: bd.GetBlockTime().UnixMicro(), + }, + }, + }, + DataSize: utils.StringsLenSum(resources), + }}, nil +} + +func (a HandlerAgentChange) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/standard/aptos/handler_event.go b/driver/controller/standard/aptos/handler_event.go new file mode 100644 index 0000000..0aa20c0 --- /dev/null +++ b/driver/controller/standard/aptos/handler_event.go @@ -0,0 +1,65 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if bd.mainData.Txn == nil || !a.Filter.Check(*bd.mainData.Txn) { + return nil, nil + } + rawTxn, err := bd.getRawTxn(a.FetchConfig, a.Filter.EventFilters) + if err != nil { + return nil, err + } + var result []standard.BindingDataInner + eventChecker := aptos.BuildEventFilter(a.Filter.EventFilters) + for i, ev := range bd.mainData.Txn.Events { + if !eventChecker(ev) { + continue + } + var rawEvent string + if rawEvent, err = bd.getRawEvent(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_APT_EVENT, + TxInnerIndex: int(ev.Index), + Data: &protos.Data{ + Value: &protos.Data_AptEvent_{ + AptEvent: &protos.Data_AptEvent{ + EventIndex: ev.Index, + RawEvent: rawEvent, + RawTransaction: rawTxn, + }, + }, + }, + DataSize: len(rawTxn) + len(rawEvent), + }) + } + return result, nil +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/aptos/handler_function.go b/driver/controller/standard/aptos/handler_function.go new file mode 100644 index 0000000..1c5c1de --- /dev/null +++ b/driver/controller/standard/aptos/handler_function.go @@ -0,0 +1,50 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentFunction struct { + controller.BaseHandlerAgent + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if bd.mainData.Txn == nil || !a.Filter.Check(*bd.mainData.Txn) { + return nil, nil + } + rawTxn, err := bd.getRawTxn(a.FetchConfig, a.Filter.EventFilters) + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_CALL, + Data: &protos.Data{ + Value: &protos.Data_AptCall_{ + AptCall: &protos.Data_AptCall{ + RawTransaction: rawTxn, + }, + }, + }, + DataSize: len(rawTxn), + }}, nil +} + +func (a HandlerAgentFunction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/aptos/handler_interval.go b/driver/controller/standard/aptos/handler_interval.go new file mode 100644 index 0000000..0665a1d --- /dev/null +++ b/driver/controller/standard/aptos/handler_interval.go @@ -0,0 +1,78 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/chain/move" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig + FetchConfig aptosdata.AccountResourceFilter +} + +func (a HandlerAgentInterval) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + if a.FetchConfig.NeedNothing() { + // is a contract move interval handler, need a APT_CALL binding data + rawTxn, err := bd.getRawTxn(aptos.TransactionFetchConfig{ + NeedAllEvents: true, + ChangeResourceTypes: move.TypeSet{move.MustBuildType("")}, + }, nil) + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_CALL, + Data: &protos.Data{ + Value: &protos.Data_AptCall_{ + AptCall: &protos.Data_AptCall{ + RawTransaction: rawTxn, + }, + }, + }, + }}, nil + } + // is a account move interval handler, need a APT_RESOURCE binding data + ars := utils.FilterArr(bd.accountResources, a.FetchConfig.Check) + resources := utils.MapSliceNoError(ars, func(ar aptosdata.AccountResource) string { return ar.Raw }) + if len(resources) == 0 { + return nil, nil + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_RESOURCE, + Data: &protos.Data{ + Value: &protos.Data_AptResource_{ + AptResource: &protos.Data_AptResource{ + Version: int64(bd.GetBlockNumber()), + RawResources: resources, + TimestampMicros: bd.GetBlockTime().UnixMicro(), + }, + }, + }, + DataSize: utils.StringsLenSum(resources), + }}, nil +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/binding_data.go b/driver/controller/standard/binding_data.go new file mode 100644 index 0000000..e70e0c6 --- /dev/null +++ b/driver/controller/standard/binding_data.go @@ -0,0 +1,69 @@ +package standard + +import ( + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/processor/protos" +) + +type bindingData struct { + controller.BlockHeader + + handlerID controller.HandlerID + data *protos.DataBinding + txIndex int + txInnerIndex int +} + +func (b bindingData) Cmp(x bindingData, mode protos.ExecutionConfig_HandlerOrderInsideTransaction) int { + if r := utils.Cmp(b.GetBlockNumber(), x.GetBlockNumber()); r != 0 { + return r + } + if r := utils.Cmp(b.txIndex, x.txIndex); r != 0 { + return r + } + if mode == protos.ExecutionConfig_BY_PROCESSOR_AND_LOG_INDEX { + if r := utils.Cmp(b.handlerID.DataSourceID, x.handlerID.DataSourceID); r != 0 { + return r + } + if r := CmpHandlerType(b.data, x.data); r != 0 { + return r + } + if r := utils.Cmp(b.handlerID.ID, x.handlerID.ID); r != 0 { + return r + } + return utils.Cmp(b.txInnerIndex, x.txInnerIndex) + } else { + if r := CmpHandlerType(b.data, x.data); r != 0 { + return r + } + if r := utils.Cmp(b.txInnerIndex, x.txInnerIndex); r != 0 { + return r + } + return utils.Cmp(b.handlerID.ID, x.handlerID.ID) + } +} + +var executeOrder = map[protos.HandlerType]int{ + protos.HandlerType_ETH_TRACE: 1, + protos.HandlerType_ETH_LOG: 2, + protos.HandlerType_ETH_TRANSACTION: 3, + protos.HandlerType_ETH_BLOCK: 4, + protos.HandlerType_SOL_INSTRUCTION: 1, + protos.HandlerType_APT_CALL: 1, + protos.HandlerType_APT_EVENT: 2, + protos.HandlerType_APT_RESOURCE: 3, + protos.HandlerType_SUI_CALL: 1, + protos.HandlerType_SUI_EVENT: 2, + protos.HandlerType_SUI_OBJECT_CHANGE: 3, + protos.HandlerType_SUI_OBJECT: 4, + protos.HandlerType_FUEL_RECEIPT: 2, + protos.HandlerType_FUEL_TRANSACTION: 3, + protos.HandlerType_FUEL_BLOCK: 4, + protos.HandlerType_COSMOS_CALL: 1, + protos.HandlerType_STARKNET_EVENT: 1, +} + +func CmpHandlerType(a, b *protos.DataBinding) int { + return utils.Cmp(executeOrder[a.GetHandlerType()], executeOrder[b.GetHandlerType()]) +} diff --git a/driver/controller/standard/config.go b/driver/controller/standard/config.go new file mode 100644 index 0000000..e1b30bc --- /dev/null +++ b/driver/controller/standard/config.go @@ -0,0 +1,16 @@ +package standard + +import ( + "time" + + "sentioxyz/sentio-core/common/envconf" +) + +var ( + enableBindingDataPartition = envconf.LoadBool("SENTIO_ENABLE_BINDING_DATA_PARTITION", false) + minBackfillSlotInterval = envconf.LoadUInt64("SENTIO_MIN_BACKFILL_SLOT_INTERVAL", 1000, envconf.WithMin(1)) + minBackfillTimeInterval = envconf.LoadDuration("SENTIO_MIN_BACKFILL_TIME_INTERVAL", time.Hour, + envconf.WithMinDuration(time.Minute)) + disableAgentTypes = envconf.LoadString("SENTIO_DISABLE_AGENT_TYPES", "") + grpcEnableCompress = envconf.LoadBool("SENTIO_GRPC_ENABLE_COMPRESS", false) +) diff --git a/driver/controller/standard/convert.go b/driver/controller/standard/convert.go new file mode 100644 index 0000000..731fea7 --- /dev/null +++ b/driver/controller/standard/convert.go @@ -0,0 +1,140 @@ +package standard + +import ( + "bytes" + "encoding/json" + "sort" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" +) + +func (b *task) ConvertTimeSeriesData(data []*protos.TimeseriesResult) ( + []timeseries.Dataset, + *controller.ExternalError, +) { + dss, err := timeseries.Convert( + b.chainID, + b.GetBlockNumber(), + b.GetBlockHash(), + b.GetBlockTime(), + b.metricConfigs, + data) + if err == nil { + return dss, nil + } + if errors.Is(err, timeseries.ErrInvalidMetaDiff) { + return nil, controller.NewExternalError(controller.ErrCodeTimeSeriesDataSchemaChanged, err) + } + if errors.Is(err, timeseries.ErrInvalidMeta) { + return nil, controller.NewExternalError(controller.ErrCodeInvalidTimeSeriesData, err) + } + // unreachable + panic(err) +} + +func (b *task) ConvertExportData(r []*protos.ExportResult) []controller.WebhookMessage { + result := make([]controller.WebhookMessage, len(r)) + for i, er := range r { + result[i] = controller.WebhookMessage{ + Name: er.GetMetadata().GetName(), + BlockTime: b.GetBlockTime(), + Channel: b.webhookChannels[er.GetMetadata().GetName()], + Payload: er.GetPayload(), + } + } + return result +} + +func ConvertTemplateInstance(r []*protos.TemplateInstance, remove bool) []controller.TemplateInstance { + return utils.MapSliceNoError(r, func(t *protos.TemplateInstance) controller.TemplateInstance { + var labels []byte + if len(t.GetBaseLabels().GetFields()) > 0 { + if raw, err := protojson.Marshal(t.GetBaseLabels()); err == nil { + var buf bytes.Buffer + if json.Compact(&buf, raw) == nil { + labels = buf.Bytes() + } + } + } + return controller.TemplateInstance{ + TemplateID: t.GetTemplateId(), + TemplateName: t.GetContract().GetName(), + Address: t.GetContract().GetAddress(), + BlockRange: controller.BlockRange{ + StartBlock: t.GetStartBlock(), + EndBlock: utils.Select(t.GetEndBlock() == 0, nil, utils.WrapPointer(t.GetEndBlock())), + }, + Labels: string(labels), + Removed: remove, + } + }) +} + +func ConvertTemplateInstanceBack( + chainID string, + templates map[uint64][]controller.TemplateInstance, +) []*protos.TemplateInstance { + dict := make(map[string][]controller.TemplateInstance) + for _, bn := range utils.GetOrderedMapKeys(templates) { + for _, tpl := range templates[bn] { + dict[tpl.UniqID()] = append(dict[tpl.UniqID()], tpl) + } + } + var result []controller.TemplateInstance + for _, tpls := range dict { + on := controller.EmptyBlockRangeSet + for _, tpl := range tpls { + if tpl.Removed { + on = on.Remove(tpl.BlockRange) + } else { + on = on.Union(tpl.BlockRange) + } + } + if on.IsEmpty() { + continue + } + result = append(result, controller.TemplateInstance{ + TemplateID: tpls[0].TemplateID, + TemplateName: tpls[0].TemplateName, + Address: tpls[0].Address, + Labels: tpls[0].Labels, + BlockRange: on.Last(), + }) + } + // sort by (StartBlock, EndBlock, TemplateID, Address, Labels) ASC, result always stable + sort.Slice(result, func(i, j int) bool { + if result[i].BlockRange.StartBlock != result[j].BlockRange.StartBlock { + return result[i].BlockRange.StartBlock < result[j].BlockRange.StartBlock + } + if controller.EqualNilAsInf(result[i].BlockRange.EndBlock, result[j].BlockRange.EndBlock) { + return controller.LessNilAsInf(result[i].BlockRange.EndBlock, result[j].BlockRange.EndBlock) + } + if result[i].TemplateID != result[j].TemplateID { + return result[i].TemplateID < result[j].TemplateID + } + if result[i].Address != result[j].Address { + return result[i].Address < result[j].Address + } + return result[i].Labels < result[j].Labels + }) + return utils.MapSliceNoError(result, func(t controller.TemplateInstance) *protos.TemplateInstance { + var labels *structpb.Struct + if t.Labels != "" { + _ = json.Unmarshal([]byte(t.Labels), &labels) + } + return &protos.TemplateInstance{ + Contract: &protos.ContractInfo{Name: t.TemplateName, ChainId: chainID, Address: t.Address}, + StartBlock: t.StartBlock, + EndBlock: t.EndOrZero(), + TemplateId: t.TemplateID, + BaseLabels: labels, + } + }) +} diff --git a/driver/controller/standard/convert_test.go b/driver/controller/standard/convert_test.go new file mode 100644 index 0000000..bff1fac --- /dev/null +++ b/driver/controller/standard/convert_test.go @@ -0,0 +1,265 @@ +package standard + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/processor/protos" +) + +func indexOf(s, substr string) int { return strings.Index(s, substr) } + +func Test_ConvertTemplateInstance(t *testing.T) { + hundred := uint64(100) + + // basic conversion without labels + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + TemplateName: "MyContract", + Address: "0x1111", + Labels: "", + Removed: false, + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Name: "MyContract", Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + }, + }, false)) + + // with EndBlock + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + EndBlock: 100, + }, + }, false)) + + // with remove=true + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + Removed: true, + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + }, + }, true)) + + // with BaseLabels — should be marshaled to JSON string + labels, err := structpb.NewStruct(map[string]any{"pid": "abc", "foo": "bar"}) + assert.NoError(t, err) + result := ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + BaseLabels: labels, + }, + }, false) + assert.Len(t, result, 1) + assert.Equal(t, `{"foo":"bar","pid":"abc"}`, result[0].Labels) + + // empty BaseLabels — Labels should be empty string + emptyLabels, err := structpb.NewStruct(map[string]any{}) + assert.NoError(t, err) + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + Labels: "", + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + BaseLabels: emptyLabels, + }, + }, false)) +} + +// Test_ConvertTemplateInstance_LabelsMarshalStability verifies that BaseLabels with multiple +// keys always marshals to the same JSON string (keys sorted alphabetically by protojson), +// so the result can be used as a stable identity key in UniqID. +func Test_ConvertTemplateInstance_LabelsMarshalStability(t *testing.T) { + // Build two Struct values with identical content but keys inserted in different orders. + labelsABC, err := structpb.NewStruct(map[string]any{"a": "1", "b": "2", "c": "3"}) + assert.NoError(t, err) + labelsCBA, err := structpb.NewStruct(map[string]any{"c": "3", "b": "2", "a": "1"}) + assert.NoError(t, err) + + convert := func(s *structpb.Struct) string { + result := ConvertTemplateInstance([]*protos.TemplateInstance{ + {Contract: &protos.ContractInfo{Address: "0x1"}, TemplateId: 1, BaseLabels: s}, + }, false) + return result[0].Labels + } + + gotABC := convert(labelsABC) + gotCBA := convert(labelsCBA) + + // Both must produce identical JSON regardless of insertion order. + assert.Equal(t, gotABC, gotCBA) + + // No unnecessary whitespace — compact JSON only. + assert.NotContains(t, gotABC, " ") + + // Keys must be sorted alphabetically (protojson guarantee): "a" before "b" before "c". + assert.Contains(t, gotABC, `"a"`) + assert.Contains(t, gotABC, `"b"`) + assert.Contains(t, gotABC, `"c"`) + assert.Less(t, indexOf(gotABC, `"a"`), indexOf(gotABC, `"b"`)) + assert.Less(t, indexOf(gotABC, `"b"`), indexOf(gotABC, `"c"`)) + + // Run 20 more times to rule out map-iteration luck. + for range 20 { + assert.Equal(t, gotABC, convert(labelsABC)) + assert.Equal(t, gotABC, convert(labelsCBA)) + } +} + +func Test_ConvertTemplateInstanceBack(t *testing.T) { + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10}, + }}, + })) + + hundred := uint64(100) + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 100, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 19, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + 15: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 15}, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 30, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + 15: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 30}, + }}, + })) + + // BaseLabels round-trip: use proto.Equal to compare protobuf messages correctly. + expectedLabels, err := structpb.NewStruct(map[string]any{"pid": "abc"}) + assert.NoError(t, err) + gotBack := ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + Labels: `{"pid":"abc"}`, + BlockRange: controller.BlockRange{StartBlock: 10}, + }}, + }) + assert.Len(t, gotBack, 1) + assert.Equal(t, int32(1), gotBack[0].TemplateId) + assert.Equal(t, "0x1111", gotBack[0].Contract.GetAddress()) + assert.Equal(t, uint64(10), gotBack[0].StartBlock) + assert.True(t, proto.Equal(expectedLabels, gotBack[0].BaseLabels)) +} diff --git a/driver/controller/standard/evm/BUILD.bazel b/driver/controller/standard/evm/BUILD.bazel new file mode 100644 index 0000000..f8b9b53 --- /dev/null +++ b/driver/controller/standard/evm/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "evm", + srcs = [ + "block_data.go", + "handler.go", + "handler_interval.go", + "handler_log.go", + "handler_trace.go", + "handler_transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/evm", + visibility = ["//visibility:public"], + deps = [ + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/evm", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/evm/block_data.go b/driver/controller/standard/evm/block_data.go new file mode 100644 index 0000000..41ef49d --- /dev/null +++ b/driver/controller/standard/evm/block_data.go @@ -0,0 +1,93 @@ +package evm + +import ( + "encoding/json" + + "sentioxyz/sentio-core/driver/controller" + evmExtend "sentioxyz/sentio-core/driver/controller/data/evm" +) + +type BlockData struct { + evmExtend.BlockHeader + + mainData evmExtend.BlockMainData + extendData evmExtend.BlockExtendData + + headerJSON string + txnJSON map[string]string + receiptJSON map[string]string + receiptWithLogsJSON map[string]string + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getHeaderJSON() string { + if b.headerJSON == "" { + b.headerJSON = string(b.BlockHeader.Raw) + } + return b.headerJSON +} + +func (b *BlockData) getTransactionJSON(txHash string) string { + if b.txnJSON == nil { + b.txnJSON = make(map[string]string) + } + if r, has := b.txnJSON[txHash]; has { + return r + } + if tx, has := b.extendData.Transactions[txHash]; !has { + return "" + } else { + r, _ := json.Marshal(tx) + b.txnJSON[txHash] = string(r) + return b.txnJSON[txHash] + } +} + +func (b *BlockData) getReceiptJSON(txHash string, withLogs bool) string { + var cache map[string]string + if withLogs { + if b.receiptWithLogsJSON == nil { + b.receiptWithLogsJSON = make(map[string]string) + } + cache = b.receiptWithLogsJSON + } else { + if b.receiptJSON == nil { + b.receiptJSON = make(map[string]string) + } + cache = b.receiptJSON + } + if pb, has := cache[txHash]; has { + return pb + } + if receipt, has := b.extendData.Receipts[txHash]; !has { + return "" + } else { + if !withLogs { + receipt.Logs = nil + } + r, _ := json.Marshal(receipt) + cache[txHash] = string(r) + return cache[txHash] + } +} diff --git a/driver/controller/standard/evm/handler.go b/driver/controller/standard/evm/handler.go new file mode 100644 index 0000000..a357d50 --- /dev/null +++ b/driver/controller/standard/evm/handler.go @@ -0,0 +1,313 @@ +package evm + +import ( + "context" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type EvmHandlerAgent interface { + standard.HandlerAgent[*BlockData] + + GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) +} + +type HandlerController struct { + *standard.BaseHandlerController[evm.Client, *BlockData, EvmHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client evm.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[evm.Client, *BlockData, EvmHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("EVM::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + evm.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from evm.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + _, logger := log.FromContext(ctx) + logger.Debugf("will build block data in block #%d with %d logs %d traces %d intervals in main data", + blockNumber, len(from.Logs), len(from.Traces), len(from.Intervals)) + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.BlockHeader, err = c.Client.GetHeader(ctx, blockNumber); err != nil { + return nil, false, err + } + // check block hash of main data with the header got above + for _, l := range from.Logs { + if l.BlockHash.String() != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the log %s, expected is %s", + l.BlockHash.String(), controller.GetBlockSummary(result.BlockHeader))) + } + } + for _, t := range from.Traces { + if t.BlockHash != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the trace %s, expected is %s", + t.BlockHash, controller.GetBlockSummary(result.BlockHeader))) + } + } + // take the main data and ask the handler controller what extend data is needed + var r evm.BlockExtendRequirement + if r, err = c.getBlockExtendRequirements(ctx, &result); err != nil { + return nil, false, err + } + // actually get the extended data + if result.extendData, err = c.Client.GetBlock(ctx, blockNumber, r); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + logger.Debugf("built %d task in block #%d with handlerIDs %v", + len(result.taskList), + blockNumber, + utils.Stat(utils.MapSliceNoError( + utils.MapSliceNoError(result.taskList, controller.Task.GetHandlerID), + controller.HandlerID.String, + ))) + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetContractStartBlock(ctx, address, start, latest) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, accountConfig := range c.Config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + dataSource := standard.BuildDataSource("EVM", c.ChainConfig.ChainID, "Account", accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + for _, logConfig := range accountConfig.GetLogConfigs() { + agent := HandlerAgentLog{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", logConfig, blockRange), + Client: c.Client, + FetchConfig: logConfig.GetFetchConfig(), + } + agent.Filters, err = NewLogFilters(logConfig.GetFilters(), true, "") + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + var hasTxnHandler bool + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("EVM", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + if blockRange.StartBlock, err = c.getAddressStart(ctx, contractAddress, blockRange.StartBlock, latest); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, logConfig := range contractConfig.GetLogConfigs() { + agent := HandlerAgentLog{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", logConfig, blockRange), + Client: c.Client, + FetchConfig: logConfig.GetFetchConfig(), + } + agent.Filters, err = NewLogFilters(logConfig.GetFilters(), false, contractAddress) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, traceConfig := range contractConfig.GetTraceConfigs() { + agent := HandlerAgentTrace{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "trace", traceConfig, blockRange), + FetchConfig: traceConfig.GetFetchConfig(), + } + if contractAddress != "" { + agent.Filter.Address = []string{strings.ToLower(contractAddress)} + } + if traceConfig.GetSignature() != "" { + agent.Filter.Signature = []string{traceConfig.GetSignature()} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + if len(contractConfig.GetTransactionConfig()) > 0 { + if contractAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("transaction handler only support global processor")) + } + if len(contractConfig.GetTransactionConfig()) > 1 || hasTxnHandler { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("there can only be one transaction handler")) + } + txnConfig := contractConfig.GetTransactionConfig()[0] + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "transaction", txnConfig, blockRange), + FetchConfig: txnConfig.GetFetchConfig(), + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + hasTxnHandler = true + } + + for _, intervalConfig := range contractConfig.GetIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + FetchConfig: intervalConfig.GetFetchConfig(), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) getDataRequirement() (dr evm.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentTransaction: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: data.IntervalConfig{ + BlockInterval: &data.BlockInterval{Backfill: 1, Watching: 1}, + }, + BlockRange: ag.Range, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + case HandlerAgentTrace: + dr.Trace = append(dr.Trace, evm.TraceRequirement{ + TraceFilter: ag.Filter, + BlockRange: ag.Range, + }) + case HandlerAgentLog: + dr.Log = append(dr.Log, evm.LogRequirement{ + LogFilter: utils.Reduce(ag.Filters, evm.LogFilter.Merge), + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) getBlockExtendRequirements( + ctx context.Context, + blockData *BlockData, +) (req evm.BlockExtendRequirement, err error) { + var ar evm.BlockExtendRequirement + for _, agent := range c.Agents { + if ar, err = agent.GetExtendRequirements(ctx, blockData); err != nil { + return + } + req.Merge(ar) + } + return +} diff --git a/driver/controller/standard/evm/handler_interval.go b/driver/controller/standard/evm/handler_interval.go new file mode 100644 index 0000000..e782818 --- /dev/null +++ b/driver/controller/standard/evm/handler_interval.go @@ -0,0 +1,132 @@ +package evm + +import ( + "context" + "encoding/json" + "math" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + if !data.ContainsInterval(d.mainData.Intervals, a.IntervalConfig) { + return r, nil + } + if a.FetchConfig.GetTransaction() { + r.AllTransactions = true + } + if a.FetchConfig.GetTransactionReceipt() { + r.AllTransactionReceipts = true + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.AllTransactionReceiptLogs = true + } + if a.FetchConfig.GetTrace() { + r.AllTraces = true + } + return r, nil +} + +func (a HandlerAgentInterval) BuildBindingDataList( + _ context.Context, + d *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(d.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + rawBlock := d.getHeaderJSON() + if a.FetchConfig.GetTransaction() || a.FetchConfig.GetTransactionReceipt() || a.FetchConfig.GetTrace() { + // Splice transactions / receipts / traces (each already raw JSON) into the header + // object to build the composite raw_block. Header fields are kept as raw bytes. + var block map[string]json.RawMessage + if err := json.Unmarshal(d.BlockHeader.Raw, &block); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal block header %d", d.GetBlockNumber()) + } + if a.FetchConfig.GetTransaction() { + txns := make([]json.RawMessage, 0, len(d.BlockHeader.TxHashes)) + for _, txHash := range d.BlockHeader.TxHashes { + txns = append(txns, json.RawMessage(d.getTransactionJSON(txHash))) + } + raw, err := json.Marshal(txns) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal transactions for block %d", d.GetBlockNumber()) + } + block["transactions"] = raw + } + if a.FetchConfig.GetTransactionReceipt() { + receipts := make([]json.RawMessage, 0, len(d.BlockHeader.TxHashes)) + for _, txHash := range d.BlockHeader.TxHashes { + receipts = append(receipts, json.RawMessage(d.getReceiptJSON(txHash, a.FetchConfig.GetTransactionReceiptLogs()))) + } + raw, err := json.Marshal(receipts) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal transaction receipts for block %d", d.GetBlockNumber()) + } + block["transactionReceipts"] = raw + } + if a.FetchConfig.GetTrace() { + var traces []json.RawMessage + for _, txHash := range d.BlockHeader.TxHashes { + for _, trace := range d.extendData.Traces[txHash] { + traces = append(traces, json.RawMessage(trace.Raw)) + } + } + raw, err := json.Marshal(traces) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal traces for block %d", d.GetBlockNumber()) + } + block["traces"] = raw + } + raw, err := json.Marshal(block) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal block %d", d.GetBlockNumber()) + } + rawBlock = string(raw) + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_ETH_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_EthBlock_{ + EthBlock: &protos.Data_EthBlock{ + RawBlock: rawBlock, + }, + }, + }, + DataSize: len(rawBlock), + }}, nil +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/evm/handler_log.go b/driver/controller/standard/evm/handler_log.go new file mode 100644 index 0000000..24bdf25 --- /dev/null +++ b/driver/controller/standard/evm/handler_log.go @@ -0,0 +1,158 @@ +package evm + +import ( + "context" + "encoding/json" + "strings" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentLog struct { + controller.BaseHandlerAgent + + Client evm.Client `json:"-"` // used to check address is a ERC20 address + + FetchConfig *protos.EthFetchConfig + Filters []evm.LogFilter // linked by OR +} + +func (a HandlerAgentLog) GetExtendRequirements( + ctx context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + + logs, err := evm.FilterLogs(ctx, a.Client, d.mainData.Logs, a.Filters...) + if err != nil { + return r, err + } + txnSet := make(map[string]bool) + for _, log := range logs { + txnSet[log.TxHash.String()] = true + } + for txnHash := range txnSet { + if a.FetchConfig.GetTransaction() { + r.SpecialTransactions = append(r.SpecialTransactions, txnHash) + } + if a.FetchConfig.GetTransactionReceipt() { + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, txnHash) + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, txnHash) + } + } + return r, nil +} + +func (a HandlerAgentLog) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + var logs []types.Log + logs, err = evm.FilterLogs(ctx, a.Client, d.mainData.Logs, a.Filters...) + if err != nil { + return nil, err + } + for _, log := range logs { + var rawLog string + var raw []byte + if raw, err = json.Marshal(&log); err != nil { + return nil, err + } else { + rawLog = string(raw) + } + size := len(rawLog) + var rawBlock *string + if a.FetchConfig.GetBlock() { + rawBlock = new(d.getHeaderJSON()) + size += len(*rawBlock) + } + var rawTransaction *string + if a.FetchConfig.GetTransaction() { + rawTransaction = new(d.getTransactionJSON(log.TxHash.String())) + size += len(*rawTransaction) + } + var rawReceipt *string + if a.FetchConfig.GetTransactionReceipt() { + rawReceipt = new(d.getReceiptJSON(log.TxHash.String(), a.FetchConfig.GetTransactionReceiptLogs())) + size += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_LOG, + TxIndex: int(log.TxIndex), + TxInnerIndex: int(log.Index), + Data: &protos.Data{ + Value: &protos.Data_EthLog_{ + EthLog: &protos.Data_EthLog{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawLog: rawLog, + RawBlock: rawBlock, + RawTransaction: rawTransaction, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: size, + } + r = append(r, data) + } + return +} + +func (a HandlerAgentLog) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + "FetchConfig": a.FetchConfig, + } +} + +func NewLogFilters(filters []*protos.LogFilter, accountLogFilter bool, contractAddress string) ([]evm.LogFilter, error) { + if len(filters) == 0 { + return nil, errors.Errorf("filters is empty") + } + result := make([]evm.LogFilter, len(filters)) + for i, filter := range filters { + result[i] = evm.LogFilter{ + Topics: utils.MapSliceNoError(filter.Topics, func(t *protos.Topic) []string { + return t.GetHashes() + }), + } + if accountLogFilter { + switch v := filter.AddressOrType.(type) { + case *protos.LogFilter_AddressType: + at := v.AddressType + if at == protos.AddressType_ERC20 { + result[i].AddressShouldBeERC20 = true + } else { + return nil, errors.Errorf("unsupported address type %s", at.String()) + } + case *protos.LogFilter_Address: + result[i].Address = []string{strings.ToLower(v.Address)} + default: + return nil, errors.Errorf("unsupported address type %T", v) + } + } else if contractAddress != "" { + result[i].Address = []string{strings.ToLower(contractAddress)} + } + } + return result, nil +} diff --git a/driver/controller/standard/evm/handler_trace.go b/driver/controller/standard/evm/handler_trace.go new file mode 100644 index 0000000..701c2ec --- /dev/null +++ b/driver/controller/standard/evm/handler_trace.go @@ -0,0 +1,114 @@ +package evm + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentTrace struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig + Filter evm.TraceFilter +} + +func (a HandlerAgentTrace) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + txnSet := make(map[string]bool) + for _, trace := range d.mainData.Traces { + if a.Filter.Check(trace) { + txnSet[trace.TransactionHash] = true + } + } + for txnHash := range txnSet { + if a.FetchConfig.GetTransaction() { + r.SpecialTransactions = append(r.SpecialTransactions, txnHash) + } + if a.FetchConfig.GetTransactionReceipt() { + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, txnHash) + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, txnHash) + } + } + return r, nil +} + +func (a HandlerAgentTrace) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + for _, trace := range d.mainData.Traces { + if !a.Filter.Check(trace) { + continue + } + rawTrace := string(trace.Raw) + var rawBlock, rawTransaction, rawReceipt *string + if a.FetchConfig.GetBlock() { + s := d.getHeaderJSON() + rawBlock = &s + } + if a.FetchConfig.GetTransaction() { + s := d.getTransactionJSON(trace.TransactionHash) + rawTransaction = &s + } + if a.FetchConfig.GetTransactionReceipt() { + s := d.getReceiptJSON(trace.TransactionHash, a.FetchConfig.GetTransactionReceiptLogs()) + rawReceipt = &s + } + dataSize := len(rawTrace) + if rawBlock != nil { + dataSize += len(*rawBlock) + } + if rawTransaction != nil { + dataSize += len(*rawTransaction) + } + if rawReceipt != nil { + dataSize += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_TRACE, + TxIndex: int(trace.TransactionIndex), + Data: &protos.Data{ + Value: &protos.Data_EthTrace_{ + EthTrace: &protos.Data_EthTrace{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawTrace: rawTrace, + RawBlock: rawBlock, + RawTransaction: rawTransaction, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: dataSize, + } + r = append(r, data) + } + return +} + +func (a HandlerAgentTrace) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/evm/handler_transaction.go b/driver/controller/standard/evm/handler_transaction.go new file mode 100644 index 0000000..746acf8 --- /dev/null +++ b/driver/controller/standard/evm/handler_transaction.go @@ -0,0 +1,71 @@ +package evm + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentTransaction struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig +} + +func (a HandlerAgentTransaction) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (r evm.BlockExtendRequirement, err error) { + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + r.AllTransactions = true + r.AllTransactionReceipts = a.FetchConfig.GetTransactionReceipt() + r.AllTransactionReceiptLogs = a.FetchConfig.GetTransactionReceiptLogs() + return r, nil +} + +func (a HandlerAgentTransaction) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + for txIndex, txHash := range d.BlockHeader.TxHashes { + rawTransaction := d.getTransactionJSON(txHash) + rawBlock := new(d.getHeaderJSON()) + size := len(rawTransaction) + len(*rawBlock) + var rawReceipt *string + if a.FetchConfig.GetTransactionReceipt() { + rawReceipt = new(d.getReceiptJSON(txHash, a.FetchConfig.GetTransactionReceiptLogs())) + size += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_TRANSACTION, + TxIndex: txIndex, + Data: &protos.Data{ + Value: &protos.Data_EthTransaction_{ + EthTransaction: &protos.Data_EthTransaction{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawTransaction: rawTransaction, + RawBlock: rawBlock, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: size, + } + r = append(r, data) + } + return +} +func (a HandlerAgentTransaction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/fuel/BUILD.bazel b/driver/controller/standard/fuel/BUILD.bazel new file mode 100644 index 0000000..ebf84af --- /dev/null +++ b/driver/controller/standard/fuel/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fuel", + srcs = [ + "block_data.go", + "handler.go", + "handler_interval.go", + "handler_receipt.go", + "handler_transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/fuel", + visibility = ["//visibility:public"], + deps = [ + "//chain/fuel", + "//common/log", + "//common/protojson", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/fuel", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_fuel_go//types", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/fuel/block_data.go b/driver/controller/standard/fuel/block_data.go new file mode 100644 index 0000000..f1a453a --- /dev/null +++ b/driver/controller/standard/fuel/block_data.go @@ -0,0 +1,78 @@ +package fuel + +import ( + "encoding/json" + "reflect" + + "github.com/pkg/errors" + "github.com/sentioxyz/fuel-go/types" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/fuel" +) + +type BlockData struct { + fuel.Block + + mainData fuel.BlockMainData + + blockPb *structpb.Struct + blockPbSize int + transactionPb []*structpb.Struct + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} + +func (d *BlockData) getBlockPb() (*structpb.Struct, int, error) { + if d.blockPb == nil { + blockPb := new(structpb.Struct) + j, err := json.Marshal(d.Block) + if err != nil { + return nil, 0, errors.Wrapf(err, "marshal header of block %d failed", d.GetBlockNumber()) + } + err = protojson.Unmarshal(j, blockPb) + if err != nil { + return nil, 0, errors.Wrapf(err, "build structpb of block %d failed", d.GetBlockNumber()) + } + d.blockPb, d.blockPbSize = blockPb, len(j) + } + return d.blockPb, d.blockPbSize, nil +} + +var fuelTxTyp = reflect.TypeOf(types.Transaction{}) + +func (d *BlockData) getTxPb(i int) *structpb.Struct { + if i >= len(d.mainData.Txs) { + panic(errors.Errorf("index %d out of range [0,%d) in BlockData #%d", i, len(d.mainData.Txs), d.GetBlockNumber())) + } + if len(d.transactionPb) == 0 { + d.transactionPb = make([]*structpb.Struct, len(d.mainData.Txs)) + } + if d.transactionPb[i] == nil { + d.transactionPb[i] = utils.ConvertToStructpb(&d.mainData.Txs[i].Transaction, fuelTxTyp) + } + return d.transactionPb[i] +} diff --git a/driver/controller/standard/fuel/handler.go b/driver/controller/standard/fuel/handler.go new file mode 100644 index 0000000..d4faee7 --- /dev/null +++ b/driver/controller/standard/fuel/handler.go @@ -0,0 +1,277 @@ +package fuel + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + chainFuel "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/fuel" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type FuelHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[fuel.Client, *BlockData, FuelHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client fuel.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[fuel.Client, *BlockData, FuelHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.BaseHandlerController.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetContractCreateBlockHeight(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("FUEL", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + if blockRange.StartBlock, err = c.getAddressStart(ctx, contractAddress, blockRange.StartBlock, latest); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + // interval + for _, intervalConfig := range contractConfig.IntervalConfigs { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // asset transfer + for _, assetConfig := range contractConfig.AssetConfigs { + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "transfer", assetConfig, blockRange), + Filters: make([]chainFuel.TransactionFilter, len(assetConfig.Filters)), + } + if len(assetConfig.Filters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter for handler %s", agent.GetHandlerID().String())) + } + for i, filterConfig := range assetConfig.Filters { + agent.Filters[i] = chainFuel.TransactionFilter{ + TransferFilter: &chainFuel.TransferFilter{ + AssetID: strings.ToLower(filterConfig.GetAssetId()), + From: strings.ToLower(filterConfig.GetFromAddress()), + To: strings.ToLower(filterConfig.GetToAddress()), + }, + ExcludeFailed: true, + } + if contractAddress != "" { + agent.Filters[i].CallFilter = &chainFuel.CallFilter{ + ContractID: contractAddress, + } + } + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // all transactions + for _, txConfig := range contractConfig.FuelTransactionConfigs { + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "transaction", txConfig, blockRange), + Filters: []chainFuel.TransactionFilter{{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + ExcludeFailed: true, + }}, + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // receipt + for _, receiptConfig := range contractConfig.FuelReceiptConfigs { + if receiptConfig.GetLog() != nil { + // log + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", receiptConfig, blockRange), + Filters: make([]chainFuel.TransactionFilter, len(receiptConfig.GetLog().GetLogIds())), + } + for i, logID := range receiptConfig.GetLog().GetLogIds() { + rb, parseErr := strconv.ParseUint(logID, 0, 64) + if parseErr != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Wrapf(parseErr, + "parse logId %q to uint failed for handler %s", logID, agent.GetHandlerID().String())) + } + agent.Filters[i] = chainFuel.TransactionFilter{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + LogFilter: &chainFuel.LogFilter{ + ContractID: contractAddress, + LogRb: &rb, + }, + ExcludeFailed: true, + } + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + if receiptConfig.GetTransfer() != nil { + // receipt transfer + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "receiptTransfer", receiptConfig, blockRange), + Filters: []chainFuel.TransactionFilter{{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + ReceiptTransferFilter: &chainFuel.ReceiptTransferFilter{ + AssetID: receiptConfig.GetTransfer().GetAssetId(), + From: receiptConfig.GetTransfer().GetFrom(), + To: receiptConfig.GetTransfer().GetTo(), + }, + ExcludeFailed: true, + }}, + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + } + return nil +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("FUEL::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + fuel.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*3, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from fuel.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.Block, err = c.Client.GetBlock(ctx, blockNumber); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getDataRequirement() (dr fuel.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentTransaction: + dr.Tx = append(dr.Tx, fuel.TransactionRequirement{ + Filters: ag.Filters, + BlockRange: ag.Range, + }) + case HandlerAgentReceipt: + dr.Tx = append(dr.Tx, fuel.TransactionRequirement{ + Filters: ag.Filters, + BlockRange: ag.Range, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} diff --git a/driver/controller/standard/fuel/handler_interval.go b/driver/controller/standard/fuel/handler_interval.go new file mode 100644 index 0000000..33e3cad --- /dev/null +++ b/driver/controller/standard/fuel/handler_interval.go @@ -0,0 +1,53 @@ +package fuel + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + } +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + blockPb, size, err := bd.getBlockPb() + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_FUEL_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_FuelBlock_{ + FuelBlock: &protos.Data_FuelBlock{ + Block: blockPb, + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: size, + }}, nil +} diff --git a/driver/controller/standard/fuel/handler_receipt.go b/driver/controller/standard/fuel/handler_receipt.go new file mode 100644 index 0000000..e62ad45 --- /dev/null +++ b/driver/controller/standard/fuel/handler_receipt.go @@ -0,0 +1,73 @@ +package fuel + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentReceipt struct { + controller.BaseHandlerAgent + + Filters []fuel.TransactionFilter +} + +func (a HandlerAgentReceipt) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + } +} + +func (a HandlerAgentReceipt) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for i, tx := range bd.mainData.Txs { + ok := utils.HasAny(a.Filters, func(filter fuel.TransactionFilter) bool { + return filter.Check(tx.Transaction) + }) + if !ok { + continue + } + receiptIndexes := make(map[int]bool) + receipts := fuel.GetTxnReceipt(tx.Status) + for _, filter := range a.Filters { + if filter.LogFilter != nil { + for _, receiptIndex := range filter.LogFilter.Check(receipts) { + receiptIndexes[receiptIndex] = true + } + } + if filter.ReceiptTransferFilter != nil { + for _, receiptIndex := range filter.ReceiptTransferFilter.Check(receipts) { + receiptIndexes[receiptIndex] = true + } + } + } + for _, receiptIndex := range utils.GetOrderedMapKeys(receiptIndexes) { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_FUEL_RECEIPT, + TxIndex: int(tx.TransactionIndex), + TxInnerIndex: receiptIndex, + Data: &protos.Data{ + Value: &protos.Data_FuelLog{ + FuelLog: &protos.Data_FuelReceipt{ + Transaction: bd.getTxPb(i), + ReceiptIndex: int64(receiptIndex), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: 1000, + }) + } + } + return result, nil +} diff --git a/driver/controller/standard/fuel/handler_transaction.go b/driver/controller/standard/fuel/handler_transaction.go new file mode 100644 index 0000000..0bf319c --- /dev/null +++ b/driver/controller/standard/fuel/handler_transaction.go @@ -0,0 +1,55 @@ +package fuel + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentTransaction struct { + controller.BaseHandlerAgent + + Filters []fuel.TransactionFilter +} + +func (a HandlerAgentTransaction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + } +} + +func (a HandlerAgentTransaction) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for i, tx := range bd.mainData.Txs { + ok := utils.HasAny(a.Filters, func(filter fuel.TransactionFilter) bool { + return filter.Check(tx.Transaction) + }) + if !ok { + continue + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_FUEL_TRANSACTION, + TxIndex: int(tx.TransactionIndex), + Data: &protos.Data{ + Value: &protos.Data_FuelTransaction_{ + FuelTransaction: &protos.Data_FuelTransaction{ + Transaction: bd.getTxPb(i), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: 1000, + }) + } + return result, nil +} diff --git a/driver/controller/standard/handler.go b/driver/controller/standard/handler.go new file mode 100644 index 0000000..ef4987a --- /dev/null +++ b/driver/controller/standard/handler.go @@ -0,0 +1,335 @@ +package standard + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" +) + +type BindingDataInner struct { + Data *protos.Data + DataSize int + HandlerType protos.HandlerType + TxIndex int + TxInnerIndex int +} + +type HandlerAgent[BKD controller.BlockHeader] interface { + controller.HandlerAgent + + BuildBindingDataList(ctx context.Context, bd BKD) ([]BindingDataInner, error) +} + +type streamPool chan protos.ProcessorV3_ProcessBindingsStreamClient + +type HandlerConfig struct { + ContractConfigs []*protos.ContractConfig + AccountConfigs []*protos.AccountConfig +} + +func (c HandlerConfig) String() string { + return utils.MustJSONMarshal(c) +} + +type BaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]] struct { + Processor *models.Processor + InitResult *protos.InitResponse + ChainConfig *chain.ConfigV2 + Client CLI + + Config HandlerConfig // result of SetTemplates + Agents []HA // built from Config + + metricConfigs map[timeseries.MetaType]map[string]*protos.MetricConfig // load from InitResult + webhookChannels map[string]string // load from InitResult + + addressStart map[string]uint64 + addressStartData string + + processorClients []protos.ProcessorV3Client + processStreams streamPool + waiter *waiter +} + +func NewBaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]]( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client CLI, + processorClients []protos.ProcessorV3Client, +) *BaseHandlerController[CLI, BKD, HA] { + webhookChannels := make(map[string]string) + for _, wc := range initResult.GetExportConfigs() { + webhookChannels[wc.GetName()] = wc.GetChannel() + } + return &BaseHandlerController[CLI, BKD, HA]{ + Processor: processor, + InitResult: initResult, + ChainConfig: chainConfig, + Client: client, + metricConfigs: timeseries.BuildMetricConfigs(initResult.GetMetricConfigs()), + webhookChannels: webhookChannels, + processorClients: processorClients, + } +} + +func (c *BaseHandlerController[CLI, BKD, HA]) SetTemplates( + ctx context.Context, + templates map[uint64][]controller.TemplateInstance, +) *controller.ExternalError { + _, logger := log.FromContext(ctx) + req := &protos.UpdateTemplatesRequest{ + ChainId: c.ChainConfig.ChainID, + TemplateInstances: ConvertTemplateInstanceBack(c.ChainConfig.ChainID, templates), + } + + var confText string + for i, cli := range c.processorClients { + // update templates + if _, err := cli.UpdateTemplates(ctx, req); err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("update templates failed: %v", err)) + } + // get handler config + var config HandlerConfig + if resp, err := cli.GetConfig(ctx, &protos.ProcessConfigRequest{}); err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("get handler config failed: %v", err)) + } else { + config = HandlerConfig{ + ContractConfigs: utils.FilterArr(resp.GetContractConfigs(), func(cc *protos.ContractConfig) bool { + return cc.GetContract().GetChainId() == c.ChainConfig.ChainID + }), + AccountConfigs: utils.FilterArr(resp.GetAccountConfigs(), func(cc *protos.AccountConfig) bool { + return cc.GetChainId() == c.ChainConfig.ChainID + }), + } + } + // check handler config + if i == 0 { + c.Config, confText = config, config.String() + } else if another := config.String(); confText != another { + logger.Errorw("configs from different processor has diff", "config1", confText, "config2", another) + return controller.NewExternalError(controller.ErrCodeProcessorConfigsHasDiff, + errors.Errorf("configs from different processor has diff")) + } + } + + logger.Infow("got config", "config", confText) + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) PrepareExecute(ctx context.Context) *controller.ExternalError { + clientCount := uint64(len(c.processorClients)) + streamSize := max(controller.ProcessConcurrency, clientCount) + c.processStreams = make(streamPool, streamSize) + var opts = []grpc.CallOption{ + grpc.UseCompressor(utils.Select[string](grpcEnableCompress, gzip.Name, "")), + } + for i := uint64(0); i < streamSize; i++ { + stream, err := c.processorClients[i%clientCount].ProcessBindingsStream(ctx, opts...) + if err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("open stream for process binding failed: %v", err)) + } + c.processStreams <- stream + } + c.waiter = &waiter{ + ready: concurrency.NewResourceWaiter[uint64](), + finish: concurrency.NewResourceWaiter[partitionWithIndex](), + } + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) DisableAgents(ctx context.Context) { + if len(strings.TrimSpace(disableAgentTypes)) == 0 { + return + } + _, logger := log.FromContext(ctx) + das := utils.BuildSet(utils.MapSliceNoError(strings.Split(disableAgentTypes, ","), strings.TrimSpace)) + c.Agents = utils.FilterArr(c.Agents, func(a HA) bool { + typ := fmt.Sprintf("%T", a) + if das[typ] { + logger.Warnw("disable agent", "type", typ, "agent", a.Snapshot()) + return false + } + return true + }) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) FinishExecute() { + close(c.processStreams) + for stream := range c.processStreams { + _ = stream.CloseSend() + } +} + +func (c *BaseHandlerController[CLI, BKD, HA]) BuildTaskList( + ctx context.Context, + d BKD, +) ([]controller.Task, int, error) { + var result []bindingData + var totalSize int + for _, agent := range c.Agents { + if !agent.GetRange().Contains(d.GetBlockNumber()) { + continue + } + if inners, err := agent.BuildBindingDataList(ctx, d); err != nil { + return nil, 0, err + } else { + for _, inner := range inners { + result = append(result, bindingData{ + BlockHeader: d, + handlerID: agent.GetHandlerID(), + data: &protos.DataBinding{ + Data: inner.Data, + HandlerType: inner.HandlerType, + HandlerIds: []int32{agent.GetHandlerID().ID}, + ChainId: c.ChainConfig.ChainID, + }, + txIndex: inner.TxIndex, + txInnerIndex: inner.TxInnerIndex, + }) + totalSize += inner.DataSize + } + } + } + // The purpose of using stable sorting is to ensure that tasks of the same handler remain in the order + // in which they were generated. + sort.SliceStable(result, func(i, j int) bool { + return result[i].Cmp(result[j], c.InitResult.ExecutionConfig.GetHandlerOrderInsideTransaction()) < 0 + }) + var r []controller.Task + for _, bd := range result { + r = append(r, &task{ + bindingData: bd, + sp: c.processStreams, + waiter: c.waiter, + chainID: c.ChainConfig.ChainID, + processor: c.Processor, + metricConfigs: c.metricConfigs, + webhookChannels: c.webhookChannels, + }) + } + return r, totalSize, nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) BuildReportRequirements( + currentBlockNumber uint64, +) []data.IntervalRequirement { + endBlock := c.GetBlockRange().EndBlock + // In the backfill phase, at least one non-empty BlockData is generated for every DAY. + // In the watching phase, at least one non-empty BlockData is generated for every MINUTE. + reqs := []data.IntervalRequirement{{ + BlockRange: controller.BlockRange{StartBlock: currentBlockNumber, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 24, + Watching: time.Minute, + }}, + }} + if endBlock != nil { + reqs = append(reqs, data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: *endBlock, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{BlockInterval: &data.BlockInterval{ + Backfill: 1, + Watching: 1, + }}, + }) + } + return reqs +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetBlockRange() controller.BlockRange { + return controller.GetHandleAgentsBlockRange(c.Agents) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetAgentStat() map[string]int { + stat := make(map[string]int) + for _, ag := range c.Agents { + stat[fmt.Sprintf("%T", ag)] += 1 + } + return stat +} + +const checkpointDataKeyAddressStart = "AddressStart" + +func (c *BaseHandlerController[CLI, BKD, HA]) LoadAddressStart(checkpoint *controller.Checkpoint) *controller.ExternalError { + c.addressStart = make(map[string]uint64) + if checkpoint == nil || checkpoint.Data == nil { + return nil + } + raw, has := checkpoint.Data[checkpointDataKeyAddressStart] + if !has { + return nil + } + err := json.Unmarshal([]byte(raw), &c.addressStart) + if err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "load address start failed")) + } + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetAddressStart( + address string, + start uint64, + loader func() (uint64, error), +) (uint64, error) { + if c.InitResult.GetExecutionConfig().GetSkipStartBlockValidation() || + c.ChainConfig.SkipStartBlockValidation || controller.SkipStartBlockValidation { + return start, nil + } + if address == "" { + return start, nil + } + var has bool + if start, has = c.addressStart[address]; has { + return start, nil + } + var err error + start, err = loader() + if err != nil { + return 0, errors.Wrapf(err, "get start block for %s failed", address) + } + c.addressStart[address] = start + return start, nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) AddressStartReady() { + b, _ := json.Marshal(c.addressStart) + c.addressStartData = string(b) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) DumpAddressStart(checkpointData map[string]string) { + checkpointData[checkpointDataKeyAddressStart] = c.addressStartData +} + +func (c *BaseHandlerController[CLI, BKD, HA]) Snapshot() any { + return map[string]any{ + "initResult": c.InitResult, + "chainConfig": c.ChainConfig, + "handlerConfig": c.Config, + "agents": utils.MapSliceNoError(c.Agents, HA.Snapshot), + "agentStat": c.GetAgentStat(), + "addressStart": c.addressStart, + "processorCount": len(c.processorClients), + "streamCount": cap(c.processStreams), + } +} diff --git a/driver/controller/standard/helper.go b/driver/controller/standard/helper.go new file mode 100644 index 0000000..5308a54 --- /dev/null +++ b/driver/controller/standard/helper.go @@ -0,0 +1,29 @@ +package standard + +import ( + "bytes" + + "sentioxyz/sentio-core/common/utils" +) + +func AdjustAddress(address string) string { + return utils.Select(address == "" || address == "*", "", address) +} + +func AdjustEndBlock(endBlock uint64) *uint64 { + return utils.Select(endBlock == 0, nil, &endBlock) +} + +func BuildDataSource(chainType, chainID, srcType, address string) string { + var b bytes.Buffer + b.WriteString(chainType) + b.WriteString(":") + b.WriteString(chainID) + b.WriteString(":") + b.WriteString(srcType) + if address != "" { + b.WriteString(":") + b.WriteString(address) + } + return b.String() +} diff --git a/driver/controller/standard/interval.go b/driver/controller/standard/interval.go new file mode 100644 index 0000000..4f7329b --- /dev/null +++ b/driver/controller/standard/interval.go @@ -0,0 +1,34 @@ +package standard + +import ( + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/processor/protos" +) + +func NewIntervalConfig(c *protos.OnIntervalConfig) (data.IntervalConfig, error) { + if c.GetSlot() > 0 || c.GetSlotInterval() != nil { + interval := data.BlockInterval{ + Backfill: max(uint64(c.GetSlotInterval().GetBackfillInterval()), uint64(c.GetSlot()), minBackfillSlotInterval), + Watching: max(uint64(c.GetSlotInterval().GetRecentInterval()), uint64(c.GetSlot()), 1), + } + return data.IntervalConfig{BlockInterval: &interval}, nil + } else if c.GetMinutes() > 0 || c.GetMinutesInterval() != nil { + interval := data.TimeInterval{ + Backfill: max( + time.Duration(c.GetMinutesInterval().GetBackfillInterval())*time.Minute, + time.Duration(c.GetMinutes())*time.Minute, + minBackfillTimeInterval), + Watching: max( + time.Duration(c.GetMinutesInterval().GetRecentInterval())*time.Minute, + time.Duration(c.GetMinutes())*time.Minute, + time.Minute), + } + return data.IntervalConfig{TimeInterval: &interval}, nil + } else { + return data.IntervalConfig{}, errors.Errorf("invalid interval config %#v", c) + } +} diff --git a/driver/controller/standard/sol/BUILD.bazel b/driver/controller/standard/sol/BUILD.bazel new file mode 100644 index 0000000..8b18287 --- /dev/null +++ b/driver/controller/standard/sol/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "sol", + srcs = [ + "block_data.go", + "handler.go", + "handler_instruction.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sol", + visibility = ["//visibility:public"], + deps = [ + "//chain/sol", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sol", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_gagliardetto_solana_go//:solana-go", + "@com_github_gagliardetto_solana_go//rpc", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/sol/block_data.go b/driver/controller/standard/sol/block_data.go new file mode 100644 index 0000000..3a1f04f --- /dev/null +++ b/driver/controller/standard/sol/block_data.go @@ -0,0 +1,91 @@ +package sol + +import ( + "encoding/json" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/sol" +) + +// BlockData is built directly from BlockMainData: the main-data fetchers already returned the block +// header (interval) and the full matching transactions (instruction), so no extra fetch is needed. +type BlockData struct { + mainData sol.BlockMainData + + blockJSON string + txJSONDict map[solana.Signature]string // key is txSig + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetBlockNumber() uint64 { + return d.mainData.Slot +} + +func (d *BlockData) GetBlockHash() string { + return d.mainData.Blockhash +} + +func (d *BlockData) GetBlockParentHash() string { + return d.mainData.PreviousBlockhash +} + +func (d *BlockData) GetBlockTime() time.Time { + if d.mainData.BlockTime != nil { + return d.mainData.BlockTime.Time() + } + return time.Time{} +} + +func (d *BlockData) getBlockJSON() (string, error) { + if d.blockJSON != "" { + return d.blockJSON, nil + } + if d.mainData.Block == nil || d.mainData.Block.GetBlockResult == nil { + return "", errors.Errorf("block %d has no header", d.GetBlockNumber()) + } + b, err := json.Marshal(d.mainData.Block.GetBlockResult) + if err != nil { + return "", errors.Wrapf(err, "marshal block %d failed", d.GetBlockNumber()) + } + d.blockJSON = string(b) + return d.blockJSON, nil +} + +func (d *BlockData) getTxJSON(tx solcore.WrappedTransaction) (string, error) { + if r, has := d.txJSONDict[tx.Signature]; has { + return r, nil + } + b, err := json.Marshal(tx.ToParsedTransactionResult(d.GetBlockNumber(), d.mainData.BlockTime)) + if err != nil { + return "", errors.Wrapf(err, "marshal tx %d/%s failed", d.GetBlockNumber(), tx.Signature) + } + r := string(b) + d.txJSONDict[tx.Signature] = r + return r, nil +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} diff --git a/driver/controller/standard/sol/handler.go b/driver/controller/standard/sol/handler.go new file mode 100644 index 0000000..46dba1c --- /dev/null +++ b/driver/controller/standard/sol/handler.go @@ -0,0 +1,205 @@ +package sol + +import ( + "context" + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sol" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type HandlerController struct { + *standard.BaseHandlerController[sol.Client, *BlockData, standard.HandlerAgent[*BlockData]] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client sol.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[sol.Client, *BlockData, standard.HandlerAgent[*BlockData]]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("SOL::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + sol.BuildBlockMainDataFetcher(fetchNamePrefix, req, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*30, // may need to get a lot of transaction + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from sol.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + _, logger := log.FromContext(ctx) + // The main-data fetchers already returned the block header (interval) and the full + // matching transactions (instruction), so the block data is built without any fetch. + result := BlockData{ + mainData: from, + txJSONDict: make(map[solana.Signature]string), + checkpointData: make(map[string]string), + } + var err error + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + logger.Debugf("built %d task in block #%d with handlerIDs %v", + len(result.taskList), + blockNumber, + utils.Stat(utils.MapSliceNoError( + utils.MapSliceNoError(result.taskList, controller.Task.GetHandlerID), + controller.HandlerID.String, + ))) + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("SOL", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + parsedContractAddress, parseContractAddrErr := solana.PublicKeyFromBase58(contractAddress) + blockRange.StartBlock, err = c.GetAddressStart( + contractAddress, + blockRange.StartBlock, + func() (uint64, error) { + ns, has, getErr := c.Client.GetContractStartBlock(ctx, parsedContractAddress, blockRange.StartBlock, latest) + if getErr != nil { + return 0, getErr + } + if has { + return ns, nil + } + return latest + 1, nil + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + if config := contractConfig.GetInstructionConfig(); config != nil { + if parseContractAddrErr != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Wrapf( + parseContractAddrErr, "contract address %q is invalid for the instruction handler", contractAddress)) + } + agent := HandlerAgentInstruction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "instruction", controller.SimpleHandlerConfig{}, blockRange), + Address: parsedContractAddress, + ProcessInnerInstruction: config.GetInnerInstruction(), + ProcessParsedInstruction: config.GetParsedInstruction(), + ProcessRawInstruction: config.GetRawDataInstruction(), + FetchTx: config.GetFetchTx(), + } + if !agent.ProcessParsedInstruction && !agent.ProcessRawInstruction { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Errorf( + "unexpected config for handler %s: neigher parsed nor raw data is needed", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + } + + for _, intervalConfig := range contractConfig.GetIntervalConfigs() { + if contractAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract %s cannot have interval config", contractAddress)) + } + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) getDataRequirement() (dr sol.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentInstruction: + dr.Tx = append(dr.Tx, sol.TransactionRequirement{ + BlockRange: ag.Range, + Programs: []solana.PublicKey{ag.Address}, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + BlockRange: ag.Range, + IntervalConfig: ag.IntervalConfig, + }) + } + } + return dr +} diff --git a/driver/controller/standard/sol/handler_instruction.go b/driver/controller/standard/sol/handler_instruction.go new file mode 100644 index 0000000..72372fe --- /dev/null +++ b/driver/controller/standard/sol/handler_instruction.go @@ -0,0 +1,133 @@ +package sol + +import ( + "context" + "encoding/json" + + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInstruction struct { + controller.BaseHandlerAgent + + Address solana.PublicKey + + // configures about building binding data + ProcessInnerInstruction bool + ProcessParsedInstruction bool + ProcessRawInstruction bool + FetchTx bool +} + +// BuildBindingDataList builds the per-instruction binding data for this handler's program. +// +// NOTE on the BigQuery data source: when a transaction is served from the BigQuery tier (archival +// history below the ClickHouse range), it carries ONLY the instructions of the program(s) the +// super node was queried for — not the transaction's full instruction set (this is a deliberate +// cost optimization in the archival-tier transaction lookup, which is clustered by program_id). +// That is fine here: this handler only emits binding data for instructions whose +// ProgramId == a.Address (the program it targets), which the BigQuery query always includes; and +// the raw transaction attached below (via getTxJSON) still has complete transaction-level data +// (account keys, balances, token balances, logs, status/err, fee, compute units). A handler must +// not rely on unrelated programs' instructions being present in the raw transaction. +func (a HandlerAgentInstruction) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for _, tx := range bd.mainData.Transactions { + if tx.Transaction == nil { + continue + } + // get instructions + var instructions []*rpc.ParsedInstruction + var indexStartOffset int + if a.ProcessInnerInstruction && tx.Meta != nil { + for _, innerInstruction := range tx.Meta.InnerInstructions { + instructions = append(instructions, innerInstruction.Instructions...) + } + indexStartOffset = -len(instructions) + } + instructions = append(instructions, tx.Transaction.Message.Instructions...) + // build binding data for each instruction + for i, instruction := range instructions { + if instruction.ProgramId != a.Address { + continue + } + if (!a.ProcessParsedInstruction || instruction.Parsed == nil) && !a.ProcessRawInstruction { + continue // no data + } + dataSize := len(instruction.Accounts)*45 + len(instruction.Data) + // get parsed + var rawParsed *string + if a.ProcessParsedInstruction && instruction.Parsed != nil { + b, marshalErr := json.Marshal(instruction.Parsed) + if marshalErr != nil { + return nil, errors.Wrapf(marshalErr, + "marshal #%d instruction parsed in transaction %d/%s failed", i, bd.GetBlockNumber(), tx.Signature) + } + s := string(b) + rawParsed = &s + dataSize += len(b) + } + // get instructionData + var instructionData string + if a.ProcessRawInstruction { + instructionData = instruction.Data.String() + } + // get raw transaction + var rawTx *string + if a.FetchTx { + rawTxJSON, getRawTxErr := bd.getTxJSON(tx) + if getRawTxErr != nil { + return nil, getRawTxErr + } + rawTx = &rawTxJSON + dataSize += len(rawTxJSON) + } + // build binding data + data := &protos.Data{ + Value: &protos.Data_SolInstruction_{ + SolInstruction: &protos.Data_SolInstruction{ + Slot: bd.GetBlockNumber(), + ProgramAccountId: a.Address.String(), + Accounts: utils.MapSliceNoError(instruction.Accounts, solana.PublicKey.String), + InstructionData: instructionData, + RawParsed: rawParsed, + RawTransaction: rawTx, + }, + }, + } + // append result + // calculate the TxInnerIndex of the binding data + // - index of inner instruction is in [-len(InnerInstructions), -1] + // - index of normal instruction is in [0, len(NormalInstructions)-1] + result = append(result, standard.BindingDataInner{ + Data: data, + DataSize: dataSize, + HandlerType: protos.HandlerType_SOL_INSTRUCTION, + TxIndex: int(tx.TransactionIndex), + TxInnerIndex: i + indexStartOffset, + }) + } + } + return result, nil +} + +func (a HandlerAgentInstruction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Address": a.Address.String(), + "ProcessInnerInstruction": a.ProcessInnerInstruction, + "ProcessParsedInstruction": a.ProcessParsedInstruction, + "ProcessRawInstruction": a.ProcessRawInstruction, + "FetchTx": a.FetchTx, + } +} diff --git a/driver/controller/standard/sol/handler_interval.go b/driver/controller/standard/sol/handler_interval.go new file mode 100644 index 0000000..e9a9a65 --- /dev/null +++ b/driver/controller/standard/sol/handler_interval.go @@ -0,0 +1,61 @@ +package sol + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + } +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + // NOTE on the BigQuery data source: for an interval handler over the archival range, the block is + // served by the BigQuery tier, whose getBlocksByInterval returns the block HEADER ONLY — no + // transaction signatures (a deliberate BigQuery cost optimization; see the archival store and the + // data-layer GetBlocksByInterval). So rawBlock here has the header fields (slot, hashes, time) + // but an empty transaction/signature list. The SOL_BLOCK interval handler is block-header + // oriented, so this is fine; a processor that needs the block's transactions must not rely on an + // interval handler over the BigQuery range. + rawBlock, err := bd.getBlockJSON() + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_SOL_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SolBlock_{ + SolBlock: &protos.Data_SolBlock{ + RawBlock: rawBlock, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawBlock), + }}, nil +} diff --git a/driver/controller/standard/sui/BUILD.bazel b/driver/controller/standard/sui/BUILD.bazel new file mode 100644 index 0000000..862d014 --- /dev/null +++ b/driver/controller/standard/sui/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sui", + srcs = [ + "agents.go", + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sui", + visibility = ["//visibility:public"], + deps = [ + "//chain/move", + "//chain/sui", + "//chain/sui/types", + "//common/compress", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "sui_test", + srcs = ["handler_interval_test.go"], + embed = [":sui"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/driver/controller/standard/sui/agents.go b/driver/controller/standard/sui/agents.go new file mode 100644 index 0000000..de5436c --- /dev/null +++ b/driver/controller/standard/sui/agents.go @@ -0,0 +1,291 @@ +package sui + +import ( + "context" + "strings" + + "sentioxyz/sentio-core/chain/move" + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" +) + +// BuildSuiAgents turns the parsed processor config into the SUI handler agents, +// emitting each built agent via emit. It is the format-agnostic core shared by +// the json-rpc handler controller (this package) and the grpc one +// (standard/sui/grpc): both build the SAME agents/filters here, and only differ +// in how the agent's BuildBindingDataList reads data and serializes the binding. +// The grpc controller wraps each emitted agent into a grpc agent that embeds it. +func BuildSuiAgents( + ctx context.Context, + config standard.HandlerConfig, + chainConfig *chain.ConfigV2, + sdkVersion string, + client sui.Client, + first uint64, + getAddressStart func(ctx context.Context, address string, start uint64) (uint64, error), + emit func(SuiHandlerAgent), +) *controller.ExternalError { + _, logger := log.FromContext(ctx) + + processorVersion, err := processor.ParseVersion(sdkVersion) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "parse processor sdk version %q failed", sdkVersion)) + } + + for dataSourceID, accountConfig := range config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + dataSource := standard.BuildDataSource("SUI", chainConfig.ChainID, "Account", accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + + if blockRange.StartBlock, err = getAddressStart(ctx, accountAddress, blockRange.StartBlock); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, intervalConfig := range accountConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + Client: client, + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + switch intervalConfig.GetOwnerType() { + case protos.MoveOwnerType_ADDRESS, protos.MoveOwnerType_OBJECT, protos.MoveOwnerType_WRAPPED_OBJECT: + if accountAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unexpected config for handler %s: address should not be empty because owner type is %s", + agent.GetHandlerID().String(), intervalConfig.GetOwnerType().String())) + } + agent.Filter.OwnerFilter = &chainsui.ObjectChangeOwnerFilter{OwnerID: []string{accountAddress}} + agent.UnwrapDynamicObject = true + switch intervalConfig.GetOwnerType() { + case protos.MoveOwnerType_ADDRESS: + agent.Filter.OwnerFilter.OwnerType = []string{"address"} + case protos.MoveOwnerType_OBJECT: + agent.Filter.OwnerFilter.OwnerType = []string{"object"} + agent.NeedSelf = true + case protos.MoveOwnerType_WRAPPED_OBJECT: + agent.Filter.OwnerFilter.OwnerType = []string{"object"} + } + if !intervalConfig.GetResourceFetchConfig().GetOwned() { + agent.Filter.OwnerFilter.OwnerType = nil + } + case protos.MoveOwnerType_TYPE: + if accountAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Errorf( + "unexpected config for handler %s: address is %s, it should be empty because owner type is %s", + agent.GetHandlerID().String(), accountAddress, intervalConfig.GetOwnerType().String())) + } + var objectType move.Type + objectType, err = move.BuildType(intervalConfig.GetType()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid object type %q", + agent.GetHandlerID().String(), intervalConfig.GetType())) + } + agent.Filter.TypePattern = move.TypeSet{objectType} + agent.UnwrapDynamicObject = false + default: + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unexpected config for handler %s: unknown owner type %s", + agent.GetHandlerID().String(), intervalConfig.GetOwnerType().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, moveCallConfig := range accountConfig.GetMoveCallConfigs() { + agent, extErr := buildFunctionAgent(dataSource, dataSourceID, accountAddress, moveCallConfig, blockRange) + if extErr != nil { + return extErr + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range accountConfig.GetMoveResourceChangeConfigs() { + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + } + agent.Filter.TypePattern, err = utils.MapSlice(changeConfig.GetTypes(), move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + for dataSourceID, contractConfig := range config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("SUI", chainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + + if blockRange.StartBlock, err = getAddressStart(ctx, contractAddress, blockRange.StartBlock); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + var pkgHistory []string // used for search real event type + if contractAddress != "" { + if !chainConfig.KeepSuiEventTypePackage && len(contractConfig.GetMoveEventConfigs()) > 0 { + if pkgHistory, err = client.GetPackageHistory(ctx, contractAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeFetchDataFailed, + errors.Wrapf(err, "get package history for contract %s failed", contractAddress)) + } + } else { + pkgHistory = []string{contractAddress} + } + } + + for _, eventConfig := range contractConfig.GetMoveEventConfigs() { + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "event", eventConfig, blockRange), + Filter: chainsui.TransactionFilter{ + FailedIsOK: eventConfig.GetFetchConfig().GetIncludeFailedTransaction(), + }, + FetchConfig: chainsui.TransactionFetchConfig{ + NeedInputs: true, + NeedEffects: true, + NeedAllEvents: eventConfig.GetFetchConfig().GetAllEvents(), + }, + } + // sdk before v2.32, includeInputs and includeResourceChanges always be true + if processorVersion.Major > 2 || (processorVersion.Major == 2 && processorVersion.Minor >= 32) { + agent.FetchConfig.NeedInputs = eventConfig.GetFetchConfig().GetInputs() + agent.FetchConfig.NeedEffects = eventConfig.GetFetchConfig().GetResourceChanges() + } + for _, f := range eventConfig.GetFilters() { + var ff chainsui.EventFilterV2 + if sender := f.GetEventAccount(); sender != "" { + ff.Sender = &sender + } + // Address is needed as the packageId to form a complete event type; the address part of the + // event type maybe the upgraded package id, so iterate the whole package history. + for _, addr := range pkgHistory { + var typ move.Type + typ, err = move.BuildType(addr + "::" + f.GetType()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid event type %q", + agent.GetHandlerID().String(), f.GetType())) + } + ff.TypePattern = append(ff.TypePattern, typ) + } + agent.Filter.EventFilters = append(agent.Filter.EventFilters, ff) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, moveCallConfig := range contractConfig.GetMoveCallConfigs() { + agent, extErr := buildFunctionAgent(dataSource, dataSourceID, contractAddress, moveCallConfig, blockRange) + if extErr != nil { + return extErr + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range contractConfig.GetMoveResourceChangeConfigs() { + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + } + agent.Filter.TypePattern, err = utils.MapSlice(changeConfig.GetTypes(), move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + return nil +} + +// buildFunctionAgent builds a move-call (function) agent; shared by both the account and contract loops. +func buildFunctionAgent( + dataSource string, + dataSourceID int, + address string, + moveCallConfig *protos.MoveCallHandlerConfig, + blockRange controller.BlockRange, +) (HandlerAgentFunction, *controller.ExternalError) { + agent := HandlerAgentFunction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "call", moveCallConfig, blockRange), + FetchConfig: chainsui.TransactionFetchConfig{ + NeedInputs: true, + NeedEffects: true, + NeedAllEvents: true, + }, + } + for _, f := range moveCallConfig.GetFilters() { + var ff chainsui.FunctionFilter + ff.Kind = utils.WrapPointer("ProgrammableTransaction") + ff.CommandFilter = &chainsui.CommandFilter{} + if address != "" { + packageID, parseErr := types.StrToObjectID(address) + if parseErr != nil { + return agent, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(parseErr, "unexpected config for handler %s: address %q is not a valid object id", + agent.GetHandlerID().String(), address)) + } + ff.CommandFilter.CallPackage = utils.WrapPointer(packageID.String()) + } + callModule, callFunc, _ := strings.Cut(f.GetFunction(), "::") + if callModule != "" && callModule != "*" { + ff.CommandFilter.CallModule = &callModule + } + if callFunc != "" && callFunc != "*" { + ff.CommandFilter.CallFunction = &callFunc + } + if ff.CommandFilter.IsEmpty() { + ff.CommandFilter = nil + } + if prefix := f.GetPublicKeyPrefix(); prefix != "" { + if !strings.HasPrefix(prefix, "0x") { + prefix = "0x" + prefix + } + ff.MultiSigPublicKeyPrefix = &prefix + } + if from := f.GetFromAndToAddress().GetFrom(); from != "" { + ff.Sender = &from + } + if to := f.GetFromAndToAddress().GetTo(); to != "" { + ff.Receiver = &to + } + ff.FailedIsOK = f.GetIncludeFailed() && moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction() + agent.Filter.FunctionFilters = append(agent.Filter.FunctionFilters, ff) + } + agent.Filter.FailedIsOK = moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction() + if len(agent.Filter.FunctionFilters) > 0 { + agent.Filter.FailedIsOK = utils.Reduce( + utils.MapSliceNoError(agent.Filter.FunctionFilters, func(ff chainsui.FunctionFilter) bool { + return ff.FailedIsOK + }), + func(a, b bool) bool { return a || b }, + ) + } + return agent, nil +} diff --git a/driver/controller/standard/sui/block_data.go b/driver/controller/standard/sui/block_data.go new file mode 100644 index 0000000..98c39df --- /dev/null +++ b/driver/controller/standard/sui/block_data.go @@ -0,0 +1,87 @@ +package sui + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/sui" +) + +type BlockData struct { + controller.BlockHeader + + mainData sui.BlockMainData + checkpointData map[string]string + + objMgr *ObjectDictSetManager + cachedTxn map[string]string + cachedChanges map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getTxn( + txIndex int, + eventFilters []chainsui.EventFilterV2, + fetchConfig chainsui.TransactionFetchConfig, +) (string, error) { + key := fmt.Sprintf("%d/%s", txIndex, fetchConfig) + if !fetchConfig.NeedAllEvents { + key = fmt.Sprintf("%s/%v", key, eventFilters) + } + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if str, has := b.cachedTxn[key]; has { + return str, nil + } + tx := b.mainData.Txs[txIndex] + raw, err := json.Marshal(fetchConfig.PruneTransaction(tx, eventFilters)) + if err != nil { + return "", errors.Wrapf(err, "marshal sui tx %s with fetch config %s and event filters %s failed", + tx.Digest.String(), fetchConfig.String(), utils.MustJSONMarshal(eventFilters)) + } + str := string(raw) + b.cachedTxn[key] = str + return str, nil +} + +func (b *BlockData) getChange(index int) (string, error) { + if b.cachedChanges == nil { + b.cachedChanges = make(map[int]string) + } + if str, has := b.cachedChanges[index]; has { + return str, nil + } + raw, err := json.Marshal(b.mainData.ObjectChanges[index]) + if err != nil { + return "", errors.Wrapf(err, "marshal sui object change #%d in block %d failed: %#v ", + index, b.GetBlockNumber(), b.mainData.ObjectChanges[index]) + } + str := string(raw) + b.cachedChanges[index] = str + return str, nil +} diff --git a/driver/controller/standard/sui/grpc/BUILD.bazel b/driver/controller/standard/sui/grpc/BUILD.bazel new file mode 100644 index 0000000..942f20d --- /dev/null +++ b/driver/controller/standard/sui/grpc/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "grpc", + srcs = [ + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sui/grpc", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//chain/sui/types", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/protojson", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/data/sui/grpc", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//driver/controller/standard/sui", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + "@org_golang_google_protobuf//types/known/fieldmaskpb", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "grpc_test", + srcs = ["handler_test.go"], + embed = [":grpc"], + deps = [ + "//chain/sui", + "//common/set", + "//driver/controller/data/sui", + "//driver/controller/data/sui/grpc", + "//driver/controller/standard/sui", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/driver/controller/standard/sui/grpc/block_data.go b/driver/controller/standard/sui/grpc/block_data.go new file mode 100644 index 0000000..d4e4846 --- /dev/null +++ b/driver/controller/standard/sui/grpc/block_data.go @@ -0,0 +1,92 @@ +// Package grpc is the grpc-data twin of standard/sui: same handler controller / +// agents / config parsing (reused from the parent via sui.BuildSuiAgents and the +// embedded sui agents), but each agent's BuildBindingDataList reads grpc-format +// block data (data/sui/grpc) and serializes the DataBinding raw_* fields from the +// grpc structs (ExtendedGrpc* / rpcv2.* via protojson). Selected by the launcher +// only for the SUI variation at DriverVersion >= 2. +package grpc + +import ( + "encoding/json" + "fmt" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + + "github.com/pkg/errors" +) + +type BlockData struct { + controller.BlockHeader + + mainData suigrpcdata.BlockMainData + checkpointData map[string]string + + objMgr *suihandler.ObjectDictSetManager + cachedTxn map[string]string + cachedChanges map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +// getTxn returns the grpc-format json of the (pruned) transaction at txIndex. +func (b *BlockData) getTxn( + txIndex int, + eventFilters []chainsui.EventFilterV2, + fetchConfig chainsui.TransactionFetchConfig, +) (string, error) { + key := fmt.Sprintf("%d/%s", txIndex, fetchConfig) + if !fetchConfig.NeedAllEvents { + key = fmt.Sprintf("%s/%v", key, eventFilters) + } + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if str, has := b.cachedTxn[key]; has { + return str, nil + } + tx := b.mainData.Txs[txIndex] + raw, err := json.Marshal(fetchConfig.PruneGrpcTransaction(tx, eventFilters)) + if err != nil { + return "", errors.Wrapf(err, "marshal grpc sui tx at index %d in block %d failed", txIndex, b.GetBlockNumber()) + } + str := string(raw) + b.cachedTxn[key] = str + return str, nil +} + +func (b *BlockData) getChange(index int) (string, error) { + if b.cachedChanges == nil { + b.cachedChanges = make(map[int]string) + } + if str, has := b.cachedChanges[index]; has { + return str, nil + } + raw, err := json.Marshal(b.mainData.ObjectChanges[index]) + if err != nil { + return "", errors.Wrapf(err, "marshal grpc sui object change #%d in block %d failed", index, b.GetBlockNumber()) + } + str := string(raw) + b.cachedChanges[index] = str + return str, nil +} diff --git a/driver/controller/standard/sui/grpc/handler.go b/driver/controller/standard/sui/grpc/handler.go new file mode 100644 index 0000000..88d316c --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler.go @@ -0,0 +1,238 @@ +package grpc + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type GrpcHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[suidata.Client, *BlockData, GrpcHandlerAgent] + + objMgr suihandler.ObjectDictSetManager +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client suidata.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[suidata.Client, *BlockData, GrpcHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if err := c.objMgr.Load(checkpoint); err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "parse object set from checkpoint data failed")) + } + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetObjectCreation(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return start, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, _ uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + extErr := suihandler.BuildSuiAgents( + ctx, c.Config, c.ChainConfig, c.Processor.SdkVersion, c.Client, first, c.getAddressStart, + func(agent suihandler.SuiHandlerAgent) { + c.Agents = append(c.Agents, wrapAgent(agent)) + }, + ) + if extErr != nil { + return extErr + } + logger.Infof("built %d grpc agents", len(c.Agents)) + return nil +} + +// wrapAgent wraps a json-rpc sui agent (built by the shared BuildSuiAgents) into its grpc twin, which +// reuses the embedded agent's filters but reads grpc data when building bindings. +func wrapAgent(agent suihandler.SuiHandlerAgent) GrpcHandlerAgent { + switch ag := agent.(type) { + case suihandler.HandlerAgentEvent: + return HandlerAgentEvent{HandlerAgentEvent: ag} + case suihandler.HandlerAgentFunction: + return HandlerAgentFunction{HandlerAgentFunction: ag} + case suihandler.HandlerAgentChange: + return HandlerAgentChange{HandlerAgentChange: ag} + case suihandler.HandlerAgentInterval: + return HandlerAgentInterval{HandlerAgentInterval: ag} + default: + panic(errors.Errorf("unknown sui handler agent type %T", agent)) + } +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("SUI::%s::grpc::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + suigrpcdata.BuildBlockMainDataFetcher( + fetchNamePrefix, + req, + firstBlockNumber, + currentBlockNumber, + latest, + c.Client, + ), + latest, + 1, + 256*1024*1024, + 1000, + 0, + 20, + time.Second*10, + func(ctx context.Context, blockNumber uint64, from suigrpcdata.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + bd := BlockData{mainData: from, checkpointData: make(map[string]string)} + if from.SimpleBlock != nil { + bd.BlockHeader = *from.SimpleBlock + } else { + bd.BlockHeader, err = c.Client.GetSimpleBlock(ctx, blockNumber) + if err != nil { + return nil, false, err + } + } + if err = c.pushIntervalAgent(ctx, blockNumber, &bd); err != nil { + return nil, false, err + } + if bd.taskList, bd.taskTotalSize, err = c.BuildTaskList(ctx, &bd); err != nil { + return nil, false, err + } + c.DumpAddressStart(bd.checkpointData) + return &bd, true, nil + }, + ) +} + +func (c *HandlerController) pushIntervalAgent(ctx context.Context, blockNumber uint64, blockData *BlockData) error { + g, gctx := errgroup.WithContext(ctx) + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(blockData.mainData.Intervals, ag.IntervalConfig) { + continue + } + key := ag.ObjMgrKey() + g.Go(func() error { + newDict, err := ag.PushObjectLatestVersion(gctx, blockNumber, c.objMgr.Get(key)) + if err != nil { + return err + } + c.objMgr.Put(key, newDict) + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + blockData.objMgr = &c.objMgr + blockData.checkpointData[suihandler.CheckpointDataKey] = c.objMgr.GetData() + return nil +} + +func (c *HandlerController) getDataRequirement() (dr suidata.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, suidata.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, suidata.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentChange: + dr.ObjectChanges = append(dr.ObjectChanges, suidata.ObjectChangeRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} + +func (c *HandlerController) Snapshot() any { + sp := c.BaseHandlerController.Snapshot().(map[string]any) + sp["objectManager"] = c.objMgr.Snapshot() + return sp +} diff --git a/driver/controller/standard/sui/grpc/handler_change.go b/driver/controller/standard/sui/grpc/handler_change.go new file mode 100644 index 0000000..705b95c --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_change.go @@ -0,0 +1,47 @@ +package grpc + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentChange struct { + suihandler.HandlerAgentChange +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + checker := a.Filter.CheckerGrpc() + for i, oc := range bd.mainData.ObjectChanges { + if !checker(oc.ChangedObject) { + continue + } + var rawChange string + if rawChange, err = bd.getChange(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT_CHANGE, + TxIndex: int(oc.TxIndex), + Data: &protos.Data{ + Value: &protos.Data_SuiObjectChange_{ + SuiObjectChange: &protos.Data_SuiObjectChange{ + RawChanges: []string{rawChange}, + Slot: bd.GetBlockNumber(), + TxDigest: oc.TxDigest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: len(rawChange), + }) + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_event.go b/driver/controller/standard/sui/grpc/handler_event.go new file mode 100644 index 0000000..7751933 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_event.go @@ -0,0 +1,64 @@ +package grpc + +import ( + "context" + + chainsui "sentioxyz/sentio-core/chain/sui" + cprotojson "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// HandlerAgentEvent embeds the json-rpc event agent (reusing its Filter / +// FetchConfig / BaseHandlerAgent / Snapshot) and only overrides binding building +// to read grpc transactions and emit grpc-format raw_event / raw_transaction. +type HandlerAgentEvent struct { + suihandler.HandlerAgentEvent +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.CheckGrpcTx(tx.ExecutedTransaction) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, a.Filter.EventFilters, a.FetchConfig); err != nil { + return nil, err + } + eventChecker := chainsui.BuildGrpcEventChecker(a.Filter.EventFilters) + for evIndex, ev := range tx.GetEvents().GetEvents() { + if !eventChecker(ev) { + continue + } + var rawEvent []byte + if rawEvent, err = cprotojson.Marshal(ev); err != nil { + return nil, errors.Wrapf(err, "marshal grpc sui event #%d in tx %d in block %d failed", + evIndex, txIndex, bd.GetBlockNumber()) + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_EVENT, + TxIndex: txIndex, + TxInnerIndex: evIndex, + Data: &protos.Data{ + Value: &protos.Data_SuiEvent_{ + SuiEvent: &protos.Data_SuiEvent{ + RawEvent: string(rawEvent), + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawEvent) + len(rawTxn), + }) + } + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_function.go b/driver/controller/standard/sui/grpc/handler_function.go new file mode 100644 index 0000000..5b000c1 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_function.go @@ -0,0 +1,45 @@ +package grpc + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentFunction struct { + suihandler.HandlerAgentFunction +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.CheckGrpcTx(tx.ExecutedTransaction) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, nil, a.FetchConfig); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + TxIndex: txIndex, + HandlerType: protos.HandlerType_SUI_CALL, + Data: &protos.Data{ + Value: &protos.Data_SuiCall_{ + SuiCall: &protos.Data_SuiCall{ + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawTxn), + }) + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_interval.go b/driver/controller/standard/sui/grpc/handler_interval.go new file mode 100644 index 0000000..2d54b42 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_interval.go @@ -0,0 +1,297 @@ +package grpc + +import ( + "context" + "fmt" + "math" + "time" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/log" + cprotojson "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" + "google.golang.org/protobuf/types/known/fieldmaskpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentInterval struct { + suihandler.HandlerAgentInterval +} + +const ( + grpcObjectsConcurrency = 10 + grpcObjectsBatchSize = 50 +) + +// objectReadMask asks the super node for the fields the binding needs (the default mask is only +// object_id,version,digest). Unlike the json-rpc handler — which binds just the parsed content +// (SuiParsedData) — the grpc handler binds the whole rpcv2.Object (see RawSelf below), so the mask +// must include every field that object carries: object_type and has_public_transfer (NOT present in +// the rendered json), owner, and json itself. object_type additionally drives the dynamic-object check. +var objectReadMask = &fieldmaskpb.FieldMask{ + Paths: []string{"object_id", "version", "digest", "object_type", "has_public_transfer", "owner", "json"}, +} + +var ignoreNotExistObject = envconf.LoadBool("SENTIO_SUI_INTERVAL_HANDLER_IGNORE_NON_EXISTENT_OBJECTS", true) + +var dynamicType = types.TypeTagFromStringMust( + "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper, 0x2::object::ID>") + +type grpcObject struct { + Version uint64 + Digest string + Content string // protojson of rpcv2.Object — the grpc-format object json +} + +// PushObjectLatestVersion overrides the embedded json-rpc implementation to track the latest object +// versions from grpc object changes (sui_filterGrpcChangedObjects) instead of the json-rpc +// sui_filterObjectChangesV2, keeping the interval handler on the grpc data source end to end (it later +// fetches those versions via GetGrpcObjects). The version-tracking logic is otherwise identical. +func (a HandlerAgentInterval) PushObjectLatestVersion( + ctx context.Context, + blockNumber uint64, + dict *suihandler.ObjectDict, +) (suihandler.ObjectDict, error) { + _, logger := log.FromContext(ctx, "handler", a.HandlerID.String(), "filter", utils.MustJSONMarshal(a.Filter)) + if dict != nil && dict.BlockNumber == blockNumber { // may be retried, so dict.BlockNumber may be equal to blockNumber + return *dict, nil + } + var from uint64 + result := suihandler.ObjectDict{ + BlockNumber: blockNumber, + ObjectLatestVersion: make(map[string]uint64), + } + if dict != nil { + from = dict.BlockNumber + 1 + result.ObjectLatestVersion = utils.CopyMap(dict.ObjectLatestVersion) + } + logger.Infof("will push grpc object latest version dict in [%d,%d]", from, blockNumber) + for from <= blockNumber { + startAt := time.Now() + end := min(blockNumber, from+suihandler.MaxQueryObjectChangeRangeSize-1) + changes, err := a.Client.GetGrpcObjectChanges(ctx, from, end, a.Filter) + if err != nil { + return suihandler.ObjectDict{}, err + } + beforeSize := len(result.ObjectLatestVersion) + var deleteCount, updateCount, createCount int + for _, bn := range utils.GetOrderedMapKeys(changes) { + for _, oc := range changes[bn] { + objectID, objectVersion := oc.GetObjectId(), oc.GetOutputVersion() + if chainsui.GetChangeType(oc.ChangedObject).IsDeleted() { + delete(result.ObjectLatestVersion, objectID) + deleteCount++ + } else if _, has := result.ObjectLatestVersion[objectID]; has { + result.ObjectLatestVersion[objectID] = objectVersion + updateCount++ + } else { + result.ObjectLatestVersion[objectID] = objectVersion + createCount++ + } + } + } + logger.With("used", time.Since(startAt).String()). + Infof("pushed grpc object latest version dict in [%d,%d], size %d => %d, created %d and updated %d and deleted %d", + from, end, beforeSize, len(result.ObjectLatestVersion), createCount, updateCount, deleteCount) + from = end + 1 + } + if size := len(result.ObjectLatestVersion); size > suihandler.MaxObjectDictLen { + err := errors.Errorf("object latest version dict size for handler %s with filter %s is too big: %d > %d", + a.HandlerID.String(), utils.MustJSONMarshal(a.Filter), size, suihandler.MaxObjectDictLen) + return suihandler.ObjectDict{}, fetcher.Permanent(err) + } + return result, nil +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return + } + dict := bd.objMgr.Get(a.ObjMgrKey()) + if dict == nil { + return + } + + _, logger := log.FromContext(ctx) + var requests []*rpcv2.GetObjectRequest + for objectID, version := range dict.ObjectLatestVersion { + requests = append(requests, newObjectRequest(objectID, version)) + } + contents := make(map[string]grpcObject) + for len(requests) > 0 { + var resp []*rpcv2.GetObjectResult + if resp, err = a.Client.GetGrpcObjects(ctx, requests, grpcObjectsConcurrency, grpcObjectsBatchSize); err != nil { + return + } + var wrappedObjectIDList []string + for i, res := range resp { + req := requests[i] + obj := res.GetObject() + if obj == nil { + // deleted / not found — GetObjectResult carries an error instead of an object + message := fmt.Sprintf("object %s version %d not returned: %s", + req.GetObjectId(), req.GetVersion(), res.GetError().GetMessage()) + if !ignoreNotExistObject { + return nil, errors.Errorf(message) + } + logger.Warnf("%s, will be ignored", message) + continue + } + objectType, _ := types.TypeTagFromString(obj.GetObjectType()) + if a.UnwrapDynamicObject && objectType != nil && dynamicType.Include(*objectType) { + wrappedObjectIDList = append(wrappedObjectIDList, wrappedObjectID(obj.GetJson())) + } else { + var content []byte + // Bind the WHOLE rpcv2.Object, intentionally diverging from the json-rpc handler (which + // binds only the parsed content, SuiParsedData). The grpc object's rendered value + // (obj.GetJson()) is leaner than SuiParsedData: it flattens the move struct's fields to + // the top level and drops object_type / has_public_transfer (those live as sibling + // fields on rpcv2.Object, not inside json). Marshaling the whole object preserves that + // info for the processor. Example — sui-mainnet object + // 0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d version 309855314 + // grpc json (fields flattened, no object_type / has_public_transfer): + // { + // "id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d", + // "name": {"name": "0xe859a7ebc84e7573d1e81ef99946f8821aeb0ff67454e579a32dd216da239621"}, + // "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" + // } + // whereas json-rpc content wraps the fields and adds type info: + // { + // "dataType": "moveObject", + // "type": "...", + // "hasPublicTransfer": false, + // "fields": { + // "id": {"id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d"}, + // "name": {...}, + // "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" + // } + // } + if content, err = cprotojson.Marshal(obj); err != nil { + return nil, errors.Wrapf(err, "marshal grpc object %s failed", obj.GetObjectId()) + } + contents[obj.GetObjectId()] = grpcObject{ + Version: obj.GetVersion(), + Digest: obj.GetDigest(), + Content: string(content), + } + } + } + requests = requests[:0] + if len(wrappedObjectIDList) > 0 { + var stat []chainsui.ObjectStat + if stat, err = a.Client.GetObjectsStat(ctx, 0, bd.GetBlockNumber(), wrappedObjectIDList); err != nil { + return nil, err + } + for i, objectID := range wrappedObjectIDList { + if stat[i].Count > 0 { + requests = append(requests, newObjectRequest(objectID, stat[i].MaxObjectVersion)) + } else { + logger.Warnf("Object %s has no history", objectID) + } + } + } + } + + if a.Filter.TypePattern != nil { + for objectID, obj := range contents { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: utils.WrapPointer(obj.Content), + ObjectId: objectID, + ObjectVersion: obj.Version, + ObjectDigest: obj.Digest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(obj.Content), + }) + } + } + if a.Filter.OwnerFilter != nil { + var dataSize int + var owned []string + var self *grpcObject + for objectID, obj := range contents { + if utils.IndexOf(a.Filter.OwnerFilter.OwnerID, objectID) < 0 { + owned = append(owned, obj.Content) + dataSize += len(obj.Content) + } else { + self = utils.WrapPointer(contents[objectID]) + } + } + var selfContent *string + var selfVersion uint64 + var selfDigest string + if self != nil { + selfContent = utils.WrapPointer(self.Content) + selfVersion = self.Version + selfDigest = self.Digest + dataSize += len(self.Content) + } else if a.NeedSelf { + return + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: selfContent, + ObjectId: a.Filter.OwnerFilter.OwnerID[0], + ObjectVersion: selfVersion, + ObjectDigest: selfDigest, + RawObjects: owned, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: dataSize, + }) + } + return +} + +func newObjectRequest(objectID string, version uint64) *rpcv2.GetObjectRequest { + return &rpcv2.GetObjectRequest{ + ObjectId: utils.WrapPointer(objectID), + Version: utils.WrapPointer(version), + ReadMask: objectReadMask, + } +} + +// wrappedObjectID extracts the wrapped object id from a dynamic-object-field object's grpc json. The +// json-rpc handler reads content.fields.value, but the grpc rendered json flattens the move struct's +// fields to the top level, so "value" sits directly under the root. Verified against sui-mainnet object +// 0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d version 309855314 (a +// 0x2::dynamic_field::Field, object::ID>), whose grpc json is +// +// { +// "id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d", +// "name": {"name": "0xe859a7ebc84e7573d1e81ef99946f8821aeb0ff67454e579a32dd216da239621"}, +// "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" +// } +// +// → value = 0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0 (the wrapped object id). +func wrappedObjectID(v *structpb.Value) string { + return v.GetStructValue().GetFields()["value"].GetStringValue() +} diff --git a/driver/controller/standard/sui/grpc/handler_test.go b/driver/controller/standard/sui/grpc/handler_test.go new file mode 100644 index 0000000..dba61fc --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_test.go @@ -0,0 +1,127 @@ +package grpc + +import ( + "context" + "encoding/json" + "testing" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/set" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func grpcTxWithEvent() *chainsui.ExtendedGrpcTransaction { + return &chainsui.ExtendedGrpcTransaction{ + Checkpoint: 100, + CheckpointDigest: "ckpt", + TimestampMs: 1700000000000, + Epoch: 5, + TxIndex: 0, + ExecutedTransaction: &rpcv2.ExecutedTransaction{ + Digest: proto.String("tx1"), + Effects: &rpcv2.TransactionEffects{Status: &rpcv2.ExecutionStatus{Success: proto.Bool(true)}}, + Events: &rpcv2.TransactionEvents{Events: []*rpcv2.Event{{ + PackageId: proto.String("0x0000000000000000000000000000000000000000000000000000000000000002"), + Module: proto.String("m"), + EventType: proto.String("0x2::m::E"), + Sender: proto.String("0xabc"), + }}}, + }, + } +} + +func newBlockData(txs ...*chainsui.ExtendedGrpcTransaction) *BlockData { + bd := &BlockData{mainData: suigrpcdata.BlockMainData{Txs: txs}} + bd.BlockHeader = suidata.SimpleBlock{Checkpoint: 100, Digest: "ckpt", TimestampMS: 1700000000000} + return bd +} + +// matchAnyFilter passes the tx-level CheckGrpcTx (an empty EventFilterV2 matches any event of a +// successful tx) so the binding-serialization is what's under test, not the filtering. +func matchAnyFilter() chainsui.TransactionFilter { + return chainsui.TransactionFilter{EventFilters: []chainsui.EventFilterV2{{}}} +} + +func TestEventHandlerGrpcBinding(t *testing.T) { + agent := HandlerAgentEvent{suihandler.HandlerAgentEvent{ + Filter: matchAnyFilter(), + FetchConfig: chainsui.TransactionFetchConfig{NeedAllEvents: true}, + }} + result, err := agent.BuildBindingDataList(context.Background(), newBlockData(grpcTxWithEvent())) + require.NoError(t, err) + require.Len(t, result, 1) + + se := result[0].Data.GetSuiEvent() + require.NotNil(t, se) + assert.Equal(t, uint64(100), se.GetSlot()) + + // raw_event is protojson of rpcv2.Event (enum/field names in camelCase, not numbers). + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(se.GetRawEvent()), &ev)) + assert.Equal(t, "0x2::m::E", ev["eventType"]) + assert.Equal(t, "m", ev["module"]) + + // raw_transaction is the grpc ExtendedGrpcTransaction shape: header fields + nested executedTransaction. + var tx map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(se.GetRawTransaction()), &tx)) + assert.Contains(t, tx, "checkpoint") + assert.Contains(t, tx, "executedTransaction") +} + +func TestFunctionHandlerGrpcBinding(t *testing.T) { + agent := HandlerAgentFunction{suihandler.HandlerAgentFunction{ + Filter: matchAnyFilter(), + FetchConfig: chainsui.TransactionFetchConfig{NeedAllEvents: true}, + }} + result, err := agent.BuildBindingDataList(context.Background(), newBlockData(grpcTxWithEvent())) + require.NoError(t, err) + require.Len(t, result, 1) + + sc := result[0].Data.GetSuiCall() + require.NotNil(t, sc) + var tx map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(sc.GetRawTransaction()), &tx)) + assert.Contains(t, tx, "checkpoint") + assert.Contains(t, tx, "executedTransaction") +} + +func TestChangeHandlerGrpcBinding(t *testing.T) { + const objectID = "0x0000000000000000000000000000000000000000000000000000000000000abc" + oc := &chainsui.ExtendedGrpcChangedObject{ + Checkpoint: 100, + TxIndex: 3, + TxDigest: "txd", + ChangedObject: &rpcv2.ChangedObject{ + ObjectId: proto.String(objectID), + ObjectType: proto.String("0x2::coin::Coin"), + }, + } + agent := HandlerAgentChange{suihandler.HandlerAgentChange{ + Filter: chainsui.ObjectChangeFilter{ObjectIDIn: set.New[string](objectID)}, + }} + bd := newBlockData() + bd.mainData.ObjectChanges = []*chainsui.ExtendedGrpcChangedObject{oc} + + result, err := agent.BuildBindingDataList(context.Background(), bd) + require.NoError(t, err) + require.Len(t, result, 1) + + soc := result[0].Data.GetSuiObjectChange() + require.NotNil(t, soc) + assert.Equal(t, "txd", soc.GetTxDigest()) + assert.Equal(t, 3, result[0].TxIndex) + require.Len(t, soc.GetRawChanges(), 1) + + // raw_changes[0] is the ExtendedGrpcChangedObject shape: header fields + nested changedObject. + var change map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(soc.GetRawChanges()[0]), &change)) + assert.Contains(t, change, "checkpoint") + assert.Contains(t, change, "changedObject") +} diff --git a/driver/controller/standard/sui/handler.go b/driver/controller/standard/sui/handler.go new file mode 100644 index 0000000..30a005c --- /dev/null +++ b/driver/controller/standard/sui/handler.go @@ -0,0 +1,225 @@ +package sui + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type SuiHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[sui.Client, *BlockData, SuiHandlerAgent] + + objMgr ObjectDictSetManager +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *chain.ConfigV2, + client sui.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[sui.Client, *BlockData, SuiHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if err := c.objMgr.Load(checkpoint); err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "parse object set from checkpoint data failed")) + } + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetObjectCreation(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + // if has is false, object with id `address` may not exist, in this situation keep the user's StartBlock + return start, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, _ uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + extErr := BuildSuiAgents( + ctx, c.Config, c.ChainConfig, c.Processor.SdkVersion, c.Client, first, c.getAddressStart, + func(agent SuiHandlerAgent) { + c.Agents = append(c.Agents, agent) + }, + ) + if extErr != nil { + return extErr + } + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("SUI::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + sui.BuildBlockMainDataFetcher( + fetchNamePrefix, + req, + firstBlockNumber, + currentBlockNumber, + latest, + c.Client, + ), + latest, + 1, // The transfer process must be performed strictly in block order，so it cannot be performed concurrently. + 256*1024*1024, + 1000, + 0, // The push process may take a long time, so no timeout is set. + 20, + time.Second*10, + func(ctx context.Context, blockNumber uint64, from sui.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + bd := BlockData{mainData: from, checkpointData: make(map[string]string)} + // Always need the header. Prefer the one the data fetchers prefetched concurrently (off this + // strictly block-ordered path); only fall back to a serial RPC if it's missing, which keeps + // the order-independent sui_getSimpleCheckpoint out of the throughput-limiting path. + if from.SimpleBlock != nil { + bd.BlockHeader = *from.SimpleBlock + } else { + bd.BlockHeader, err = c.Client.GetSimpleBlock(ctx, blockNumber) + if err != nil { + return nil, false, err + } + } + // interval agent should push object latest version + if err = c.pushIntervalAgent(ctx, blockNumber, &bd); err != nil { + return nil, false, err + } + // build binding data + if bd.taskList, bd.taskTotalSize, err = c.BuildTaskList(ctx, &bd); err != nil { + return nil, false, err + } + c.DumpAddressStart(bd.checkpointData) + return &bd, true, nil + }, + ) +} + +func (c *HandlerController) pushIntervalAgent(ctx context.Context, blockNumber uint64, blockData *BlockData) error { + g, gctx := errgroup.WithContext(ctx) + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(blockData.mainData.Intervals, ag.IntervalConfig) { + continue + } + key := ag.ObjMgrKey() + g.Go(func() error { + if newDict, err := ag.PushObjectLatestVersion(gctx, blockNumber, c.objMgr.Get(key)); err != nil { + return err + } else { + c.objMgr.Put(key, newDict) + return nil + } + }) + } + if err := g.Wait(); err != nil { + return err + } + blockData.objMgr = &c.objMgr + blockData.checkpointData[CheckpointDataKey] = c.objMgr.GetData() + return nil +} + +func (c *HandlerController) getDataRequirement() (dr sui.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, sui.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, sui.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentChange: + dr.ObjectChanges = append(dr.ObjectChanges, sui.ObjectChangeRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} + +func (c *HandlerController) Snapshot() any { + sp := c.BaseHandlerController.Snapshot().(map[string]any) + sp["objectManager"] = c.objMgr.Snapshot() + return sp +} diff --git a/driver/controller/standard/sui/handler_change.go b/driver/controller/standard/sui/handler_change.go new file mode 100644 index 0000000..2496068 --- /dev/null +++ b/driver/controller/standard/sui/handler_change.go @@ -0,0 +1,69 @@ +package sui + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentChange struct { + controller.BaseHandlerAgent + + Filter sui.ObjectChangeFilter +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + checker := a.Filter.Checker() + for i, oc := range bd.mainData.ObjectChanges { + if !checker(oc) { + continue + } + // rawChange is the result of json marshal types.ObjectChange + // the required fields include: + // - objectId + // - digest + // - version + // - previousVersion (may not exist) + // - type (changeType) + // - owner + // - objectType + var rawChange string + if rawChange, err = bd.getChange(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT_CHANGE, + TxIndex: utils.Select(oc.TxIndex < 0, math.MaxInt, oc.TxIndex), + Data: &protos.Data{ + Value: &protos.Data_SuiObjectChange_{ + SuiObjectChange: &protos.Data_SuiObjectChange{ + RawChanges: []string{rawChange}, + Slot: bd.GetBlockNumber(), + TxDigest: oc.TxDigest.String(), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: len(rawChange), + }) + } + return +} + +func (a HandlerAgentChange) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/standard/sui/handler_event.go b/driver/controller/standard/sui/handler_event.go new file mode 100644 index 0000000..db80f19 --- /dev/null +++ b/driver/controller/standard/sui/handler_event.go @@ -0,0 +1,73 @@ +package sui + +import ( + "context" + "encoding/json" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.Check(tx) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, a.Filter.EventFilters, a.FetchConfig); err != nil { + return nil, err + } + eventChecker := sui.BuildEventChecker(a.Filter.EventFilters) + for evIndex, ev := range tx.Events { + if !eventChecker(ev) { + continue + } + var rawEvent []byte + if rawEvent, err = json.Marshal(ev); err != nil { + return nil, errors.Wrapf(err, "marshal sui event #%d in tx %d/%s in block %d failed", + evIndex, tx.TransactionPosition, tx.Digest.String(), bd.GetBlockNumber()) + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_EVENT, + TxIndex: txIndex, + TxInnerIndex: evIndex, + Data: &protos.Data{ + Value: &protos.Data_SuiEvent_{ + SuiEvent: &protos.Data_SuiEvent{ + RawEvent: string(rawEvent), + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawEvent) + len(rawTxn), + }) + } + } + return +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/sui/handler_function.go b/driver/controller/standard/sui/handler_function.go new file mode 100644 index 0000000..b81233c --- /dev/null +++ b/driver/controller/standard/sui/handler_function.go @@ -0,0 +1,58 @@ +package sui + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentFunction struct { + controller.BaseHandlerAgent + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.Check(tx) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, nil, a.FetchConfig); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + TxIndex: txIndex, + HandlerType: protos.HandlerType_SUI_CALL, + Data: &protos.Data{ + Value: &protos.Data_SuiCall_{ + SuiCall: &protos.Data_SuiCall{ + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: tx.Checkpoint.Uint64(), + }, + }, + }, + DataSize: len(rawTxn), + }) + } + return +} + +func (a HandlerAgentFunction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/sui/handler_interval.go b/driver/controller/standard/sui/handler_interval.go new file mode 100644 index 0000000..377b726 --- /dev/null +++ b/driver/controller/standard/sui/handler_interval.go @@ -0,0 +1,449 @@ +package sui + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/compress" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type ObjectDictSetManager struct { + mu sync.RWMutex + data map[string]ObjectDict + cachedData string + cachedBlockNumber *uint64 +} + +func (m *ObjectDictSetManager) Get(key string) *ObjectDict { + m.mu.RLock() + defer m.mu.RUnlock() + if od, has := m.data[key]; has { + return &od + } + return nil +} + +func (m *ObjectDictSetManager) Put(key string, value ObjectDict) { + m.mu.Lock() + defer m.mu.Unlock() + m.data[key] = value +} + +const CheckpointDataKey = "ObjectDictSetManager" + +func (m *ObjectDictSetManager) Load(checkpoint *controller.Checkpoint) error { + var raw string + if checkpoint != nil { + raw = checkpoint.Data[CheckpointDataKey] + } + return m.load(raw) +} + +// GetData use c.cachedData so the unsaved checkpoint will not dup the same data, can save memory +func (m *ObjectDictSetManager) GetData() string { + m.mu.RLock() + defer m.mu.RUnlock() + if len(m.data) == 0 { + return "" + } + var bn uint64 + for _, dict := range m.data { + bn = max(bn, dict.BlockNumber) + } + if m.cachedBlockNumber == nil || *m.cachedBlockNumber < bn { + m.cachedBlockNumber, m.cachedData = &bn, m.dump() + } + return m.cachedData +} + +func (m *ObjectDictSetManager) dump() string { + var buf bytes.Buffer + for key, dict := range m.data { + if buf.Len() > 0 { + buf.WriteRune('\n') + } + buf.WriteString(key) + buf.WriteRune('@') + buf.WriteString(dict.Dump()) + } + b, _ := compress.Dump(buf.String()) + return string(b) +} + +func (m *ObjectDictSetManager) load(data string) error { + var raw string + if err := compress.Load([]byte(data), &raw); err != nil { + return errors.Wrapf(err, "decompress data failed") + } + result := make(map[string]ObjectDict) + var line string + var blockNumber uint64 + for len(raw) > 0 { + line, raw, _ = strings.Cut(raw, "\n") + key, value, _ := strings.Cut(line, "@") + var dict ObjectDict + if err := dict.Load(value); err != nil { + return errors.Wrapf(err, "load object dictionary for %s failed", key) + } + blockNumber = max(blockNumber, dict.BlockNumber) + result[key] = dict + } + m.mu.Lock() + m.data = result + if len(result) > 0 { + m.cachedBlockNumber, m.cachedData = &blockNumber, data + } else { + m.cachedBlockNumber, m.cachedData = nil, "" + } + m.mu.Unlock() + return nil +} + +var objectDictPreviewCount = envconf.LoadUInt64("SENTIO_SUI_INTERVAL_HANDLER_OBJECT_DICT_PREVIEW_COUNT", 100) + +func (m *ObjectDictSetManager) Snapshot() any { + m.mu.RLock() + defer m.mu.RUnlock() + dataPreview := make(map[string]any) + for key, dict := range m.data { + dataPreview[key] = dict.Snapshot(int(objectDictPreviewCount)) + } + return dataPreview +} + +type ObjectDict struct { + BlockNumber uint64 + ObjectLatestVersion map[string]uint64 +} + +func (d *ObjectDict) Snapshot(previewCount int) any { + preview := make(map[string]uint64) + for objectID, latestVersion := range d.ObjectLatestVersion { + preview[objectID] = latestVersion + if len(preview) >= previewCount { + break + } + } + return map[string]any{ + "blockNumber": d.BlockNumber, + "size": len(d.ObjectLatestVersion), + "preview": preview, + } +} + +func (d *ObjectDict) Dump() string { + var buf bytes.Buffer + buf.WriteString(strconv.FormatUint(d.BlockNumber, 16)) + for _, objectID := range utils.GetOrderedMapKeys(d.ObjectLatestVersion) { + buf.WriteRune('#') + buf.WriteString(objectID) + buf.WriteRune(':') + buf.WriteString(strconv.FormatUint(d.ObjectLatestVersion[objectID], 16)) + } + return buf.String() +} + +func (d *ObjectDict) Load(raw string) (err error) { + var part string + part, raw, _ = strings.Cut(raw, "#") + if d.BlockNumber, err = strconv.ParseUint(part, 16, 64); err != nil { + return errors.Wrapf(err, "parse block number from %q failed", part) + } + d.ObjectLatestVersion = make(map[string]uint64) + for len(raw) > 0 { + part, raw, _ = strings.Cut(raw, "#") + objectID, ver, _ := strings.Cut(part, ":") + var version uint64 + if version, err = strconv.ParseUint(ver, 16, 64); err != nil { + return errors.Wrapf(err, "parse version for objectID %s from %q failed", objectID, ver) + } + d.ObjectLatestVersion[objectID] = version + } + return nil +} + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + Client sui.Client `json:"-"` // used to check address is a ERC20 address + + IntervalConfig data.IntervalConfig + Filter chainsui.ObjectChangeFilter + NeedSelf bool + UnwrapDynamicObject bool +} + +const ( + MaxObjectDictLen = 100000 + MaxQueryObjectChangeRangeSize = 10000 +) + +func (a HandlerAgentInterval) PushObjectLatestVersion( + ctx context.Context, + blockNumber uint64, + dict *ObjectDict, +) (ObjectDict, error) { + _, logger := log.FromContext(ctx, "handler", a.HandlerID.String(), "filter", utils.MustJSONMarshal(a.Filter)) + if dict != nil && dict.BlockNumber == blockNumber { // may be retried, so dict.BlockNumber may be equal to blockNumber + return *dict, nil + } + var from uint64 + result := ObjectDict{ + BlockNumber: blockNumber, + ObjectLatestVersion: make(map[string]uint64), + } + if dict != nil { + from = dict.BlockNumber + 1 + result.ObjectLatestVersion = utils.CopyMap(dict.ObjectLatestVersion) + } + logger.Infof("will push object latest version dict in [%d,%d]", from, blockNumber) + for from <= blockNumber { + startAt := time.Now() + end := min(blockNumber, from+MaxQueryObjectChangeRangeSize-1) + changes, err := a.Client.GetObjectChanges(ctx, from, end, a.Filter) + if err != nil { + return ObjectDict{}, err + } + beforeSize := len(result.ObjectLatestVersion) + var deleteCount int + var updateCount int + var createCount int + for _, bn := range utils.GetOrderedMapKeys(changes) { + for _, oc := range changes[bn] { + objectID, objectVersion := oc.ObjectID.String(), oc.Version.Uint64() + if oc.Type.IsDeleted() { + logger.Debugf("pushing and deleted object %s/%d", objectID, objectVersion) + delete(result.ObjectLatestVersion, objectID) + deleteCount++ + } else if preVersion, has := result.ObjectLatestVersion[objectID]; has { + logger.Debugf("pushing and updated object %s/%d=>%d", objectID, preVersion, objectVersion) + result.ObjectLatestVersion[objectID] = objectVersion + updateCount++ + } else { + logger.Debugf("pushing and created object %s/%d", objectID, objectVersion) + result.ObjectLatestVersion[objectID] = objectVersion + createCount++ + } + } + } + logger.With("used", time.Since(startAt).String()). + Infof("pushed object latest version dict in [%d,%d], size %d => %d, created %d and updated %d and deleted %d", + from, end, beforeSize, len(result.ObjectLatestVersion), createCount, updateCount, deleteCount) + from = end + 1 + } + if size := len(result.ObjectLatestVersion); size > MaxObjectDictLen { + err := errors.Errorf("object latest version dict size for handler %s with filter %s is too big: %d > %d", + a.HandlerID.String(), utils.MustJSONMarshal(a.Filter), size, MaxObjectDictLen) + return ObjectDict{}, fetcher.Permanent(err) + } + return result, nil +} + +type objectDetails struct { + Version string `json:"version"` + Digest string `json:"digest"` + Content json.RawMessage `json:"content"` + Type string `json:"type"` +} + +type object struct { + Version uint64 + Digest string + Content json.RawMessage +} + +var ignoreNotExistObject = envconf.LoadBool("SENTIO_SUI_INTERVAL_HANDLER_IGNORE_NON_EXISTENT_OBJECTS", true) + +var dynamicType = types.TypeTagFromStringMust( + "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper, 0x2::object::ID>") + +func (a HandlerAgentInterval) ObjMgrKey() string { + return utils.MustJSONMarshal(a.Filter) +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return + } + + dict := bd.objMgr.Get(a.ObjMgrKey()) + if dict == nil { + return + } + + _, logger := log.FromContext(ctx) + var opts = types.SuiObjectDataOptions{ + ShowContent: true, + ShowType: true, + } + var requests []types.SuiGetPastObjectRequest + for objectID, version := range dict.ObjectLatestVersion { + requests = append(requests, types.SuiGetPastObjectRequest{ + ObjectID: types.StrToObjectIDMust(objectID), + Version: types.Uint64ToNumber(version), + }) + } + contents := make(map[string]object) + for len(requests) > 0 { + var resp []types.SuiPastObjectResponse + if resp, err = a.Client.TryMultiGetPastObjects(ctx, requests, opts); err != nil { + return + } + var wrappedObjectIDList []string + for i, obj := range resp { + req := requests[i] + switch obj.Status { + case types.SuiPastObjectStatusVersionFound: + var details objectDetails + if err = json.Unmarshal(obj.Details, &details); err != nil { + return nil, errors.Wrapf(err, "object %s version %d unmarshal details failed", + req.ObjectID, req.Version.Uint64()) + } + objectType, _ := types.TypeTagFromString(details.Type) + if a.UnwrapDynamicObject && objectType != nil && dynamicType.Include(*objectType) { + var moveObj struct { + Fields struct { + Value string `json:"value"` + } `json:"fields"` + } + if err = json.Unmarshal(details.Content, &moveObj); err != nil { + return nil, errors.Wrapf(err, "object %s version %d unmarshal move object failed", + req.ObjectID, req.Version.Uint64()) + } + wrappedObjectIDList = append(wrappedObjectIDList, moveObj.Fields.Value) + } else { + contents[req.ObjectID.String()] = object{ + Version: req.Version.Uint64(), + Digest: details.Digest, + Content: details.Content, + } + } + case types.SuiPastObjectStatusObjectDeleted: + default: + message := fmt.Sprintf("object %s version %d has unexpected status %s", + req.ObjectID, req.Version.Uint64(), obj.Status) + if !ignoreNotExistObject { + return nil, errors.Errorf(message) + } + logger.Warnf("%s, will be ignored", message) + } + } + requests = requests[:0] + if len(wrappedObjectIDList) > 0 { + var stat []chainsui.ObjectStat + if stat, err = a.Client.GetObjectsStat(ctx, 0, bd.GetBlockNumber(), wrappedObjectIDList); err != nil { + return nil, err + } + for i, objectID := range wrappedObjectIDList { + if stat[i].Count > 0 { + requests = append(requests, types.SuiGetPastObjectRequest{ + ObjectID: types.StrToObjectIDMust(objectID), + Version: types.Uint64ToNumber(stat[i].MaxObjectVersion), + }) + } else { + logger.Warnf("Object %s has no history", objectID) + } + } + } + } + + if a.Filter.TypePattern != nil { + for objectID, obj := range contents { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: utils.WrapPointer(string(obj.Content)), + ObjectId: objectID, + ObjectVersion: obj.Version, + ObjectDigest: obj.Digest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(obj.Content), + }) + } + } + if a.Filter.OwnerFilter != nil { + var dataSize int + var owned []string + var self *object + for objectID, obj := range contents { + if utils.IndexOf(a.Filter.OwnerFilter.OwnerID, objectID) < 0 { + owned = append(owned, string(obj.Content)) + dataSize += len(obj.Content) + } else { + self = utils.WrapPointer(contents[objectID]) + } + } + var selfContent *string + var selfVersion uint64 + var selfDigest string + if self != nil { + selfContent = utils.WrapPointer(string(self.Content)) + selfVersion = self.Version + selfDigest = self.Digest + dataSize += len(self.Content) + } else if a.NeedSelf { + // need self but self not exist + return + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: selfContent, + ObjectId: a.Filter.OwnerFilter.OwnerID[0], + ObjectVersion: selfVersion, + ObjectDigest: selfDigest, + RawObjects: owned, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: dataSize, + }) + } + return +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "Filter": a.Filter, + "NeedSelf": a.NeedSelf, + "UnwrapDynamicObject": a.UnwrapDynamicObject, + } +} diff --git a/driver/controller/standard/sui/handler_interval_test.go b/driver/controller/standard/sui/handler_interval_test.go new file mode 100644 index 0000000..c4892fe --- /dev/null +++ b/driver/controller/standard/sui/handler_interval_test.go @@ -0,0 +1,50 @@ +package sui + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_loadDump(t *testing.T) { + m0 := ObjectDictSetManager{} + assert.NoError(t, m0.load("")) + + m0.Put("x1", ObjectDict{ + BlockNumber: 100, + ObjectLatestVersion: map[string]uint64{ + "0x1": 100, + }, + }) + + m0.Put("x2", ObjectDict{ + BlockNumber: 101, + ObjectLatestVersion: map[string]uint64{ + "0x21": 999, + "0x22": 0, + }, + }) + + x3 := ObjectDict{ + BlockNumber: 100000000, + ObjectLatestVersion: map[string]uint64{}, + } + for i := 0; i < 100000; i++ { + id := fmt.Sprintf("0x%016x%016x%016x%016x", rand.Uint64(), rand.Uint64(), rand.Uint64(), rand.Uint64()) + ver := rand.Uint64() + x3.ObjectLatestVersion[id] = ver + } + m0.Put("x3", x3) + + d0 := m0.GetData() + t.Logf("data0: %d", len(d0)) + + var m1 ObjectDictSetManager + assert.NoError(t, m1.load(d0)) + + assert.Equal(t, m0.data, m1.data) + assert.Equal(t, m0.cachedBlockNumber, m1.cachedBlockNumber) + assert.Equal(t, m0.cachedData, m1.cachedData) +} diff --git a/driver/controller/standard/task.go b/driver/controller/standard/task.go new file mode 100644 index 0000000..ae66e73 --- /dev/null +++ b/driver/controller/standard/task.go @@ -0,0 +1,658 @@ +package standard + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type partitionWithIndex struct { + partition string + index uint64 +} +type waiter struct { + ready *concurrency.ResourceWaiter[uint64] + finish *concurrency.ResourceWaiter[partitionWithIndex] +} + +type task struct { + bindingData + + sp streamPool + waiter *waiter + + metricConfigs map[timeseries.MetaType]map[string]*protos.MetricConfig + webhookChannels map[string]string + chainID string + processor *models.Processor + + stream protos.ProcessorV3_ProcessBindingsStreamClient + index controller.TaskIndex + partition string + + timer *timer.Timer + + logger *log.SentioLogger +} + +func (b *task) errLogger() *log.SentioLogger { + return b.logger.With("binding", b.data.String()) +} + +func (b *task) Init(ctx context.Context, index controller.TaskIndex, progressbar controller.ProgressBar) { + b.index = index + if enableBindingDataPartition { + b.waiter.ready.NewResource(b.index.Global) + } else { + b.waiter.finish.NewResource(partitionWithIndex{index: b.index.Global}) + } + b.timer = timer.NewTimer() + _, b.logger = log.FromContext(ctx, + "block", controller.GetBlockSummary(b), + "latest", controller.GetBlockSummary(progressbar.LatestBlock), + "index", index, + "handler", b.handlerID.String()) +} + +func (b *task) GetHandlerID() controller.HandlerID { + return b.handlerID +} + +func (b *task) title() string { + return fmt.Sprintf("#%d binding data %d/%d for handler %s in block %s", + b.index.Global, b.index.InBlock, b.index.TotalInBlock, b.handlerID, controller.GetBlockSummary(b)) +} + +func (b *task) Summary() string { + return b.title() +} + +func (b *task) Exec( + ctx context.Context, + checkpointCtrl controller.CheckpointController, +) (extErr *controller.ExternalError) { + b.logger.Debug("task started") + start := b.timer.Start("ALL") + hmi := controller.TaskInfo{ + Processor: b.processor, + ChainID: b.chainID, + Handler: b.handlerID.Name, + Category: b.handlerID.Type, + DataSource: b.handlerID.DataSource, + } + var stat statistic + ctx = controller.N.BeforeEntityOperation(ctx, hmi) // to pass metric attrs to noticeController of entityController + defer func() { + used := start.End() + usedReport := b.timer.ReportDistribution("ALL", "*") + b.logger = b.logger.With("used", usedReport, "stat", stat) + if extErr == nil { + b.logger.Debugw("task succeed") + b.waiter.finish.ResourceReady(partitionWithIndex{partition: b.partition, index: b.index.Global}) + } else if errors.Is(extErr.Wrapped(), context.Canceled) { + b.logger.Warnfe(extErr, "task canceled") + } else { + b.errLogger().Errore(extErr, "task failed") + } + controller.N.TaskDone(ctx, hmi, extErr == nil, used) + for eventName, count := range stat.TimeSeries[timeseries.MetaTypeEvent] { + controller.N.DataEmitted(ctx, hmi, "event", "", eventName, int64(count)) + } + for counterName, count := range stat.TimeSeries[timeseries.MetaTypeCounter] { + controller.N.DataEmitted(ctx, hmi, "metric", "counter", counterName, int64(count)) + } + for gaugeName, count := range stat.TimeSeries[timeseries.MetaTypeGauge] { + controller.N.DataEmitted(ctx, hmi, "metric", "gauge", gaugeName, int64(count)) + } + for subtype, st := range stat.Entity { + for entityName, count := range st { + controller.N.DataEmitted(ctx, hmi, "entity", subtype, entityName, int64(count)) + } + } + }() + if extErr = b.getStream(ctx); extErr != nil { + return extErr + } + defer b.returnStream() + if extErr = b.sendBindingData(ctx); extErr != nil { + return extErr + } + if enableBindingDataPartition { + if extErr = b.recvPartition(ctx); extErr != nil { + return extErr + } + b.waiter.finish.NewResource(partitionWithIndex{partition: b.partition, index: b.index.Global}) + b.waiter.ready.ResourceReady(b.index.Global) + err := b.waiter.ready.Wait(ctx, func(u uint64) bool { + return u < b.index.Global + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("waiting all previous tasks got partition failed: %v", err)) + } + if extErr = b.sendStartCommand(ctx); extErr != nil { + return extErr + } + } + stat, extErr = b.waitProcess(ctx, checkpointCtrl) + return extErr +} + +func (b *task) getStream(ctx context.Context) *controller.ExternalError { + select { + case b.stream = <-b.sp: + return nil + case <-ctx.Done(): + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("get stream for %s failed: %v", b.title(), ctx.Err())) + } +} + +func (b *task) returnStream() { + if b.stream != nil { + b.sp <- b.stream + } +} + +func (b *task) streamSend( + ctx context.Context, + req *protos.ProcessStreamRequest, + what, mark string, + timeout time.Duration, +) *controller.ExternalError { + start := b.timer.Start(mark) + err := timer.Wait(ctx, timeout, time.Minute, func() error { + return b.stream.Send(req) + }, func(used time.Duration) { + b.logger.Warnf("stream send %s already waited %s", what, used.String()) + }) + if err != nil { + if errors.Is(err, context.Canceled) { + b.logger.Warnfe(err, "stream send %s canceled", what) + } else { + b.errLogger().Errorfe(err, "stream send %s failed", what) + } + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Wrapf(err, "send %s for %s failed", what, b.title())) + } + used := start.End() + b.logger.With("used", used.String()).Debugf("stream sent %s", what) + return nil +} + +func (b *task) streamReceive( + ctx context.Context, + mark string, + timeout time.Duration, +) (*protos.ProcessStreamResponseV3, *controller.ExternalError) { + defer b.timer.Start(mark).End() + var resp *protos.ProcessStreamResponseV3 + err := timer.Wait(ctx, timeout, time.Minute, func() (recvErr error) { + resp, recvErr = b.stream.Recv() + return recvErr + }, func(used time.Duration) { + b.logger.Warnf("stream receive already waited %s", used.String()) + }) + if err != nil { + if errors.Is(err, context.Canceled) { + b.logger.Warnfe(err, "stream receive canceled") + } else { + b.errLogger().Errorfe(err, "stream receive failed") + } + return nil, controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Wrapf(err, "receive for %s failed", b.title())) + } + if uint64(resp.GetProcessId()) != b.index.Global { + return nil, controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("unexpected ProcessID #%d for %s", resp.GetProcessId(), b.title())) + } + return resp, nil +} + +func (b *task) sendBindingData(ctx context.Context) *controller.ExternalError { + return b.streamSend(ctx, &protos.ProcessStreamRequest{ + ProcessId: int32(b.index.Global), + Value: &protos.ProcessStreamRequest_Binding{Binding: b.data}, + }, "binding data", "SB", time.Minute*30) +} + +func (b *task) recvPartition(ctx context.Context) *controller.ExternalError { + resp, extErr := b.streamReceive(ctx, "RP", time.Minute*30) + if extErr != nil { + return extErr + } + for _, p := range resp.GetPartitions().GetPartitions() { + b.partition = p.GetUserValue() + } + return nil +} + +func (b *task) sendStartCommand(ctx context.Context) *controller.ExternalError { + return b.streamSend(ctx, &protos.ProcessStreamRequest{ + ProcessId: int32(b.index.Global), + Value: &protos.ProcessStreamRequest_Start{Start: true}, + }, "start command", "SS", time.Minute*30) +} + +type statistic struct { + Get int + List int + ListEntities int + Upsert int + UpsertEntities int + Update int + UpdateEntities int + Delete int + DeleteEntities int + + Entity map[string]map[string]int // Entity[op][name] + TimeSeries map[timeseries.MetaType]map[string]int + Export map[string]int +} + +func (b *task) waitProcess( + ctx context.Context, + checkpointCtrl controller.CheckpointController, +) (statistic, *controller.ExternalError) { + var stat statistic + stat.Entity = make(map[string]map[string]int) + stat.TimeSeries = make(map[timeseries.MetaType]map[string]int) + stat.Export = make(map[string]int) + for { + resp, extErr := b.streamReceive(ctx, "RR", time.Minute*30) + if extErr != nil { + return stat, extErr + } + if resp.GetResult() != nil { + if errMsg := resp.GetResult().GetStates().GetError(); errMsg != "" { + b.errLogger().Errorf("got error in final result: %s", errMsg) + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("got error in %s: %s", b.title(), errMsg)) + } + // Event v2 and metric (gauge/counter) v2 data are no longer supported by + // the streaming driver (driver v3+). A processor that still emits them is + // running an unsupported SDK and must be rejected instead of silently + // dropping the data. + if len(resp.GetResult().GetEvents()) > 0 { + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("event v2 data is no longer supported in %s, please upgrade the processor SDK", b.title())) + } + if len(resp.GetResult().GetGauges()) > 0 || len(resp.GetResult().GetCounters()) > 0 { + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("metric v2 data is no longer supported in %s, please upgrade the processor SDK", b.title())) + } + if tsr := resp.GetResult().GetTimeseriesResult(); len(tsr) > 0 { + b.logger.Debugf("got %d time series data in final process result", len(tsr)) + if data, convertErr := b.ConvertTimeSeriesData(tsr); convertErr != nil { + b.errLogger().Errorfe(convertErr, "convert time series data failed") + return stat, convertErr.Wrapf("invalid time series data in %s", b.title()) + } else { + checkpointCtrl.InsertTimeSeriesData(b.GetBlockNumber(), b.index, data) + timeseries.Statistic(data, stat.TimeSeries) + } + } + if epr := resp.GetResult().GetExports(); len(epr) > 0 { + b.logger.Debugf("got %d export data in final process result", len(epr)) + data := b.ConvertExportData(epr) + checkpointCtrl.InsertWebhookData(b.GetBlockNumber(), b.index, data) + controller.StatisticWebhookMessages(data, stat.Export) + } + return stat, nil + } + + if req := resp.GetTplRequest(); req != nil { + b.logger.Debugf("got %d templates", len(req.GetTemplates())) + extErr = checkpointCtrl.NewTemplateInstance(ctx, b, ConvertTemplateInstance(req.GetTemplates(), req.GetRemove())) + if extErr != nil { + return stat, extErr.Wrapf("add template instance in %s failed", b.title()) + } + continue + } + if req := resp.GetTsRequest(); req != nil { + b.logger.Debugf("got %d time series data", len(req.GetData())) + if data, convertErr := b.ConvertTimeSeriesData(req.GetData()); convertErr != nil { + b.errLogger().Errorfe(convertErr, "convert time series data failed: %v", req.GetData()) + return stat, convertErr.Wrapf("invalid time series data in %s", b.title()) + } else { + checkpointCtrl.InsertTimeSeriesData(b.GetBlockNumber(), b.index, data) + timeseries.Statistic(data, stat.TimeSeries) + } + continue + } + + // must be an db request + dbReq := resp.GetDbRequest() + dbResp := protos.DBResponse{OpId: dbReq.GetOpId()} + reqLogger := b.logger.With("opid", dbReq.GetOpId()) + reqErrLogger := func() *log.SentioLogger { + return reqLogger.With("binding", b.data.String(), "dbreq", dbReq.String()) + } + + // wait resource + if dbReq.GetGet() != nil || dbReq.GetList() != nil { + wr := b.timer.Start("WR") + err := b.waiter.finish.Wait(ctx, func(p partitionWithIndex) bool { + return p.partition == b.partition && p.index < b.index.Global + }) + if err != nil { + return stat, controller.NewExternalError(controller.ErrCodeSystem, + errors.Wrapf(err, "wait previous task finish for %s failed", b.title())) + } + waitUsed := wr.End() + reqLogger = reqLogger.With("waitUsed", waitUsed.String()) + } + + // do the db operation + var what string + start := b.timer.Start("DE") + if dbReq.GetGet() != nil { + entity, id := dbReq.GetGet().GetEntity(), dbReq.GetGet().GetId() + reqLogger = reqLogger.With("dbop", "get", "entity", entity, "id", id) + entityType := checkpointCtrl.GetEntityOrInterfaceType(entity) + if entityType == nil { + reqErrLogger().Errorf("get unknown entity") + return stat, controller.NewExternalError(controller.ErrCodeGetUnknownEntity, + errors.Errorf("get unknown entity %q for %s", entity, b.title())) + } + box, getErr := checkpointCtrl.GetEntity(ctx, entityType, id, b.GetBlockNumber()) + if getErr != nil { + reqErrLogger().Errorfe(getErr, "get entity failed") + return stat, getErr.Wrapf("get entity %s/%s in %s failed", entity, id, b.title()) + } + if box != nil && box.Data != nil { + data, convertErr := box.ToRichStruct(entityType) + if convertErr != nil { + reqErrLogger().Errorfe(getErr, "convert entity to RichStruct failed") + return stat, controller.NewExternalError(controller.ErrCodeInvalidEntityData, + errors.Wrapf(convertErr, "convert entity %s/%s in %s failed", entity, id, b.title())) + } + dbResp.Value = &protos.DBResponse_EntityList{ + EntityList: &protos.EntityList{ + Entities: []*protos.Entity{{ + Entity: box.Entity, + GenBlockNumber: box.GenBlockNumber, + GenBlockTime: timestamppb.New(box.GenBlockTime), + GenBlockChain: b.chainID, + Data: data, + }}, + }, + } + } else { + // not created or deleted, return an empty list + dbResp.Value = &protos.DBResponse_EntityList{ + EntityList: &protos.EntityList{}, + } + } + stat.Get++ + what = fmt.Sprintf("entity get response #%d", stat.Get) + } + if dbReq.GetList() != nil { + const defaultPageSize = 10000 + entity := dbReq.GetList().GetEntity() + pageSize := int(dbReq.GetList().GetPageSize()) + if pageSize <= 0 { + pageSize = defaultPageSize + } + reqLogger = reqLogger.With( + "dbop", "list", + "entity", entity, + "cursor", dbReq.GetList().GetCursor(), + "pageSize", pageSize, + "filters", dbReq.GetList().GetFilters()) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("list unknown entity") + return stat, controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("list unknown entity %q for %s", entity, b.title())) + } + filters := make([]persistent.EntityFilter, len(dbReq.GetList().GetFilters())) + for fi, ft := range dbReq.GetList().GetFilters() { + field := entityType.GetFieldByName(ft.GetField()) + if field == nil { + reqErrLogger().Errorf("field %s.%s in entity list filter is not exist", entityType.Name, ft.GetField()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s.%s in entity list filter is not exist for %s", + entityType.Name, ft.GetField(), b.title())) + } + if entityType.GetForeignKeyFieldByName(ft.GetField()).IsReverseField() { + reqErrLogger().Errorf("field %s.%s in entity list filter is a reverse foreign key", + entityType.Name, ft.GetField()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s.%s in entity list filter is a reverse foreign key for %s", + entityType.Name, ft.GetField(), b.title())) + } + fieldTitle := fmt.Sprintf("%s.%s %s", entityType.Name, ft.GetField(), field.Type.String()) + filters[fi] = persistent.EntityFilter{Field: field} + switch ft.GetOp() { + case protos.DBRequest_EQ: + filters[fi].Op = persistent.EntityFilterOpEq + case protos.DBRequest_NE: + filters[fi].Op = persistent.EntityFilterOpNe + case protos.DBRequest_GT: + filters[fi].Op = persistent.EntityFilterOpGt + case protos.DBRequest_GE: + filters[fi].Op = persistent.EntityFilterOpGe + case protos.DBRequest_LT: + filters[fi].Op = persistent.EntityFilterOpLt + case protos.DBRequest_LE: + filters[fi].Op = persistent.EntityFilterOpLe + case protos.DBRequest_IN: + filters[fi].Op = persistent.EntityFilterOpIn + case protos.DBRequest_NOT_IN: + filters[fi].Op = persistent.EntityFilterOpNotIn + case protos.DBRequest_LIKE: + filters[fi].Op = persistent.EntityFilterOpLike + case protos.DBRequest_NOT_LIKE: + filters[fi].Op = persistent.EntityFilterOpNotLike + case protos.DBRequest_HAS_ALL: + filters[fi].Op = persistent.EntityFilterOpHasAll + case protos.DBRequest_HAS_ANY: + filters[fi].Op = persistent.EntityFilterOpHasAny + default: + reqErrLogger().Errorf("unknown list filter op %s", ft.GetOp()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("unknown list filter operator %v for %s", ft.GetOp(), b.title())) + } + filterValueType := field.Type + if ft.GetOp() == protos.DBRequest_HAS_ALL || ft.GetOp() == protos.DBRequest_HAS_ANY { + fieldType := schema.BreakType(field.Type) + if fieldType.CountListLayer() != 1 { + reqErrLogger().Errorf("field %s cannot use op %s", fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s cannot use op %s for %s", fieldTitle, ft.GetOp().String(), b.title())) + } + filterValueType = schema.BreakType(field.Type).SkipListLayer(1).Join() + } + for _, val := range ft.GetValue().GetValues() { + value, convertErr := persistent.FromRichValue(val, filterValueType) + if convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert filter value %v for field %s with op %s to go value failed", + val, fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Wrapf(convertErr, "convert filter value %v for field %s with op %s to go value for %s failed", + val, fieldTitle, ft.GetOp().String(), b.title())) + } + filters[fi].Value = append(filters[fi].Value, value) + } + if initErr := filters[fi].Init(); initErr != nil { + reqErrLogger().Errorfe(initErr, "init filter for field %s with op %s", fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Wrapf(initErr, "init filter for field %s with op %s for %s failed", + fieldTitle, ft.GetOp().String(), b.title())) + } + } + boxes, nextCursor, listErr := checkpointCtrl.ListEntity( + ctx, entityType, filters, dbReq.GetList().GetCursor(), pageSize, b.GetBlockNumber()) + if listErr != nil { + reqErrLogger().Errorfe(listErr, "list entity failed") + return stat, listErr.Wrapf("list entity for %s failed", b.title()) + } + data := &protos.EntityList{Entities: make([]*protos.Entity, len(boxes))} + for k, box := range boxes { + one, convertErr := box.ToRichStruct(entityType) + if convertErr != nil { + reqErrLogger().With("box", box.String()).Errorfe(convertErr, "convert entity to RichStruct failed") + return stat, controller.NewExternalError(controller.ErrCodeInvalidEntityData, + errors.Wrapf(convertErr, "convert entity %s to RichStruct for %s failed ", box.String(), b.title())) + } + data.Entities[k] = &protos.Entity{ + Entity: box.Entity, + GenBlockNumber: box.GenBlockNumber, + GenBlockTime: timestamppb.New(box.GenBlockTime), + GenBlockChain: b.chainID, + Data: one, + } + } + dbResp.Value = &protos.DBResponse_EntityList{EntityList: data} + dbResp.NextCursor = nextCursor + stat.List++ + stat.ListEntities += len(boxes) + what = fmt.Sprintf("entity list response #%d/%d", stat.List, stat.ListEntities) + } + if dbReq.GetUpsert() != nil { + reqLogger = reqLogger.With("dbop", "upsert", "count", len(dbReq.GetUpsert().GetId())) + entities := dbReq.GetUpsert().GetEntity() + ids := dbReq.GetUpsert().GetId() + datas := dbReq.GetUpsert().GetEntityData() + if len(ids) != len(datas) || len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal", + len(entities), len(ids), len(datas)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpsertEntityRequest, + errors.Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal in db upsert request for %s", + len(entities), len(ids), len(datas), b.title())) + } + for k := range ids { + entity, id, data := entities[k], ids[k], datas[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("set unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeUpsertUnknownEntity, + errors.Errorf("set unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if convertErr := box.FromRichStruct(entityType, data); convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert entity %s/%s failed", summary, data.String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpsertEntityRequest, + errors.Wrapf(convertErr, "convert entity %s/%s failed for %s failed", summary, data.String(), b.title())) + } + if setErr := checkpointCtrl.SetEntity(ctx, entityType, box); setErr != nil { + reqErrLogger().Errorfe(setErr, "set entity %s %s failed", summary, box.String()) + return stat, setErr.Wrapf("set entity %s %s for %s failed", summary, box.String(), b.title()) + } + reqLogger.Debugf("set entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "upsert", entity, 1) + } + stat.Upsert++ + stat.UpsertEntities += len(ids) + what = fmt.Sprintf("entity upsert response #%d/%d", stat.Upsert, stat.UpsertEntities) + } + if dbReq.GetUpdate() != nil { + reqLogger = reqLogger.With("dbop", "update", "count", len(dbReq.GetUpdate().GetId())) + entities := dbReq.GetUpdate().GetEntity() + ids := dbReq.GetUpdate().GetId() + datas := dbReq.GetUpdate().GetEntityData() + if len(ids) != len(datas) || len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal", + len(entities), len(ids), len(datas)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpdateEntityRequest, + errors.Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal in db update request for %s", + len(entities), len(ids), len(datas), b.title())) + } + for k := range ids { + entity, id, data := entities[k], ids[k], datas[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("set unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeUpdateUnknownEntity, + errors.Errorf("set unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if convertErr := box.FromEntityUpdateData(entityType, data); convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert entity %s/%s failed", summary, data.String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpdateEntityRequest, + errors.Wrapf(convertErr, "convert entity %s/%s failed for %s failed", summary, data.String(), b.title())) + } + if setErr := checkpointCtrl.SetEntity(ctx, entityType, box); setErr != nil { + reqErrLogger().Errorfe(setErr, "set entity %s %s failed", summary, box.String()) + return stat, setErr.Wrapf("set entity %s %s for %s failed", summary, box.String(), b.title()) + } + reqLogger.Debugf("set entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "update", entity, 1) + } + stat.Update++ + stat.UpdateEntities += len(ids) + what = fmt.Sprintf("entity update response #%d/%d", stat.Update, stat.UpdateEntities) + } + if dbReq.GetDelete() != nil { + reqLogger = reqLogger.With("dbop", "delete", "count", len(dbReq.GetDelete().GetId())) + entities, ids := dbReq.GetDelete().GetEntity(), dbReq.GetDelete().GetId() + if len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id) = %d,%d must be equal", len(entities), len(ids)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidDeleteEntityRequest, + errors.Errorf("len(entity),length(id) = %d,%d must be equal in db delete request for %s", + len(entities), len(ids), b.title())) + } + for k := range ids { + entity, id := entities[k], ids[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("delete unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeDeleteUnknownEntity, + errors.Errorf("delete unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if delErr := checkpointCtrl.SetEntity(ctx, entityType, box); delErr != nil { + reqErrLogger().Errorfe(delErr, "delete entity %s failed", summary) + return stat, delErr.Wrapf("delete entity %s for %s failed", summary, b.title()) + } + reqLogger.Debugf("delete entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "delete", entity, 1) + } + stat.Delete++ + stat.DeleteEntities += len(ids) + what = fmt.Sprintf("entity delete response #%d/%d", stat.Delete, stat.DeleteEntities) + } + reqLogger.With("used", start.End().String()).Debugf("%s is ready", what) + + // send db result + req := &protos.ProcessStreamRequest{ + ProcessId: resp.GetProcessId(), + Value: &protos.ProcessStreamRequest_DbResult{DbResult: &dbResp}, + } + if sendErr := b.streamSend(ctx, req, what, "SE", time.Minute*30); sendErr != nil { + return stat, sendErr + } + } +} diff --git a/driver/controller/startup/BUILD.bazel b/driver/controller/startup/BUILD.bazel new file mode 100644 index 0000000..20494da --- /dev/null +++ b/driver/controller/startup/BUILD.bazel @@ -0,0 +1,89 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "startup", + srcs = [ + "checkpoint.go", + "entity.go", + "quota.go", + "standard.go", + "startup.go", + "subgraph.go", + "timeseries.go", + "utils.go", + "webhook.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/startup", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//chain/sui/types", + "//common/chains", + "//common/chx", + "//common/clickhousemanager", + "//common/compress", + "//common/concurrency", + "//common/envconf", + "//common/errgroup", + "//common/gonanoid", + "//common/log", + "//common/protojson", + "//common/set", + "//common/sparsify", + "//common/tracker", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/aptos", + "//driver/controller/data/evm", + "//driver/controller/data/fuel", + "//driver/controller/data/sol", + "//driver/controller/data/sui", + "//driver/controller/standard", + "//driver/controller/standard/aptos", + "//driver/controller/standard/evm", + "//driver/controller/standard/fuel", + "//driver/controller/standard/sol", + "//driver/controller/standard/sui", + "//driver/controller/standard/sui/grpc", + "//driver/controller/subgraph", + "//driver/entity/clickhouse", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/exitcode", + "//driver/subgraph/manifest", + "//driver/timeseries", + "//driver/timeseries/clickhouse", + "//processor/protos", + "//service/common/errors", + "//service/common/models", + "//service/common/rpc", + "//service/database_registry/protos", + "//service/processor/models", + "//service/processor/protos", + "//service/usage/protos", + "//service/webhook/protos", + "@com_github_clickhouse_clickhouse_go_v2//:clickhouse-go", + "@com_github_ipfs_go_ipfs_api//:go-ipfs-api", + "@com_github_pkg_errors//:errors", + "@com_google_cloud_go_pubsub//:pubsub", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//encoding/gzip", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "startup_test", + srcs = ["startup_test.go"], + embed = [":startup"], + deps = [ + "//common/log", + "//service/common/errors", + "//service/processor/models", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/startup/checkpoint.go b/driver/controller/startup/checkpoint.go new file mode 100644 index 0000000..d73b5b6 --- /dev/null +++ b/driver/controller/startup/checkpoint.go @@ -0,0 +1,191 @@ +package startup + +import ( + "context" + "encoding/json" + "math" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/compress" + "sentioxyz/sentio-core/driver/controller" + sentioerror "sentioxyz/sentio-core/service/common/errors" + "sentioxyz/sentio-core/service/processor/models" + "sentioxyz/sentio-core/service/processor/protos" +) + +type checkpointStore struct { + processor *models.Processor + chainID string + cli protos.ProcessorServiceClient + chainState *models.ChainState + latestCheckpoint *controller.Checkpoint +} + +func newCheckpointStore( + processor *models.Processor, + chainID string, + cli protos.ProcessorServiceClient, + chainState *models.ChainState, +) *checkpointStore { + return &checkpointStore{ + processor: processor, + chainID: chainID, + cli: cli, + chainState: chainState, + } +} + +func LoadCheckpoints(cs *models.ChainState) (checkpoints []controller.Checkpoint, err error) { + if len(cs.IndexerState) == 0 { + return nil, nil + } + if err = compress.Load(cs.IndexerState, &checkpoints); err != nil { + return nil, errors.Wrap(err, "unmarshal checkpoints failed") + } + return checkpoints, nil +} + +func LoadTemplates(cs *models.ChainState) (templates map[uint64][]controller.TemplateInstance, err error) { + if len(cs.Templates) == 0 { + return make(map[uint64][]controller.TemplateInstance), nil + } + if err = compress.Load([]byte(cs.Templates), &templates); err != nil { + return nil, errors.Wrapf(err, "unmarshal templates for chain %s failed", cs.ChainID) + } + return templates, nil +} + +func SetCheckpoints(cs *models.ChainState, checkpoints []controller.Checkpoint) error { + // set indexer_state + b, err := compress.Dump(checkpoints) + if err != nil { + return errors.Wrapf(err, "dump checkpoints failed") + } + if len(checkpoints) > 2 && len(b) > maxIndexerStateSize { + return errors.Wrapf(controller.ErrCheckpointsTooBig, + "length of IndexerState in %d checkpoints is %d, over the limit %d", + len(checkpoints), len(b), maxIndexerStateSize) + } + cs.IndexerState = b + // set progress and state + cs.ProcessedBlockNumber = -1 + cs.ProcessedBlockHash = "" + cs.ProcessedTimestampMicros = 0 + cs.InitialStartBlockNumber = 0 + cs.EstimatedLatestBlockNumber = math.MaxInt64 + cs.LastBlockNumber = 0 + cs.State = int32(protos.ChainState_Status_CATCHING_UP) + if len(checkpoints) > 0 { + last := checkpoints[len(checkpoints)-1] + cs.ProcessedBlockNumber = int64(last.BlockNumber) + cs.ProcessedBlockHash = last.BlockHash + cs.ProcessedTimestampMicros = last.BlockTime.UnixMicro() + cs.InitialStartBlockNumber = int64(last.FullBlockRange.StartBlock) + cs.EstimatedLatestBlockNumber = int64(last.CurrentLastBlockNumber()) + if last.FullBlockRange.EndBlock != nil { + cs.LastBlockNumber = int64(*last.FullBlockRange.EndBlock) + } + if last.InWatching() || last.AllDone() { + cs.State = int32(protos.ChainState_Status_PROCESSING_LATEST) + } + } + // remove error + cs.ErrorRecord = sentioerror.ErrorRecord{} + return nil +} + +func SetTemplates(cs *models.ChainState, templates map[uint64][]controller.TemplateInstance) error { + b, err := compress.Dump(templates) + if err != nil { + return errors.Wrapf(err, "dump templates failed") + } + cs.Templates = string(b) + return nil +} + +func SetHandlerStat(cs *models.ChainState, agentStat map[string]int) error { + b, err := json.Marshal(agentStat) + if err != nil { + return errors.Wrapf(err, "dump agent stat failed") + } + cs.HandlerStat = b + return nil +} + +func (s *checkpointStore) Load(ctx context.Context) ( + checkpoints []controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + err error, +) { + if checkpoints, err = LoadCheckpoints(s.chainState); err != nil { + return nil, nil, err + } + if templates, err = LoadTemplates(s.chainState); err != nil { + return nil, nil, err + } + if len(checkpoints) > 0 { + s.latestCheckpoint = &checkpoints[len(checkpoints)-1] + } + return +} + +const maxIndexerStateSize = 50 * 1024 * 1024 // 50MB + +func (s *checkpointStore) Save( + ctx context.Context, + checkpoints []controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + agentStat map[string]int, +) error { + var latest *controller.Checkpoint + if len(checkpoints) > 0 { + latest = &checkpoints[len(checkpoints)-1] + } + reorg := (latest == nil && s.latestCheckpoint != nil) || + (latest != nil && s.latestCheckpoint != nil && latest.BlockNumber < s.latestCheckpoint.BlockNumber) + var reorgBlocks uint64 + var reduceToBlock int64 + if reorg { + if latest == nil { + reorgBlocks = s.latestCheckpoint.BlockNumber - s.latestCheckpoint.FullBlockRange.StartBlock + 1 + reduceToBlock = -1 + } else { + reorgBlocks = s.latestCheckpoint.BlockNumber - latest.BlockNumber + reduceToBlock = int64(latest.BlockNumber) + } + } + + cs := *s.chainState + if err := SetCheckpoints(&cs, checkpoints); err != nil { + return err + } + if err := SetTemplates(&cs, templates); err != nil { + return err + } + if err := SetHandlerStat(&cs, agentStat); err != nil { + return err + } + + if err := updateChainState(ctx, s.cli, cs); err != nil { + return err + } + s.chainState = &cs + s.latestCheckpoint = latest + + if reorg { + controller.N.ReorgDone(ctx, s.processor, s.chainID, reorgBlocks, reduceToBlock) + } + return nil +} + +func (s *checkpointStore) SaveError(ctx context.Context, extErr *controller.ExternalError) error { + cs := *s.chainState + cs.ErrorRecord = newErrorRecord(extErr) + cs.State = int32(protos.ChainState_Status_ERROR) + if err := updateChainState(ctx, s.cli, cs); err != nil { + return err + } + s.chainState = &cs + return nil +} diff --git a/driver/controller/startup/entity.go b/driver/controller/startup/entity.go new file mode 100644 index 0000000..2e72fbb --- /dev/null +++ b/driver/controller/startup/entity.go @@ -0,0 +1,155 @@ +package startup + +import ( + "context" + "errors" + "time" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/clickhouse" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" +) + +type entityController struct { + *persistent.Controller +} + +func newEntityController( + store *clickhouse.Store, + chainID string, + storeCacheSize int, + storeFullCacheSize int, + monitor persistent.MetricsMonitor, +) *entityController { + return &entityController{ + Controller: persistent.NewController( + clickhouse.NewChainStore(store, chainID, storeCacheSize, storeFullCacheSize), + monitor, + ), + } +} + +func (c entityController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + var blockNumber int64 = -1 + if checkpoint != nil { + blockNumber = int64(checkpoint.BlockNumber) + } + if err := c.Controller.Reorg(ctx, blockNumber); err != nil { + return controller.NewExternalError(controller.ErrCodeCleanEntityDataFailed, err) + } + return nil +} + +var maxUncommitedEntityChanges = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_ENTITY_CHANGES", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c entityController) CachedTooMuch(blockNumber uint64) bool { + return uint64(c.Controller.CountUncommittedChanges(blockNumber)) > maxUncommitedEntityChanges +} + +func (c entityController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, map[string]int, *controller.ExternalError) { + created, updated, err := c.Controller.Commit(ctx, blockNumber, blockTime) + if err != nil { + if errors.Is(err, persistent.ErrUpdateImmutable) { + return created, updated, controller.NewExternalError(controller.ErrCodeUpdateImmutableEntity, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return created, updated, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return created, updated, controller.NewExternalError(controller.ErrCodeSaveEntityDataFailed, err) + } + return created, updated, nil +} + +func (c entityController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (*persistent.EntityBox, *controller.ExternalError) { + box, err := c.Controller.GetEntity(ctx, typ, id, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return box, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return box, controller.NewExternalError(controller.ErrCodeGetEntityFromDBFailed, err) + } + return box, nil +} + +func (c entityController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (*persistent.EntityBox, *controller.ExternalError) { + box, err := c.Controller.GetEntityInBlock(ctx, typ, id, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return box, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return box, controller.NewExternalError(controller.ErrCodeGetEntityFromDBFailed, err) + } + return box, nil +} + +func (c entityController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) ([]*persistent.EntityBox, *string, *controller.ExternalError) { + boxes, next, err := c.Controller.ListEntity(ctx, entityType, filters, cursor, limit, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidListFilter) { + return boxes, next, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return boxes, next, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return boxes, next, controller.NewExternalError(controller.ErrCodeListEntityFromDBFailed, err) + } + return boxes, next, nil +} + +func (c entityController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *controller.ExternalError) { + boxes, target, err := c.Controller.ListRelated(ctx, entityType, id, fieldName, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidField) { + return boxes, target, controller.NewExternalError(controller.ErrCodeListRelatedEntityWithInvalidField, err) + } + return boxes, target, controller.NewExternalError(controller.ErrCodeListEntityFromDBFailed, err) + } + return boxes, target, nil +} + +func (c entityController) SetEntity( + ctx context.Context, + entityType *schema.Entity, + box persistent.UncommittedEntityBox, +) *controller.ExternalError { + if err := c.Controller.SetEntity(ctx, entityType, box); err != nil { + if errors.Is(err, persistent.ErrUpdateImmutable) { + return controller.NewExternalError(controller.ErrCodeUpdateImmutableEntity, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return controller.NewExternalError(controller.ErrCodeSaveEntityDataFailed, err) + } + return nil +} diff --git a/driver/controller/startup/quota.go b/driver/controller/startup/quota.go new file mode 100644 index 0000000..3adc184 --- /dev/null +++ b/driver/controller/startup/quota.go @@ -0,0 +1,128 @@ +package startup + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/service/processor/models" + "sentioxyz/sentio-core/service/usage/protos" +) + +type quotaService struct { + chainID string + processor *models.Processor + cli protos.UsageServiceClient +} + +func newQuotaService(chainID string, processor *models.Processor, cli protos.UsageServiceClient) *quotaService { + return "aService{chainID: chainID, processor: processor, cli: cli} +} + +func (s *quotaService) CheckOverQuota(ctx context.Context) (*controller.OverQuota, error) { + req := protos.CheckOverLimitRequest{ + ProjectId: s.processor.ProjectID, + ProcessorId: &s.processor.ID, + Now: timestamppb.Now(), + Sku: "metric", // We can use any SKU here, since we only care about the overall quota. + } + resp, err := s.cli.CheckOverLimit(ctx, &req) + if err != nil { + return nil, errors.Wrapf(err, "check over quota failed") + } + if len(resp.GetOver()) == 0 { + return nil, nil + } + return &controller.OverQuota{ + Msg: strings.Join(resp.Over, "\n"), + Detail: strings.Join(resp.OverDetail, "\n"), + }, nil +} + +func (s *quotaService) SaveUsage(ctx context.Context, used controller.Usage, inWatching bool) error { + var version = fmt.Sprintf("%d", s.processor.Version) + type record struct { + sku string + count int + tags map[string]string + } + var records []record + // metric v3 + for _, metricType := range []timeseries.MetaType{timeseries.MetaTypeCounter, timeseries.MetaTypeGauge} { + for name, count := range used.TimeSeries[metricType] { + records = append(records, record{ + sku: "metricv3", + count: count, + tags: map[string]string{"name": name, "version": version, "type": string(metricType)}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "metricV3", string(metricType), name, int64(count)) + } + } + // event v3 + for name, count := range used.TimeSeries[timeseries.MetaTypeEvent] { + records = append(records, record{ + sku: "eventv3", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "eventV3", "", name, int64(count)) + } + // webhook + for name, count := range used.Export { + records = append(records, record{ + sku: "webhook", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + } + // entity + for name, count := range used.EntityCreated { + records = append(records, record{ + sku: "entity_created", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "entity", "created", name, int64(count)) + } + for name, count := range utils.MergeMapSum(used.EntityCreated, used.EntityUpdated) { + records = append(records, record{ + sku: "entity", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + } + for name, count := range used.EntityUpdated { + controller.N.DataSaved(ctx, s.processor, s.chainID, "entity", "updated", name, int64(count)) + } + + // build request and async save + var req protos.AsyncSaveRequest + for _, r := range records { + if r.count == 0 { + continue + } + req.Dialogues = append(req.Dialogues, &protos.Dialogue{ + RequestTime: timestamppb.Now(), + Succeed: true, + Units: uint64(r.count), + Tags: &protos.Tags{ + ProjectId: s.processor.ProjectID, + ProcessorId: &s.processor.ID, + Sku: utils.Select(inWatching, r.sku, r.sku+"_backfill"), + CustomTags: r.tags, + }, + }) + } + if _, err := s.cli.AsyncSave(ctx, &req); err != nil { + _, logger := log.FromContext(ctx, "records", records) + logger.Warnfe(err, "save usage failed") + } + return nil +} diff --git a/driver/controller/startup/standard.go b/driver/controller/startup/standard.go new file mode 100644 index 0000000..243ace0 --- /dev/null +++ b/driver/controller/startup/standard.go @@ -0,0 +1,368 @@ +package startup + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + evmchain "sentioxyz/sentio-core/chain/evm" + suitypes "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/chains" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + evmdata "sentioxyz/sentio-core/driver/controller/data/evm" + fueldata "sentioxyz/sentio-core/driver/controller/data/fuel" + soldata "sentioxyz/sentio-core/driver/controller/data/sol" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/driver/controller/standard/aptos" + "sentioxyz/sentio-core/driver/controller/standard/evm" + "sentioxyz/sentio-core/driver/controller/standard/fuel" + "sentioxyz/sentio-core/driver/controller/standard/sol" + "sentioxyz/sentio-core/driver/controller/standard/sui" + suigrpc "sentioxyz/sentio-core/driver/controller/standard/sui/grpc" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/subgraph/manifest" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/common/rpc" + protossvc "sentioxyz/sentio-core/service/processor/protos" + + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type standardStartupController struct { + baseStartupController + + processorClients []protos.ProcessorV3Client + initResult *protos.InitResponse +} + +func (c *standardStartupController) buildProcessorUrlList() ([]string, error) { + processorUrlList := []string{c.config.ProcessorUrl} + if c.processor.NumWorkers <= 1 { + return processorUrlList, nil + } + host, port, has := strings.Cut(c.config.ProcessorUrl, ":") + var basePort uint64 = 80 + if has { + basePort, _ = strconv.ParseUint(port, 10, 64) + } + for i := uint64(1); i < uint64(c.processor.NumWorkers); i++ { + processorUrlList = append(processorUrlList, fmt.Sprintf("%s:%d", host, basePort+i)) + } + return processorUrlList, nil +} + +func (c *standardStartupController) buildMainControllers(ctx context.Context) ( + ctrls map[string]*controller.MainController, + exitCode exitcode.Code, + err error, +) { + _, logger := log.FromContext(ctx) + ctrls = make(map[string]*controller.MainController) + + // connect to webhook service and create webhook subscription + if err = c.createWebhookSubscription(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "create webhook subscription failed") + } + + // create gcp pub sub topic + if err = c.createPubSubTopic(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "create pubsub topic failed") + } + + // connect to clickhouse + if err = c.connectClickhouse(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "connect to clickhouse failed") + } + + // build time series store + if err = c.buildTimeSeriesStore(ctx); err != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(err, "build time series store failed") + } + + // load all templates + var templates []*protos.TemplateInstance + for _, cs := range c.processor.ChainStates { + if chainTemplates, err := LoadTemplates(cs); err != nil { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "invaid templates")) + } else { + templates = append(templates, standard.ConvertTemplateInstanceBack(cs.ChainID, chainTemplates)...) + } + } + + // connect to all processor and init them + processorUrlList, buildErr := c.buildProcessorUrlList() + if buildErr != nil { + return ctrls, exitcode.NeverRetry, buildErr + } + c.processorClients = make([]protos.ProcessorV3Client, len(processorUrlList)) + var initResultText string + for i, processorUrl := range processorUrlList { + // connect to processor + conn, connErr := rpc.DialInsecure(processorUrl) + if connErr != nil { + return ctrls, 0, errors.Wrapf(connErr, "connect to processor #%d %s failed", i, processorUrl) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.processorClients[i] = protos.NewProcessorV3Client(conn) + logger.Infof("connected to processor #%d %s", i, processorUrl) + + // call start function + for { + _, startErr := c.processorClients[i].Start(ctx, &protos.StartRequest{TemplateInstances: templates}) + if startErr == nil { + break + } + if status.Code(startErr) == codes.InvalidArgument { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(startErr, "call start for processor failed") + } + logger.Warnfe(startErr, "call start for processor #%d failed, will retry after %s", i, initRetryInterval) + if startErr = utils.Sleep(ctx, initRetryInterval); startErr != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(startErr, "call start for processor failed") + } + } + logger.Infof("called start for processor #%d", i) + + // get init result + resp, getConfigErr := c.processorClients[i].GetConfig(ctx, &protos.ProcessConfigRequest{}) + if getConfigErr != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(getConfigErr, "get processor config failed") + } + chainIDSet := set.New[string]() + for _, cc := range resp.GetContractConfigs() { + chainIDSet.Add(cc.GetContract().GetChainId()) + } + for _, ac := range resp.GetAccountConfigs() { + chainIDSet.Add(ac.GetChainId()) + } + chainIDList := chainIDSet.DumpValues() + sort.Strings(chainIDList) + initResult := &protos.InitResponse{ + ChainIds: chainIDList, + DbSchema: resp.GetDbSchema(), + Config: resp.GetConfig(), + ExecutionConfig: resp.GetExecutionConfig(), + MetricConfigs: resp.GetMetricConfigs(), + ExportConfigs: resp.GetExportConfigs(), + EventLogConfigs: resp.GetEventLogConfigs(), + } + + // check init result + if i == 0 { + c.initResult, initResultText = initResult, string(protojson.MustJSONMarshal(initResult)) + } else if another := string(protojson.MustJSONMarshal(initResult)); another != initResultText { + logger.With("this", another, "before", initResultText).Errorf("configs from processor #%d was different", i) + return ctrls, exitcode.AlwaysRetry, errors.Errorf("configs from different processor has diff") + } + logger.Infof("got config from processor #%d succeed", i) + } + logger.Infow("init processors succeed", "initResult", initResultText) + + // check chain + if len(c.initResult.GetChainIds()) == 0 { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no chain in processor")) + } + for _, chainID := range c.initResult.GetChainIds() { + if _, has := c.chainConfigs[chainID]; !has { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("chain %s is not supported", chainID)) + } + } + + // build entity store + if schemeCnt := c.initResult.GetDbSchema().GetGqlSchema(); len(schemeCnt) > 0 { + if extErr := c.buildEntityStore(ctx, schemeCnt); extErr != nil { + if extErr.IsUserError() { + return ctrls, exitcode.NeverRetry, extErr + } + return ctrls, exitcode.AlwaysRetry, extErr + } + req := &protossvc.SetProcessorEntitySchemaRequest{ProcessorId: c.config.ProcessorID, Schema: schemeCnt} + if _, err = c.processorClient.SetProcessorEntitySchema(ctx, req); err != nil { + return ctrls, exitcode.NeverRetry, errors.Wrapf(err, "update processor entity schema failed") + } + } + + // build main controller for each chain + for _, chainID := range c.initResult.GetChainIds() { + ctrls[chainID], exitCode, err = c.buildMainController(ctx, chainID) + if err != nil { + err = errors.Wrapf(err, "build main controller for chain %s failed", chainID) + return + } + logger.Infof("main controller of chain %s is ready", chainID) + } + return +} + +func (c *standardStartupController) buildMainController( + ctx context.Context, + chainID string, +) (ctrl *controller.MainController, exitCode exitcode.Code, err error) { + chainConfig := c.chainConfigs[chainID] + if chainConfig.IsCustomizedEndpoint { + if err = evmchain.CheckArchiveNode(context.Background(), chainConfig.Endpoint); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "invalid customized endpoint %q for chain %s", chainConfig.Endpoint, chainID)) + } + } + chainType, hasChain := chains.GetChainType(chains.ChainID(chainID)) + if !hasChain { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unknown chain id %s", chainID)) + } + // prepare client and handler controller + var checkLink bool + var cli controller.Client + var handlerCtrl controller.HandlerController + switch { + case chainID == manifest.CustomizedChainID || chains.IsEVMChains(chainID): + checkLink = true + evmCli, newClientErr := evmdata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + chainConfig.ProcessingDelayBlocks, + controller.SubscribeMinWatchInterval, + time.Second*3, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build evm client failed") + } + handlerCtrl = evm.NewHandlerController(c.processor, c.initResult, chainConfig, evmCli, c.processorClients) + cli = evmCli + case chains.IsAptosChain(chainID): + aptosCli, newClientErr := aptosdata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build aptos client failed") + } + handlerCtrl = aptos.NewHandlerController(c.processor, c.initResult, chainConfig, aptosCli, c.processorClients) + cli = aptosCli + case chains.IsSuiChain(chainID): + suiCli, newClientErr := suidata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build sui client failed") + } + // grpc-format data is supported only for the sui variation (not iota) at DriverVersion >= 2. + if suitypes.VariationFromChainID(chains.SuiChainID(chainID)) == suitypes.VariationSUI && + c.processor.DriverVersion >= 2 { + handlerCtrl = suigrpc.NewHandlerController(c.processor, c.initResult, chainConfig, suiCli, c.processorClients) + } else { + handlerCtrl = sui.NewHandlerController(c.processor, c.initResult, chainConfig, suiCli, c.processorClients) + } + cli = suiCli + case chains.IsFuelChain(chainID): + fuelCli, newClientErr := fueldata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build fuel client failed") + } + handlerCtrl = fuel.NewHandlerController(c.processor, c.initResult, chainConfig, fuelCli, c.processorClients) + cli = fuelCli + case chains.IsSolanaChain(chainID): + solCli, newClientErr := soldata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + c.processor.DriverVersion, + ) + if newClientErr != nil { + // A retryable NewClient error (e.g. the super-node probe kept hitting transient HTTP/timeout + // errors) means the endpoint may simply be temporarily unavailable; restart the pod and try + // again instead of failing permanently. + if data.IsNewClientRetryable(newClientErr) { + return nil, exitcode.AlwaysRetry, errors.Wrapf(newClientErr, "build sol client failed") + } + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build sol client failed") + } + handlerCtrl = sol.NewHandlerController(c.processor, c.initResult, chainConfig, solCli, c.processorClients) + cli = solCli + default: + // chainID is OK but the chainType is not supported, so is a system error + return nil, exitcode.NeverRetry, errors.Errorf("chain type %s is not supported", chainType) + } + // block builder + blockBuilder := controller.NewBlockBuilder(handlerCtrl, cli, checkLink) + // webhook controller + var webhookCtrl controller.WebhookController = controller.EmptyWebhookController{} + if c.pubSubTopic != nil { + webhookCtrl = newWebhookController(c.processor, c.pubSubTopic) + } + // time series controller + var timeSeriesCtrl controller.TimeSeriesController = controller.EmptyTimeSeriesController{} + if c.timeSeriesStore != nil { + timeSeriesCtrl = newTimeSeriesController(chainID, c.timeSeriesStore) + } + // entity controller + var entityCtrl controller.EntityController = controller.EmptyEntityController{} + if c.entityStore != nil { + entityCtrl = newEntityController( + c.entityStore, + chainID, + c.config.EntityStoreCacheSize, + c.config.EntityStoreFullCacheSize, + c.config.EntityMetricsMonitor) + } + // checkpoint store + var store controller.CheckpointStore + if store, err = c.getCheckpointStore(ctx, chainID); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeSaveCheckpointFailed, err) + } + // checkpoint controller + var checkpointCtrl controller.CheckpointController + checkpointCtrl, err = controller.NewCheckpointController( + ctx, + chainID, + controller.SaveCheckpointDelay, + controller.SaveCheckpointInterval, + controller.MaxKeepCheckpointCount, + store, + c.getQuotaService(chainID), + timeSeriesCtrl, + entityCtrl, + webhookCtrl, + c.buildCommitCtx, + ) + if err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "build checkpoint controller failed") + } + // main controller + seqMode := c.initResult.GetExecutionConfig().GetSequential() + ctrl = controller.NewMainController(blockBuilder, checkpointCtrl, seqMode, c.processor, chainID) + return ctrl, exitcode.AlwaysRetry, nil +} diff --git a/driver/controller/startup/startup.go b/driver/controller/startup/startup.go new file mode 100644 index 0000000..afd292d --- /dev/null +++ b/driver/controller/startup/startup.go @@ -0,0 +1,632 @@ +package startup + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "cloud.google.com/go/pubsub" + "github.com/ClickHouse/clickhouse-go/v2" + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" + + "sentioxyz/sentio-core/common/chx" + ckhmanager "sentioxyz/sentio-core/common/clickhousemanager" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/gonanoid" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/tracker" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + entitychs "sentioxyz/sentio-core/driver/entity/clickhouse" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/timeseries" + timeserieschs "sentioxyz/sentio-core/driver/timeseries/clickhouse" + sentioerror "sentioxyz/sentio-core/service/common/errors" + commonmodels "sentioxyz/sentio-core/service/common/models" + "sentioxyz/sentio-core/service/common/rpc" + protosregistry "sentioxyz/sentio-core/service/database_registry/protos" + "sentioxyz/sentio-core/service/processor/models" + protossvc "sentioxyz/sentio-core/service/processor/protos" + protousage "sentioxyz/sentio-core/service/usage/protos" + protoswebhook "sentioxyz/sentio-core/service/webhook/protos" +) + +type baseStartupController struct { + config Config + + processorClient protossvc.ProcessorServiceClient + + usageClient protousage.UsageServiceClient + registryClient protosregistry.DatabaseRegistryServiceClient + + processor *models.Processor + chainConfigs map[string]*chain.ConfigV2 + + pubSubTopic *pubsub.Topic + + // timeseries / entity clickhouse controllers, built by the injected + // ClickhouseConnector once the processor is loaded (nil => store disabled). + tsController *chx.Controller + entityController *chx.Controller + + ipfsShell *shell.Shell + + timeSeriesStore timeseries.Store + entityStore *entitychs.Store + + release []func() +} + +func (c *baseStartupController) releaseAll() { + for i := len(c.release) - 1; i >= 0; i-- { + c.release[i]() + } +} + +func (c *baseStartupController) connectToProcessorService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + conn, err := rpc.DialAuto(c.config.ProcessorService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to processor service %s failed", c.config.ProcessorService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + + c.processorClient = protossvc.NewProcessorServiceClient(conn) + logger.Infof("connected to processor service %s", c.config.ProcessorService) + return nil +} + +func (c *baseStartupController) connectToUsageService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.UsageService == "" { + logger.Warnf("no usage service so will not connect to usage service") + return nil + } + conn, err := rpc.DialAuto(c.config.UsageService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to usage service %s failed", c.config.UsageService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.usageClient = protousage.NewUsageServiceClient(conn) + logger.Infof("connected to usage service %s", c.config.UsageService) + return nil +} + +// connectToDBRegistryService dials the gRPC endpoint that hosts +// DatabaseRegistryService and keeps the client stub on baseStartupController. +// The address is logically independent from the billing/usage service address. +// When empty (cloud deployments), this is a no-op and registration is gated +// away later by newEntityProbe and newTimeSeriesProbe based on the processor's TablePattern. +func (c *baseStartupController) connectToDBRegistryService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.DBRegistryService == "" { + logger.Warnf("no db registry service configured so will not connect to it") + return nil + } + conn, err := rpc.DialAuto(c.config.DBRegistryService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to db registry service %s failed", c.config.DBRegistryService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.registryClient = protosregistry.NewDatabaseRegistryServiceClient(conn) + logger.Infof("connected to db registry service %s", c.config.DBRegistryService) + return nil +} + +func (c *baseStartupController) loadChainsConfig(ctx context.Context) (err error) { + _, logger := log.FromContext(ctx) + c.chainConfigs, err = chain.LoadChainsConfigV2( + c.config.ChainConfigFile, chain.PatchChainsConfigEnv, c.processor.NetworkOverrides) + if err == nil { + logger.Info("loaded chain config") + } + return err +} + +func (c *baseStartupController) getProcessor(ctx context.Context) error { + _, logger := log.FromContext(ctx) + req := &protossvc.GetProcessorRequest{ProcessorId: c.config.ProcessorID} + response, err := c.processorClient.GetProcessorWithProject(ctx, req) + if err != nil { + return err + } + var p models.Processor + if err = p.FromPB(response.Processor); err != nil { + return err + } + p.Project = &commonmodels.Project{} + p.Project.FromPB(response.Project) + c.processor = &p + logger.Infof("got processor from processor service") + return nil +} + +func (c *baseStartupController) createWebhookSubscription(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.WebhookService == "" { + logger.Warn("no webhook service so will not create webhook subscription") + return nil + } + conn, err := rpc.DialAuto(c.config.WebhookService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to webhook service %s failed", c.config.WebhookService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + logger.Infof("connected to webhook service %s", c.config.WebhookService) + + req := &protoswebhook.CreateSubscriptionRequest{ProcessorId: c.config.ProcessorID} + if _, err = protoswebhook.NewWebhookServiceClient(conn).CreateSubscription(ctx, req); err != nil { + return errors.Wrapf(err, "create subscription failed") + } + logger.Infof("created webhook subscription") + return nil +} + +func (c *baseStartupController) createPubSubTopic(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.WebhookTopic == "" || c.config.PubSubProject == "" { + logger.Warnf("no webhook topic or pubsub project so will not create pubsub topic") + return nil + } + cli, err := pubsub.NewClient(ctx, c.config.PubSubProject) + if err != nil { + return errors.Wrapf(err, "create gcp pubsub client failed") + } + c.release = append(c.release, func() { + _ = cli.Close() + }) + c.pubSubTopic = cli.Topic(c.config.WebhookTopic) + logger.Infof("created pubsub topic %s in gcp", c.config.WebhookTopic) + return nil +} + +// connectClickhouse asks the injected ClickhouseConnector to build the +// timeseries and entity chx.Controller for the loaded processor. All sharding / +// housegate wiring lives in the connector implementation in the driver binary. +func (c *baseStartupController) connectClickhouse(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.ClickhouseConnector == nil { + logger.Warnf("no clickhouse connector so will not connect to clickhouse") + return nil + } + ts, entity, err := c.config.ClickhouseConnector.Connect(ctx, c.processor) + if err != nil { + return errors.Wrap(err, "connect to clickhouse failed") + } + c.tsController = ts + c.entityController = entity + return nil +} + +func (c *baseStartupController) newTimeSeriesProbe() (timeserieschs.Probe, error) { + if c.processor.TablePattern != models.TablePatternNetworkV1 { + return nil, nil + } + if c.registryClient == nil { + return nil, errors.Errorf( + "processor TablePattern=%s requires onchain database registration, "+ + "but no db registry service is configured (set -db-registry-service flag)", + c.processor.TablePattern, + ) + } + return &timeSeriesProbe{ + client: c.registryClient, + processorID: c.processor.ID, + processorReplica: c.config.ProcessorReplica, + }, nil +} + +func (c *baseStartupController) buildTimeSeriesStore(ctx context.Context) error { + // build time series clickhouse store, each chain will use it to build time series controller + _, logger := log.FromContext(ctx) + + if c.tsController == nil { + logger.Warnf("no clickhouse connection so will not create time series store") + return nil + } + ctrl := *c.tsController + + probe, err := c.newTimeSeriesProbe() + if err != nil { + return err + } + + c.timeSeriesStore = timeserieschs.NewStore(ctrl, timeserieschs.Option{}, probe) + if err := c.timeSeriesStore.Init(ctx); err != nil { + return errors.Wrapf(err, "init time series store failed") + } + logger.Info("time series store is ready") + return nil +} + +func (c *baseStartupController) newEntityProbe() (entitychs.Probe, error) { + if c.processor.TablePattern != models.TablePatternNetworkV1 { + return nil, nil + } + if c.registryClient == nil { + return nil, errors.Errorf( + "processor TablePattern=%s requires onchain database registration, "+ + "but no db registry service is configured (set -db-registry-service flag)", + c.processor.TablePattern, + ) + } + return &entityProbe{ + client: c.registryClient, + processorID: c.processor.ID, + processorReplica: c.config.ProcessorReplica, + }, nil +} + +func (c *baseStartupController) buildEntityStore(ctx context.Context, schemaText string) *controller.ExternalError { + _, logger := log.FromContext(ctx) + + if c.entityController == nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("need clickhouse connection but no clickhouse connector configured")) + } + ctrl := *c.entityController + + entityFea := entitychs.BuildFeatures(c.processor.EntitySchemaVersion) + entitySchema, buildErr := schema.ParseAndVerifySchema(schemaText, entityFea.BuildVerifyOptions()...) + if buildErr != nil { + return controller.NewExternalError(controller.ErrCodeInvalidEntitySchema, buildErr) + } + + probe, err := c.newEntityProbe() + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, err) + } + + c.entityStore = entitychs.NewStore(ctrl, entityFea, entitySchema, entitychs.DefaultCreateTableOption, probe) + if initErr := c.entityStore.InitEntitySchema(ctx); initErr != nil { + return controller.NewExternalError(controller.ErrCodeInitEntityFailed, initErr) + } + logger.Infow("entity store is ready", "schema", schemaText, "feature", entityFea) + return nil +} + +func (c *baseStartupController) buildIpfsShell(ctx context.Context) { + if c.config.IpfsNodeAddr == "" { + return + } + _, logger := log.FromContext(ctx) + c.ipfsShell = shell.NewShell(c.config.IpfsNodeAddr) + c.ipfsShell.SetTimeout(5 * time.Second) + logger.Infow("ipfs shell is ready") +} + +func (c *baseStartupController) getQuotaService(chainID string) controller.QuotaService { + if c.usageClient == nil { + return controller.EmptyQuotaService{} + } + return newQuotaService(chainID, c.processor, c.usageClient) +} + +func (c *baseStartupController) buildCommitCtx(ctx context.Context, chainID string, cur controller.Checkpoint) context.Context { + sign, _ := json.Marshal(map[string]any{ + "processor_id": c.config.ProcessorID, + "processor_replica": c.config.ProcessorReplica, + "chain_id": chainID, + "watching": cur.InWatching(), + "current_block_number": strconv.FormatUint(cur.BlockNumber, 10), + "current_block_time": cur.BlockTime.Format(time.RFC3339Nano), + }) + return ckhmanager.ContextMergeSettings(ctx, clickhouse.Settings{"log_comment": string(sign)}) +} + +func newErrorRecord(err error) (er sentioerror.ErrorRecord) { + er.ID, _ = gonanoid.GenerateLongID() + er.CreatedAt = time.Now() + er.Namespace = sentioerror.DRIVER + er.Message = err.Error() + er.Code = controller.ErrCodeSystem + var extErr *controller.ExternalError + if errors.As(err, &extErr) { + er.Code = int32(extErr.Code()) + if extErr.IsUserError() { + er.Namespace = sentioerror.PROCESSOR + } + } + return +} + +func updateChainState(ctx context.Context, cli protossvc.ProcessorServiceClient, cs models.ChainState) error { + var name string + if cs.ChainID == "meta" { + name = "meta chain state" + } else { + name = fmt.Sprintf("chain state of chain %s", cs.ChainID) + } + csPb, err := cs.ToPB() + if err != nil { + return errors.Wrapf(err, "convert %s to pb failed", name) + } + req := protossvc.UpdateChainProcessorStatusRequest{ + Id: cs.ProcessorID, + ChainState: csPb, + } + // ChainState may be very big, so need to use compressor + if _, err = cli.UpdateChainProcessorStatus(ctx, &req, grpc.UseCompressor(gzip.Name)); err != nil { + _, logger := log.FromContext(ctx, + "IndexerState", utils.StringSummaryV2(string(cs.IndexerState)), + "MeterState", utils.StringSummaryV2(string(cs.MeterState)), + "HandlerStat", utils.StringSummaryV2(string(cs.HandlerStat)), + "Templates", utils.StringSummaryV2(cs.Templates)) + logger.Warnfe(err, "update %s failed", name) + return errors.Wrapf(err, "update %s failed", name) + } + return nil +} + +func (c *baseStartupController) findChainState(chainID string) (*models.ChainState, bool) { + for _, chainState := range c.processor.ChainStates { + if chainState.ChainID == chainID { + return chainState, true + } + } + return &models.ChainState{ + ID: fmt.Sprintf("%s_%s", c.config.ProcessorID, chainID), + ChainID: chainID, + ProcessorID: c.config.ProcessorID, + ProcessedBlockNumber: -1, + ProcessedVersion: c.processor.Version, + State: int32(protossvc.ChainState_Status_CATCHING_UP), + }, false +} + +func (c *baseStartupController) getCheckpointStore( + ctx context.Context, + chainID string, +) (controller.CheckpointStore, error) { + cs, has := c.findChainState(chainID) + store := newCheckpointStore(c.processor, chainID, c.processorClient, cs) + if has { + return store, nil + } + // no chain state for the chain, create the initial one + if err := store.Save(ctx, nil, nil, nil); err != nil { + return nil, errors.Wrapf(err, "save initial chain state failed") + } + return store, nil +} + +func (c *baseStartupController) updateMetaState(ctx context.Context, metaErr error) error { + cs := models.ChainState{ + ID: fmt.Sprintf("%s_meta", c.processor.ID), + ChainID: "meta", + ProcessorID: c.processor.ID, + ProcessedBlockNumber: -1, + ProcessedVersion: -1, + State: int32(protossvc.ChainState_Status_PROCESSING_LATEST), + } + if metaErr != nil { + cs.State = int32(protossvc.ChainState_Status_ERROR) + cs.ErrorRecord = newErrorRecord(metaErr) + } + return updateChainState(ctx, c.processorClient, cs) +} + +const ( + initRetryInterval = time.Second * 5 +) + +func main(ctx context.Context, config Config) (exitCode exitcode.Code, err error) { + _, logger := log.FromContext(ctx) + logger.Infow("startup now", "startupConfig", config) + + base := baseStartupController{config: config} + defer base.releaseAll() + + // 1. connect to processor service + if err = base.connectToProcessorService(ctx); err != nil { + logger.Errorfe(err, "startup failed") + return 0, nil + } + + // 2. get processor from processor service + if err = base.getProcessor(ctx); err != nil { + return 0, errors.Wrapf(err, "get processor failed") + } + + // check if using all streaming mode + if base.processor.DriverVersion < 1 { + logger.Warnf("driver version is %d < 1, will not use all stream mode", base.processor.DriverVersion) + return -1, nil + } + + // record meta error + defer func() { + if err != nil { + logger.Errorf("startup failed: %+v", err) + if updateErr := base.updateMetaState(ctx, err); updateErr != nil { + logger.Errorfe(updateErr, "update meta chain state failed") + } + } + }() + + // 3. connect to usage service + if err = base.connectToUsageService(ctx); err != nil { + return 0, err + } + + // 3b. connect to db registry service (optional, only required for + // network_v1 processors). The check that the client stub exists when + // it is actually needed happens in newEntityProbe and newTimeSeriesProbe. + if err = base.connectToDBRegistryService(ctx); err != nil { + return 0, err + } + + // 4. load chain configs + if err = base.loadChainsConfig(ctx); err != nil { + return exitcode.AlwaysRetry, errors.Wrapf(err, "load chains config failed") + } + + // build main controllers + ctrls := make(map[string]*controller.MainController) + switch base.processor.Project.Type { + case commonmodels.ProjectTypeSentio: + std := standardStartupController{baseStartupController: base} + if ctrls, exitCode, err = std.buildMainControllers(ctx); err != nil { + return exitCode, err + } + case commonmodels.ProjectTypeSubgraph: + ss := subgraphStartupController{baseStartupController: base} + if ctrls, exitCode, err = ss.buildMainControllers(ctx); err != nil { + return exitCode, err + } + default: + return exitcode.NeverRetry, errors.Errorf("project type %s is not supported", base.processor.Project.Type) + } + + // update meta chain state + if updateErr := base.updateMetaState(ctx, nil); updateErr != nil { + logger.Errorfe(updateErr, "update meta chain state failed") + return 0, nil + } + logger.Info("startup succeed") + + // start all main controllers + g, gctx := errgroup.WithContext(ctx) + for chainID_, ctrl_ := range ctrls { + chainID, ctrl := chainID_, ctrl_ + tracker.AddOrReplaceTrackedObject("MainController::"+chainID, ctrl) + g.Go(func() error { + mainCtx, _ := log.FromContext(gctx, "chain_id", chainID) + return ctrl.Main(mainCtx) + }) + } + if err = g.Wait(); err != nil { + var extErr *controller.ExternalError + if errors.As(err, &extErr) { + switch { + case extErr.IsBillingError(): + return exitcode.OverQuota, extErr + case extErr.IsUserError(): + return exitcode.NeverRetry, extErr + } + } + return exitcode.AlwaysRetry, err + } + logger.Info("all chains are done") + <-ctx.Done() + return exitcode.AlwaysRetry, nil +} + +// ClickhouseConnector builds the timeseries and entity chx.Controller used by a +// processor's stores. Its implementation lives in the driver binary and +// encapsulates the sharding / housegate wiring, so the controller depends only +// on chx, which is in sentio-core. +// +// A nil returned controller means the corresponding store is not available +// (e.g. no clickhouse configured). +type ClickhouseConnector interface { + Connect(ctx context.Context, processor *models.Processor) (timeseries, entity *chx.Controller, err error) +} + +type Config struct { + ProcessorID string + ProcessorReplica int + ProcessorService string + UsageService string + DBRegistryService string + WebhookService string + WebhookTopic string + ProcessorUrl string + ChainConfigFile string + IpfsNodeAddr string + EntityStoreCacheSize int + EntityStoreFullCacheSize int + SubgraphTotalMemSize uint + SubgraphDebugTrace bool + // PubSubProject is the GCP project used to create the webhook pubsub topic; + // empty disables pubsub topic creation. Provided by the driver binary. + PubSubProject string + + // ClickhouseConnector builds the timeseries/entity chx.Controller for a + // processor. Its implementation lives in the driver binary. + ClickhouseConnector ClickhouseConnector + // EntityMetricsMonitor is the metric monitor for the entity store; it carries + // the otel instrument and is built by the driver binary so the controller + // does not import the binary's metrics package. + EntityMetricsMonitor persistent.MetricsMonitor +} + +func Main(config Config) { + const retryInterval = time.Second * 30 + ctx, logger := log.FromContext(concurrency.NewSignalContext(context.Background()), "processorID", config.ProcessorID) + for { + exitCode, _ := main(ctx, config) + if exitCode < 0 { + logger.Warnf("do not use all streaming mode") + return + } + if exitCode > 0 { + logger.Infof("EXIT WITH CODE %d", exitCode) + os.Exit(int(exitCode)) + } + + logger.Infof("startup failed, will retry after %s", retryInterval.String()) + select { + case <-time.After(retryInterval): + case <-ctx.Done(): + os.Exit(0) + } + } +} + +type timeSeriesProbe struct { + client protosregistry.DatabaseRegistryServiceClient + processorID string + processorReplica int +} + +func (p *timeSeriesProbe) PreCreateTable(ctx context.Context, tv chx.TableOrView) error { + metaType, _, err := timeseries.CutTableName(tv.GetName()) + if err != nil { + return err + } + _, err = p.client.EnsureTable(ctx, &protosregistry.EnsureTableRequest{ + ProcessorId: p.processorID, + ReplicaIndex: uint32(p.processorReplica), + TableId: tv.GetName(), + TableType: string(metaType), + }) + return err +} + +type entityProbe struct { + client protosregistry.DatabaseRegistryServiceClient + processorID string + processorReplica int +} + +func (p *entityProbe) PreCreateTable(ctx context.Context, tv chx.TableOrView) error { + if _, ok := tv.(chx.Table); !ok { + return nil + } + _, err := p.client.EnsureTable(ctx, &protosregistry.EnsureTableRequest{ + ProcessorId: p.processorID, + ReplicaIndex: uint32(p.processorReplica), + TableId: tv.GetName(), + TableType: "entity", + }) + return err +} diff --git a/driver/controller/startup/startup_test.go b/driver/controller/startup/startup_test.go new file mode 100644 index 0000000..61551a6 --- /dev/null +++ b/driver/controller/startup/startup_test.go @@ -0,0 +1,43 @@ +package startup + +import ( + "testing" + + "sentioxyz/sentio-core/common/log" + sentioerror "sentioxyz/sentio-core/service/common/errors" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/stretchr/testify/assert" +) + +func Test_errorRecord(t *testing.T) { + var er sentioerror.ErrorRecord + pb := er.ToPB() + log.Infof("pb.createdAt: %s", pb.GetCreatedAt().AsTime().String()) + var ner sentioerror.ErrorRecord + ner.FromPB(pb) + log.Infof("er.createdAt: %s", ner.CreatedAt.String()) + assert.True(t, ner.CreatedAt.IsZero()) +} + +func Test_buildProcessorUrls(t *testing.T) { + var s standardStartupController + + s.config.ProcessorUrl = "aaa" + s.processor = &models.Processor{NumWorkers: 1} + urls, err := s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa"}, urls) + + s.config.ProcessorUrl = "aaa" + s.processor.NumWorkers = 3 + urls, err = s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa", "aaa:81", "aaa:82"}, urls) + + s.config.ProcessorUrl = "aaa.bbb:9999" + s.processor.NumWorkers = 3 + urls, err = s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa.bbb:9999", "aaa.bbb:10000", "aaa.bbb:10001"}, urls) +} diff --git a/driver/controller/startup/subgraph.go b/driver/controller/startup/subgraph.go new file mode 100644 index 0000000..3f2e8ae --- /dev/null +++ b/driver/controller/startup/subgraph.go @@ -0,0 +1,133 @@ +package startup + +import ( + "context" + "time" + + evmchain "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/chains" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/subgraph" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/subgraph/manifest" + + "github.com/pkg/errors" +) + +type subgraphStartupController struct { + baseStartupController +} + +func (c *subgraphStartupController) buildMainControllers(ctx context.Context) ( + ctrls map[string]*controller.MainController, + exitCode exitcode.Code, + err error, +) { + ctrls = make(map[string]*controller.MainController) + // connect to clickhouse + if err = c.connectClickhouse(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "connect to clickhouse failed") + } + // build ipfs client + c.buildIpfsShell(ctx) + // manifest + var mf *manifest.Manifest + if mf, err = manifest.LoadFromIpfs(c.ipfsShell, c.processor.IpfsHash, true); err != nil { + if errors.Is(err, manifest.ErrInvalidManifest) || errors.Is(err, manifest.ErrInvalidCustomizedEndpoint) { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, err) + } + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "load subgraph manifest failed") + } + // chainID and chainConfig and client + chainID, endpoint, _ := manifest.GetChainID(mf.GetNetwork(), false) + var client evm.Client + var chainConfig *chain.ConfigV2 + if chainID == manifest.CustomizedChainID { + chainConfig = chain.NewCustomizedChainConfigV2(manifest.CustomizedChainID, endpoint) + } else if chains.IsEVMChains(chainID) { + var has bool + chainConfig, has = c.chainConfigs[chainID] + if !has { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("unsupported evm chain id %q for the network %q", chainID, mf.GetNetwork())) + } + if chainConfig.IsCustomizedEndpoint { + // chainConfig.Endpoint is not in the manifest file, it is in the NetworkOverrides, + // so need to check archive node here + if err = evmchain.CheckArchiveNode(ctx, chainConfig.Endpoint); err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, + "invalid customized endpoint %q for chain %s", chainConfig.Endpoint, chainID) + } + } + } else { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("unknown network %q in the manifest", mf.GetNetwork())) + } + client, err = evm.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + chainConfig.ProcessingDelayBlocks, + controller.SubscribeMinWatchInterval, + time.Second*3, + ) + if err != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(err, "build evm client failed") + } + // handler controller + var handlerCtrl *subgraph.HandlerController + handlerCtrl, err = subgraph.NewHandlerController( + ctx, + c.processor, + chainConfig, + client, + c.ipfsShell, + mf, + uint32(c.config.SubgraphTotalMemSize), + c.config.SubgraphDebugTrace, + ) + if err != nil { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeWasmInitFailed, err) + } + // block builder + blockBuilder := controller.NewBlockBuilder(handlerCtrl, client, true) + // c.entityStore + if extErr := c.buildEntityStore(ctx, mf.Schema.File.GetContent()); extErr != nil { + if extErr.IsUserError() { + return nil, exitcode.NeverRetry, extErr + } + return nil, exitcode.AlwaysRetry, extErr + } + // entity controller + entityCtrl := newEntityController(c.entityStore, chainID, c.config.EntityStoreCacheSize, + c.config.EntityStoreFullCacheSize, c.config.EntityMetricsMonitor) + // checkpoint store + var store controller.CheckpointStore + if store, err = c.getCheckpointStore(ctx, chainID); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeSaveCheckpointFailed, err) + } + // checkpoint controller + var checkpointCtrl controller.CheckpointController + checkpointCtrl, err = controller.NewCheckpointController( + ctx, + chainID, + controller.SaveCheckpointDelay, + controller.SaveCheckpointInterval, + controller.MaxKeepCheckpointCount, + store, + c.getQuotaService(chainID), + controller.EmptyTimeSeriesController{}, + entityCtrl, + controller.EmptyWebhookController{}, + c.buildCommitCtx, + ) + if err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "build checkpoint controller failed") + } + // main controller + ctrls[chainID] = controller.NewMainController(blockBuilder, checkpointCtrl, false, c.processor, chainID) + return ctrls, exitcode.AlwaysRetry, nil +} diff --git a/driver/controller/startup/timeseries.go b/driver/controller/startup/timeseries.go new file mode 100644 index 0000000..8dc781b --- /dev/null +++ b/driver/controller/startup/timeseries.go @@ -0,0 +1,138 @@ +package startup + +import ( + "context" + "math" + "sync" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" +) + +type timeSeriesController struct { + chainID string + store timeseries.Store + + mu sync.Mutex + cached map[uint64]map[uint64][]timeseries.Dataset // map[][] + committed *uint64 +} + +func newTimeSeriesController(chainID string, store timeseries.Store) *timeSeriesController { + return &timeSeriesController{ + chainID: chainID, + store: store, + cached: make(map[uint64]map[uint64][]timeseries.Dataset), + } +} + +func (c *timeSeriesController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + if checkpoint == nil { + c.cached = make(map[uint64]map[uint64][]timeseries.Dataset) + } else { + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn > checkpoint.BlockNumber + }) + } + var blockNumber int64 = -1 + if checkpoint != nil { + blockNumber = int64(checkpoint.BlockNumber) + } + if err := c.store.DeleteData(ctx, c.chainID, blockNumber); err != nil { + return controller.NewExternalError(controller.ErrCodeCleanTimeSeriesDataFailed, err) + } + return nil +} + +var maxUncommitedTimeSeries = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_TIME_SERIES_DATA", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c *timeSeriesController) getCachedSize(blockNumber uint64) (total uint64) { + for bn, blockData := range c.cached { + if bn > blockNumber { + continue + } + for _, dss := range blockData { + for _, ds := range dss { + total += uint64(len(ds.Rows)) + } + } + } + return total +} + +func (c *timeSeriesController) CachedTooMuch(blockNumber uint64) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.getCachedSize(blockNumber) > maxUncommitedTimeSeries +} + +func (c *timeSeriesController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (stat map[timeseries.MetaType]map[string]int, extErr *controller.ExternalError) { + // collect data to commit + var data []timeseries.Dataset + c.mu.Lock() + for _, bn := range utils.GetOrderedMapKeys(c.cached) { + if bn <= blockNumber { + data = append(data, utils.MergeArr(utils.GetMapValuesOrderByKey(c.cached[bn])...)...) + } + } + c.mu.Unlock() + + // actually save dataset + if err := c.store.AppendData(ctx, data, c.chainID, blockTime); err != nil { + if errors.Is(err, timeseries.ErrInvalidMetaDiff) { + return nil, controller.NewExternalError(controller.ErrCodeTimeSeriesDataSchemaChanged, err) + } + if errors.Is(err, timeseries.ErrInvalidMeta) { + return nil, controller.NewExternalError(controller.ErrCodeInvalidTimeSeriesData, err) + } + return nil, controller.NewExternalError(controller.ErrCodeSaveTimeSeriesDataFailed, + errors.Wrapf(err, "failed to commit timeseries data: %s", timeseries.GetDatasetsSummary(data))) + } + stat = make(map[timeseries.MetaType]map[string]int) + for _, ds := range data { + utils.IncrK2Map(stat, ds.Type, ds.Name, len(ds.Rows)) + } + + // save succeed, clean c.cached + c.mu.Lock() + defer c.mu.Unlock() + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn <= blockNumber + }) + c.committed = &blockNumber + return +} + +func (c *timeSeriesController) Insert(blockNumber uint64, taskIndex controller.TaskIndex, data []timeseries.Dataset) { + c.mu.Lock() + defer c.mu.Unlock() + org, _ := utils.GetFromK2Map(c.cached, blockNumber, taskIndex.Global) + utils.PutIntoK2Map(c.cached, blockNumber, taskIndex.Global, append(org, data...)) +} + +func (c *timeSeriesController) Snapshot() any { + c.mu.Lock() + defer c.mu.Unlock() + return map[string]any{ + "committed": c.committed, + "uncommitedTotal": c.getCachedSize(math.MaxUint64), + "uncommited": cacheSnapshot(c.cached, func(dss []timeseries.Dataset) (s int) { + for _, ds := range dss { + s += len(ds.Rows) + } + return s + }), + } +} diff --git a/driver/controller/startup/utils.go b/driver/controller/startup/utils.go new file mode 100644 index 0000000..e1e3103 --- /dev/null +++ b/driver/controller/startup/utils.go @@ -0,0 +1,48 @@ +package startup + +import ( + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/utils" +) + +func cacheSnapshot[T any](cache map[uint64]map[uint64]T, getSize func(T) int) any { + if len(cache) == 0 { + return nil + } + type segment struct { + Start uint64 + End uint64 + BlockCount int + DataCount int + } + ss := make([]segment, 0, len(cache)) + for _, bn := range utils.GetOrderedMapKeys(cache) { + s := segment{Start: bn, End: bn, BlockCount: 1} + for _, d := range cache[bn] { + s.DataCount += getSize(d) + } + ss = append(ss, s) + } + remove := sparsify.Remove(ss, func(s segment) uint64 { + return s.Start + }) + merge := func(ss []segment) (r segment) { + r = ss[0] + for i := 1; i < len(ss); i++ { + r.Start = min(ss[i].Start, r.Start) + r.End = max(ss[i].End, r.End) + r.BlockCount += ss[i].BlockCount + r.DataCount += ss[i].DataCount + } + return r + } + var result []segment + var s int + for p := 1; p < len(ss); p++ { + if !remove[p] { + result = append(result, merge(ss[s:p])) + s = p + } + } + return append(result, merge(ss[s:])) +} diff --git a/driver/controller/startup/webhook.go b/driver/controller/startup/webhook.go new file mode 100644 index 0000000..58a2844 --- /dev/null +++ b/driver/controller/startup/webhook.go @@ -0,0 +1,176 @@ +package startup + +import ( + "context" + "encoding/json" + "math" + "sync" + "time" + + "cloud.google.com/go/pubsub" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/service/processor/models" +) + +type webhookController struct { + processor *models.Processor + topic *pubsub.Topic + + mu sync.Mutex + cached map[uint64]map[uint64][]controller.WebhookMessage // map[][] + committed *uint64 +} + +func newWebhookController(processor *models.Processor, topic *pubsub.Topic) *webhookController { + return &webhookController{ + processor: processor, + topic: topic, + cached: make(map[uint64]map[uint64][]controller.WebhookMessage), + } +} + +func (c *webhookController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + if checkpoint == nil { + c.cached = make(map[uint64]map[uint64][]controller.WebhookMessage) + } else { + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn > checkpoint.BlockNumber + }) + } + // sent msg cannot be canceled + return nil +} + +type SingleWebhookMessage struct { + ExportName string `json:"export_name"` + EventID uint64 `json:"event_id"` + TimestampMicros uint64 `json:"timestamp_micros"` + Version uint64 `json:"version"` + Data string `json:"data"` +} + +const MaxMessagesPerBlock = 10000000 + +var maxUncommitedWebhookMessages = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_WEBHOOK_MESSAGES", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c *webhookController) getCachedSize(blockNumber uint64) (total uint64) { + for bn, blockMsgs := range c.cached { + if bn > blockNumber { + continue + } + for _, msgs := range blockMsgs { + total += uint64(len(msgs)) + } + } + return total +} + +func (c *webhookController) CachedTooMuch(blockNumber uint64) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.getCachedSize(blockNumber) > maxUncommitedWebhookMessages +} + +func (c *webhookController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (stat map[string]int, extErr *controller.ExternalError) { + // build SingleWebhookMessage from c.cached into dict + stat = make(map[string]int) + dict := make(map[string][]SingleWebhookMessage) + c.mu.Lock() + for _, bn := range utils.GetOrderedMapKeys(c.cached) { + if bn > blockNumber { + continue + } + blockMsgs := c.cached[bn] + if count := utils.CountMap(blockMsgs); count >= MaxMessagesPerBlock { + c.mu.Unlock() + return nil, controller.NewExternalError(controller.ErrCodeTooManyWebhookMsgEntity, + errors.Errorf("too many messages in block %d, %d > %d", bn, count, MaxMessagesPerBlock)) + } + var blockSeq uint64 + for _, messages := range utils.GetMapValuesOrderByKey(blockMsgs) { + for _, msg := range messages { + blockSeq++ + dict[msg.Channel] = append(dict[msg.Channel], SingleWebhookMessage{ + ExportName: msg.Name, + EventID: bn*MaxMessagesPerBlock + blockSeq, + TimestampMicros: uint64(msg.BlockTime.UnixMicro()), + Version: uint64(c.processor.Version), + Data: msg.Payload, + }) + stat[msg.Name] += 1 + } + } + } + c.mu.Unlock() + + // actually send SingleWebhookMessage + g, gctx := errgroup.WithContext(ctx) + for channel, messages := range dict { + data, err := json.Marshal(messages) + if err != nil { + panic(errors.Wrapf(err, "json marshal message for channel %s failed", channel)) + } + pubSubMsg := &pubsub.Message{ + Data: data, + Attributes: map[string]string{ + "channel_name": channel, + "project_id": c.processor.ProjectID, + "processor_id": c.processor.ID, + }, + } + g.Go(func() error { + _, pubErr := c.topic.Publish(gctx, pubSubMsg).Get(gctx) + if pubErr != nil { + return errors.Wrapf(pubErr, "publish message for channel %s failed", channel) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, controller.NewExternalError(controller.ErrCodeSendWebhookDataFailed, err) + } + + // send succeed, clean c.cached + c.mu.Lock() + defer c.mu.Unlock() + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn <= blockNumber + }) + c.committed = &blockNumber + return +} + +func (c *webhookController) Insert( + blockNumber uint64, + taskIndex controller.TaskIndex, + messages []controller.WebhookMessage, +) { + c.mu.Lock() + defer c.mu.Unlock() + org, _ := utils.GetFromK2Map(c.cached, blockNumber, taskIndex.Global) + utils.PutIntoK2Map(c.cached, blockNumber, taskIndex.Global, append(org, messages...)) +} + +func (c *webhookController) Snapshot() any { + c.mu.Lock() + defer c.mu.Unlock() + return map[string]any{ + "committed": c.committed, + "uncommitedTotal": c.getCachedSize(math.MaxUint64), + "uncommited": cacheSnapshot(c.cached, func(msgs []controller.WebhookMessage) (s int) { + return len(msgs) + }), + } +} diff --git a/driver/controller/subgraph/BUILD.bazel b/driver/controller/subgraph/BUILD.bazel new file mode 100644 index 0000000..3c729a1 --- /dev/null +++ b/driver/controller/subgraph/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "subgraph", + srcs = [ + "binding_data.go", + "block_data.go", + "handler.go", + "handler_block.go", + "handler_call.go", + "handler_event.go", + "instance.go", + "task.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/subgraph", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//common/concurrency", + "//common/log", + "//common/set", + "//common/timer", + "//common/utils", + "//common/wasm", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/evm", + "//driver/controller/fetcher", + "//driver/entity/persistent", + "//driver/subgraph/abiutil", + "//driver/subgraph/common", + "//driver/subgraph/ethereum", + "//driver/subgraph/manifest", + "//service/processor/models", + "@com_github_ethereum_go_ethereum//accounts/abi", + "@com_github_ethereum_go_ethereum//common", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_ethereum_go_ethereum//crypto", + "@com_github_ipfs_go_ipfs_api//:go-ipfs-api", + "@com_github_mr_tron_base58//base58", + "@com_github_pkg_errors//:errors", + ], +) diff --git a/driver/controller/subgraph/binding_data.go b/driver/controller/subgraph/binding_data.go new file mode 100644 index 0000000..4226e89 --- /dev/null +++ b/driver/controller/subgraph/binding_data.go @@ -0,0 +1,28 @@ +package subgraph + +import ( + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type taskData struct { + callHandlerParam any + + dataSource *manifest.DataSource + handlerID controller.HandlerID + txIndex int + logIndex int + + size int +} + +func (t taskData) Cmp(a taskData) int { + if r := utils.Cmp(t.txIndex, a.txIndex); r != 0 { + return r + } + if r := utils.Cmp(t.handlerID.ID, a.handlerID.ID); r != 0 { + return r + } + return utils.Cmp(t.logIndex, a.logIndex) +} diff --git a/driver/controller/subgraph/block_data.go b/driver/controller/subgraph/block_data.go new file mode 100644 index 0000000..42395ce --- /dev/null +++ b/driver/controller/subgraph/block_data.go @@ -0,0 +1,325 @@ +package subgraph + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/accounts/abi" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + evmExtend "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/subgraph/abiutil" + "sentioxyz/sentio-core/driver/subgraph/common" + "sentioxyz/sentio-core/driver/subgraph/ethereum" +) + +type BlockData struct { + evmExtend.BlockHeader + + mainData evmExtend.BlockMainData + extendData evmExtend.BlockExtendData + + cachedBlock *ethereum.Block + cachedTxn map[string]*ethereum.Transaction + cachedTxnReceipt map[string]*ethereum.TransactionReceipt + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} + +func (d *BlockData) buildBlock() (b *ethereum.Block, err error) { + if d.cachedBlock == nil { + var payload evm.ExtendedHeader + if err = json.Unmarshal(d.BlockHeader.Raw, &payload); err != nil { + return nil, errors.Wrapf(err, "unmarshal header for block %d failed: %s", + d.GetBlockNumber(), string(d.BlockHeader.Raw)) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum block %d from raw data (%s) failed: %v", + d.GetBlockNumber(), string(d.BlockHeader.Raw), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum block %d from raw data (%s) failed", + d.GetBlockNumber(), string(d.BlockHeader.Raw)) + } + } + }() + d.cachedBlock = ðereum.Block{ + Hash: wasm.MustBuildByteArrayFromHex(d.BlockHash), + ParentHash: wasm.MustBuildByteArrayFromHex(d.ParentBlockHash), + UnclesHash: wasm.MustBuildByteArrayFromHex(payload.UncleHash.Hex()), + StateRoot: wasm.MustBuildByteArrayFromHex(payload.Root.Hex()), + TransactionsRoot: wasm.MustBuildByteArrayFromHex(payload.TxHash.Hex()), + ReceiptsRoot: wasm.MustBuildByteArrayFromHex(payload.ReceiptHash.Hex()), + Number: common.MustBuildBigInt(d.BlockNumber), + GasUsed: common.MustBuildBigInt(payload.GasUsed), + GasLimit: common.MustBuildBigInt(payload.GasLimit), + Timestamp: common.MustBuildBigInt(payload.Time), + Difficulty: common.MustBuildBigInt(payload.Difficulty), + TotalDifficulty: common.MustBuildBigInt(payload.TotalDifficulty.ToInt()), + BaseFeePerGas: common.MustBuildBigInt(payload.BaseFee), + } + if payload.Author != "" { + d.cachedBlock.Author = common.MustBuildAddressFromString(payload.Author) + } + if payload.Size != nil { + d.cachedBlock.Size = common.MustBuildBigInt(uint64(*payload.Size)) + } + } + return d.cachedBlock, nil +} + +func (d *BlockData) transactionSucceed(txHash string) (bool, error) { + if receipt, has := d.extendData.Receipts[txHash]; !has { + return false, errors.Errorf("unreachable, transaction receipt %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } else { + return receipt.Status > 0, nil + } +} + +func (d *BlockData) buildTransaction(txHash string) (tx *ethereum.Transaction, err error) { + if d.cachedTxn == nil { + d.cachedTxn = make(map[string]*ethereum.Transaction) + } + var has bool + if tx, has = d.cachedTxn[txHash]; !has { + var payload evm.RPCTransaction + if payload, has = d.extendData.Transactions[txHash]; !has { + return nil, errors.Errorf("unreachable, transaction %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Transactions), d.BlockNumber) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum transaction %s from raw data (%s) failed: %v", + txHash, utils.MustJSONMarshal(payload), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum transaction %s from raw data (%s) failed", + txHash, utils.MustJSONMarshal(payload)) + } + } + }() + tx = ðereum.Transaction{ + Hash: wasm.MustBuildByteArrayFromHex(txHash), + Index: common.MustBuildBigInt(uint64(payload.TransactionIndex)), + From: common.MustBuildAddressFromString(payload.From.String()), + Value: common.MustBuildBigInt(payload.Value.ToInt()), + GasLimit: common.MustBuildBigInt(uint64(payload.Gas)), + GasPrice: common.MustBuildBigInt(payload.GasPrice.ToInt()), + Input: &wasm.ByteArray{Data: payload.Input}, + Nonce: common.MustBuildBigInt(uint64(payload.Nonce)), + } + if payload.To != nil { + tx.To = common.MustBuildAddressFromString(payload.To.String()) + } + d.cachedTxn[txHash] = tx + } + return +} + +func (d *BlockData) buildReceipt(txHash string) (receipt *ethereum.TransactionReceipt, err error) { + if d.cachedTxnReceipt == nil { + d.cachedTxnReceipt = make(map[string]*ethereum.TransactionReceipt) + } + var has bool + if receipt, has = d.cachedTxnReceipt[txHash]; !has { + var payload evm.ExtendedReceipt + if payload, has = d.extendData.Receipts[txHash]; !has { + return nil, errors.Errorf("unreachable, transaction receipt %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum transaction receipt %s from raw data (%s) failed: %v", + txHash, utils.MustJSONMarshal(payload), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum transaction receipt %s from raw data (%s) failed", + txHash, utils.MustJSONMarshal(payload)) + } + } + }() + receipt = ðereum.TransactionReceipt{ + TransactionHash: wasm.MustBuildByteArrayFromHex(txHash), + TransactionIndex: common.MustBuildBigInt(uint64(payload.TransactionIndex)), + BlockHash: &wasm.ByteArray{Data: payload.BlockHash.Bytes()}, + BlockNumber: common.MustBuildBigInt(payload.BlockNumber.ToInt()), + CumulativeGasUsed: common.MustBuildBigInt(uint64(payload.CumulativeGasUsed)), + GasUsed: common.MustBuildBigInt(uint64(payload.GasUsed)), + ContractAddress: nil, + Logs: nil, + Status: common.MustBuildBigInt(uint64(payload.Status)), + Root: &wasm.ByteArray{Data: payload.Root}, + LogsBloom: &wasm.ByteArray{Data: payload.Bloom.Bytes()}, + } + if payload.ContractAddress != nil { + receipt.ContractAddress = common.MustBuildAddressFromString(payload.ContractAddress.String()) + } + receipt.Logs = &wasm.ObjectArray[*ethereum.Log]{Data: make([]*ethereum.Log, len(payload.Logs))} + for i, rawLog := range payload.Logs { + receipt.Logs.Data[i] = ðereum.Log{ + Address: common.MustBuildAddressFromString(rawLog.Address.String()), + Topics: &wasm.ObjectArray[*wasm.ByteArray]{ + Data: utils.MapSliceNoError(rawLog.Topics, func(topic ethcommon.Hash) *wasm.ByteArray { + return &wasm.ByteArray{Data: topic.Bytes()} + }), + }, + Data: &wasm.ByteArray{Data: rawLog.Data}, + BlockHash: &wasm.ByteArray{Data: rawLog.BlockHash.Bytes()}, + BlockNumber: common.MustBuildBigInt(rawLog.BlockNumber), + TransactionHash: &wasm.ByteArray{Data: rawLog.TxHash.Bytes()}, + TransactionIndex: common.MustBuildBigInt(rawLog.TxIndex), + LogIndex: common.MustBuildBigInt(rawLog.Index), + TransactionLogIndex: common.MustBuildBigInt(i), + LogType: nil, // TODO unknown field + Removed: &common.Wrapped[wasm.Bool]{Inner: wasm.Bool(rawLog.Removed)}, + } + } + d.cachedTxnReceipt[txHash] = receipt + } + return +} + +func (d *BlockData) buildCall(trace evmExtend.Trace, funcABI *abi.Method) (*ethereum.Call, int, error) { + var size int + var payload evm.ParityTrace + err := json.Unmarshal(trace.Raw, &payload) + if err != nil { + return nil, size, errors.Wrapf(err, "unmarshal call trace failed: %s", string(trace.Raw)) + } + size += len(trace.Raw) + ca := ðereum.Call{} + ca.Block, err = d.buildBlock() + if err != nil { + return ca, size, err + } + size += len(d.BlockHeader.Raw) + ca.Transaction, err = d.buildTransaction(trace.TransactionHash) + if err != nil { + return ca, size, err + } + size += 1000 + ca.To, err = common.BuildAddressFromString(payload.Action.To) + if err != nil { + return ca, size, err + } + ca.From, err = common.BuildAddressFromString(payload.Action.From.Hex()) + if err != nil { + return ca, size, err + } + fn := abiutil.GetMethodSig(funcABI, true) + // first 4 bytes is method ID + ca.InputValues, err = ethereum.UnpackParams("input data of function "+fn, payload.Action.Input[4:], funcABI.Inputs) + if err != nil { + return ca, size, err + } + ca.OutputValues, err = ethereum.UnpackParams("output data of function "+fn, payload.Result.Output, funcABI.Outputs) + return ca, size, err +} + +func (d *BlockData) buildEvent(event types.Log, eventABI *abi.Event) (ev *ethereum.Event, size int, err error) { + txHash := event.TxHash.String() + receipt, has := d.extendData.Receipts[txHash] + if !has { + return nil, size, errors.Errorf("unreachable, receipt %s for the log %s not loaded, already loaded %v in block %d", + txHash, utils.MustJSONMarshal(event), utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } + var txLogIndex = -1 + for i, transactionLog := range receipt.Logs { + if transactionLog.Index == event.Index { + txLogIndex = i + break + } + } + if txLogIndex < 0 { + return nil, size, errors.Errorf("unreachable, log with logIndex %d not found in all transaction receipt logs %s", + event.Index, utils.MustJSONMarshal(receipt.Logs)) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum event in block %d from raw data (%s) failed: %v", + d.GetBlockNumber(), utils.MustJSONMarshal(event), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum event in block %d from raw data (%s) failed", + d.GetBlockNumber(), utils.MustJSONMarshal(event)) + } + } + }() + ev = ðereum.Event{ + Address: common.MustBuildAddressFromString(event.Address.String()), + LogIndex: common.MustBuildBigInt(uint64(event.Index)), + TransactionLogIndex: common.MustBuildBigInt(txLogIndex), + LogType: nil, // TODO unknown field + } + size += 1000 // event self + if ev.Block, err = d.buildBlock(); err != nil { + return ev, size, err + } + size += len(d.BlockHeader.Raw) + if ev.Transaction, err = d.buildTransaction(txHash); err != nil { + return ev, size, err + } + size += 1000 // tx part + if ev.Receipt, err = d.buildReceipt(txHash); err != nil { + return ev, size, err + } + size += 1000 + 1000*len(d.extendData.Receipts[txHash].Logs) // receipt main part + logs + + // arguments of the log + arguments := make(map[string]any) + + // unpack non-indexed arguments + if err = eventABI.Inputs.NonIndexed().UnpackIntoMap(arguments, event.Data); err != nil { + return nil, size, errors.Wrapf(ethereum.ErrABINotMatch, "unpack event data of raw log failed: %v", err) + } + + // parse indexed arguments + var indexedInputs abi.Arguments + for _, input := range eventABI.Inputs { + if input.Indexed { + indexedInputs = append(indexedInputs, input) + } + } + if err = abi.ParseTopicsIntoMap(arguments, indexedInputs, event.Topics[1:]); err != nil { + // TODO if the type of the indexed argument is tuple, here will got an error, it is unexpected + return nil, size, errors.Wrapf(ethereum.ErrABINotMatch, "parse topics of raw log failed: %v", err) + } + + // convert args to ev.Parameters + ev.Parameters = ðereum.EventParams{} + for _, input := range eventABI.Inputs { + ev.Parameters.Data = append(ev.Parameters.Data, ethereum.MustConvertEventParam(arguments[input.Name], input)) + } + return ev, size, nil +} diff --git a/driver/controller/subgraph/handler.go b/driver/controller/subgraph/handler.go new file mode 100644 index 0000000..e09f856 --- /dev/null +++ b/driver/controller/subgraph/handler.go @@ -0,0 +1,478 @@ +package subgraph + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" + "sentioxyz/sentio-core/service/processor/models" + + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" +) + +type HandlerAgent interface { + controller.HandlerAgent + + GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) + BuildTaskDataList(context.Context, *BlockData) ([]taskData, error) +} + +const ( + HandlerTypeEvent = "event" + HandlerTypeBlock = "block" + HandlerTypeCall = "call" +) + +type HandlerController struct { + processor *models.Processor + chainConfig *chain.ConfigV2 + client evm.Client + ipfsShell *shell.Shell + manifest *manifest.Manifest + memHardLimit uint32 + debugTrace bool + + instance *instance // TODO support multi instances + + agents []HandlerAgent // built from Manifest + + addressStart map[string]uint64 + addressStartData string + + waiter *concurrency.ResourceWaiter[uint64] +} + +func NewHandlerController( + ctx context.Context, + processor *models.Processor, + chainConfig *chain.ConfigV2, + client evm.Client, + ipfsShell *shell.Shell, + manifest *manifest.Manifest, + memHardLimit uint32, + debugTrace bool, +) (ctrl *HandlerController, err error) { + ctrl = &HandlerController{ + processor: processor, + chainConfig: chainConfig, + client: client, + ipfsShell: ipfsShell, + manifest: manifest, + memHardLimit: memHardLimit, + debugTrace: debugTrace, + } + ctrl.instance, err = ctrl.newInstance(ctx) + return ctrl, err +} + +func (c *HandlerController) chainID() string { + return c.chainConfig.ChainID +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + // reset instance + if err := c.instance.Reset(ctx); err != nil { + return controller.NewExternalError(controller.ErrCodeResetWasmInstanceFailed, err) + } + // load contract start + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + // build agents + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber(), templates); extErr != nil { + return extErr + } + // build checkpoint data for contract start + c.AddressStartReady() + // reset waiter + c.waiter = concurrency.NewResourceWaiter[uint64]() + return nil +} + +func (c *HandlerController) buildAgents( + ctx context.Context, + first, latest uint64, + templates map[uint64][]controller.TemplateInstance, +) (extErr *controller.ExternalError) { + _, logger := log.FromContext(ctx) + // collect data sources + dataSources := c.manifest.DataSources + for _, tpls := range utils.GetMapValuesOrderByKey(templates) { + for _, tpl := range tpls { + if total := len(c.manifest.Templates); tpl.TemplateID < 0 || int(tpl.TemplateID) >= total { + logger.Warnf("template id %d out of range [0,%d), will be ignored", tpl.TemplateID, total) + continue + } + dataSources = append(dataSources, c.manifest.Templates[tpl.TemplateID].NewDataSource( + tpl.Address, + manifest.BuildBigIntFromUint(tpl.StartBlock), + tpl.TemplateName, + )) + } + } + // build agents + c.agents = nil + for dataSourceID, ds := range dataSources { + blockRange := controller.BlockRange{StartBlock: max(ds.Source.GetStartBlock(), first)} + if ds.Source.EndBlock != nil { + blockRange.EndBlock = utils.WrapPointer(ds.Source.GetEndBlock()) + } + contractRange := blockRange + var err error + contractRange.StartBlock, err = c.GetAddressStart(ctx, ds.Source.Address, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + for i, eventHandler := range ds.Mapping.EventHandlers { + if ds.Source.Address == "" { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("data source #%d %s has event handler but no contract address", dataSourceID, ds.Name)) + } + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeEvent, + Name: eventHandler.Handler, + ID: int32(i), + }, + Range: contractRange, + }, + DataSource: ds, + Filter: evm.LogFilter{ + Topics: [][]string{{eventHandler.Topic0}}, + Address: []string{strings.ToLower(ds.Source.Address)}, + }, + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + for i, callHandler := range ds.Mapping.CallHandlers { + if ds.Source.Address == "" { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("data source #%d %s has call handler but no contract address", dataSourceID, ds.Name)) + } + agent := HandlerAgentCall{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeCall, + Name: callHandler.Handler, + ID: int32(i), + }, + Range: contractRange, + }, + DataSource: ds, + Filter: evm.TraceFilter{ + Signature: []string{callHandler.Signature}, + Address: []string{strings.ToLower(ds.Source.Address)}, + }, + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + for i, blockHandler := range ds.Mapping.BlockHandlers { + agent := HandlerAgentBlock{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeBlock, + Name: blockHandler.Handler, + ID: int32(i), + }, + Range: blockRange, + }, + DataSource: ds, + } + switch kind := blockHandler.Filter.GetKind(); kind { + case "once": + agent.Once = true + case "polling": + interval := blockHandler.Filter.GetEvery() + if interval <= 0 { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("every should greater than 0 in data source %s #%d block handler", ds.Name, i)) + } + agent.IntervalConfig = data.IntervalConfig{ + BlockInterval: &data.BlockInterval{Backfill: uint64(interval), Watching: uint64(interval)}, + } + default: + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("filter kind %q is not supported in data source %s #%d block handler", kind, ds.Name, i)) + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + return nil +} + +func (c *HandlerController) GetBlockRange() controller.BlockRange { + return controller.GetHandleAgentsBlockRange(c.agents) +} + +func (c *HandlerController) GetAgentStat() map[string]int { + stat := make(map[string]int) + for _, ag := range c.agents { + stat[fmt.Sprintf("%T", ag)] += 1 + } + return stat +} + +func (c *HandlerController) buildReportRequirements( + currentBlockNumber uint64, +) []data.IntervalRequirement { + endBlock := c.GetBlockRange().EndBlock + // In the backfill phase, at least one non-empty BlockData is generated for every DAY. + // In the watching phase, at least one non-empty BlockData is generated for every MINUTE. + reqs := []data.IntervalRequirement{{ + BlockRange: controller.BlockRange{StartBlock: currentBlockNumber, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 24, + Watching: time.Minute, + }}, + }} + if endBlock != nil { + reqs = append(reqs, data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: *endBlock, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{BlockInterval: &data.BlockInterval{ + Backfill: 1, + Watching: 1, + }}, + }) + } + return reqs +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.buildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("EVM::%s::", c.chainID()) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + evm.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, + 100, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from evm.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.BlockHeader, err = c.client.GetHeader(ctx, blockNumber); err != nil { + return nil, false, err + } + // check block hash of main data with the header got above + for _, l := range from.Logs { + if l.BlockHash.String() != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the log %s, expected is %s", + l.BlockHash.String(), controller.GetBlockSummary(result.BlockHeader))) + } + } + for _, t := range from.Traces { + if t.BlockHash != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the trace %s, expected is %s", + t.BlockHash, controller.GetBlockSummary(result.BlockHeader))) + } + } + // take the main data and ask the handler controller what extend data is needed + var r evm.BlockExtendRequirement + if r, err = c.getBlockExtendRequirements(ctx, &result); err != nil { + return nil, false, err + } + // actually get the extended data + if result.extendData, err = c.client.GetBlock(ctx, blockNumber, r); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + _, logger := log.FromContext(ctx, + "header", utils.MustJSONMarshal(result.BlockHeader), + "mainData", utils.MustJSONMarshal(result.mainData), + "extendData", utils.MustJSONMarshal(result.extendData)) + logger.Warnfe(err, "build task list failed") + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getDataRequirement() (dr evm.DataRequirement) { + for _, agent := range c.agents { + switch ag := agent.(type) { + case HandlerAgentBlock: + if !ag.Once { + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } else { + dr.Exact = append(dr.Exact, ag.Range.StartBlock) + } + case HandlerAgentCall: + dr.Trace = append(dr.Trace, evm.TraceRequirement{ + TraceFilter: ag.Filter, + BlockRange: ag.Range, + }) + case HandlerAgentEvent: + dr.Log = append(dr.Log, evm.LogRequirement{ + LogFilter: ag.Filter, + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) getBlockExtendRequirements( + ctx context.Context, + blockData *BlockData, +) (req evm.BlockExtendRequirement, err error) { + var ar evm.BlockExtendRequirement + for _, agent := range c.agents { + if ar, err = agent.GetExtendRequirements(ctx, blockData); err != nil { + return + } + req.Merge(ar) + } + return +} + +func (c *HandlerController) BuildTaskList( + ctx context.Context, + bd *BlockData, +) ([]controller.Task, int, error) { + var taskDatas []taskData + var taskTotalSize int + for _, agent := range c.agents { // 15308711065 + if agent.GetRange().Contains(bd.GetBlockNumber()) { + tds, err := agent.BuildTaskDataList(ctx, bd) + if err != nil { + return nil, 0, err + } + taskDatas = append(taskDatas, tds...) + for _, td := range tds { + taskTotalSize += td.size + } + } + } + // The purpose of using stable sorting is to ensure that tasks of the same handler remain in the order + // in which they were generated. + sort.SliceStable(taskDatas, func(i, j int) bool { + return taskDatas[i].Cmp(taskDatas[j]) < 0 + }) + var r []controller.Task + for _, td := range taskDatas { + r = append(r, &task{ + handlerCtrl: c, + BlockHeader: bd.BlockHeader, + taskData: td, + }) + } + return r, taskTotalSize, nil +} + +const checkpointDataKeyAddressStart = "AddressStart" + +func (c *HandlerController) LoadAddressStart(checkpoint *controller.Checkpoint) *controller.ExternalError { + c.addressStart = make(map[string]uint64) + if checkpoint == nil || checkpoint.Data == nil { + return nil + } + raw, has := checkpoint.Data[checkpointDataKeyAddressStart] + if !has { + return nil + } + err := json.Unmarshal([]byte(raw), &c.addressStart) + if err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "load address start failed")) + } + return nil +} + +func (c *HandlerController) GetAddressStart( + ctx context.Context, + address string, + start uint64, + latest uint64, +) (uint64, error) { + if c.chainConfig.SkipStartBlockValidation || controller.SkipStartBlockValidation || address == "" { + return start, nil + } + var has bool + if start, has = c.addressStart[address]; has { + return start, nil + } + var err error + start, has, err = c.client.GetContractStartBlock(ctx, address, start, latest) + if err != nil { + return 0, errors.Wrapf(err, "get start block for contract %s failed", address) + } + if !has { + start = latest + 1 + } + c.addressStart[address] = start + return start, nil +} + +func (c *HandlerController) AddressStartReady() { + b, _ := json.Marshal(c.addressStart) + c.addressStartData = string(b) +} + +func (c *HandlerController) DumpAddressStart(checkpointData map[string]string) { + checkpointData[checkpointDataKeyAddressStart] = c.addressStartData +} + +func (c *HandlerController) Epilogue() { +} + +func (c *HandlerController) Snapshot() any { + return map[string]any{ + "chainConfig": c.chainConfig, + "manifest": c.manifest, + "memHardLimit": c.memHardLimit, + "debugTrace": c.debugTrace, + "agents": utils.MapSliceNoError(c.agents, func(a HandlerAgent) any { + return a.Snapshot() + }), + "addressStart": c.addressStart, + "agentStat": c.GetAgentStat(), + "wasmInstance": c.instance.Snapshot(), + } +} diff --git a/driver/controller/subgraph/handler_block.go b/driver/controller/subgraph/handler_block.go new file mode 100644 index 0000000..b11b39f --- /dev/null +++ b/driver/controller/subgraph/handler_block.go @@ -0,0 +1,61 @@ +package subgraph + +import ( + "context" + "math" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentBlock struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + IntervalConfig data.IntervalConfig + Once bool +} + +func (a HandlerAgentBlock) GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) { + return evm.BlockExtendRequirement{}, nil +} + +func (a HandlerAgentBlock) BuildTaskDataList(_ context.Context, bd *BlockData) ([]taskData, error) { + if a.Once { + if bd.GetBlockNumber() != a.Range.StartBlock { + return nil, nil + } + } else { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + } + block, err := bd.buildBlock() + if err != nil { + return nil, fetcher.Permanent(err) + } + return []taskData{{ + callHandlerParam: block, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: utils.Select(a.Once, -1, math.MaxInt), + size: len(bd.BlockHeader.Raw), + }}, nil +} + +func (a HandlerAgentBlock) Snapshot() any { + sn := map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + } + if a.Once { + sn["Once"] = true + } else { + sn["IntervalConfig"] = a.IntervalConfig + } + return sn +} diff --git a/driver/controller/subgraph/handler_call.go b/driver/controller/subgraph/handler_call.go new file mode 100644 index 0000000..c0832e9 --- /dev/null +++ b/driver/controller/subgraph/handler_call.go @@ -0,0 +1,58 @@ +package subgraph + +import ( + "context" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentCall struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + Filter evm.TraceFilter +} + +func (a HandlerAgentCall) GetExtendRequirements(_ context.Context, bd *BlockData) (evm.BlockExtendRequirement, error) { + txHashSet := set.New[string]() + for _, trace := range bd.mainData.Traces { + if a.Filter.Check(trace) { + txHashSet.Add(trace.TransactionHash) + } + } + return evm.BlockExtendRequirement{SpecialTransactions: txHashSet.DumpValues()}, nil +} + +func (a HandlerAgentCall) BuildTaskDataList(ctx context.Context, bd *BlockData) ([]taskData, error) { + funcAbi := a.DataSource.Mapping.CallHandlers[a.HandlerID.ID].GetABI() + var r []taskData + for _, trace := range bd.mainData.Traces { + if !a.Filter.Check(trace) { + continue + } + call, size, err := bd.buildCall(trace, funcAbi) + if err != nil { + return nil, fetcher.Permanent(err) + } + r = append(r, taskData{ + callHandlerParam: call, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: int(trace.TransactionIndex), + size: size, + }) + } + return r, nil +} + +func (a HandlerAgentCall) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/subgraph/handler_event.go b/driver/controller/subgraph/handler_event.go new file mode 100644 index 0000000..0c3f70e --- /dev/null +++ b/driver/controller/subgraph/handler_event.go @@ -0,0 +1,71 @@ +package subgraph + +import ( + "context" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + Filter evm.LogFilter +} + +func (a HandlerAgentEvent) GetExtendRequirements(_ context.Context, bd *BlockData) (evm.BlockExtendRequirement, error) { + txHashSet := set.New[string]() + checker := a.Filter.BuildChecker(nil, nil) + for _, log := range bd.mainData.Logs { + if ok, _ := checker(log); ok { + txHashSet.Add(log.TxHash.String()) + } + } + txHashes := txHashSet.DumpValues() + return evm.BlockExtendRequirement{ + SpecialTransactions: txHashes, + SpecialTransactionReceipts: txHashes, + SpecialTransactionReceiptLogs: txHashes, + }, nil +} + +func (a HandlerAgentEvent) BuildTaskDataList(_ context.Context, bd *BlockData) ([]taskData, error) { + eventAbi := a.DataSource.Mapping.EventHandlers[a.HandlerID.ID].GetABI() + var r []taskData + checker := a.Filter.BuildChecker(nil, nil) + for _, log := range bd.mainData.Logs { + if ok, _ := checker(log); !ok { + continue + } + if succeed, err := bd.transactionSucceed(log.TxHash.String()); err != nil { + return nil, fetcher.Permanent(err) + } else if !succeed { + continue // tx failed + } + event, size, err := bd.buildEvent(log, eventAbi) + if err != nil { + return nil, fetcher.Permanent(err) + } + r = append(r, taskData{ + callHandlerParam: event, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: int(log.TxIndex), + logIndex: int(log.Index), + size: size, + }) + } + return r, nil +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/subgraph/instance.go b/driver/controller/subgraph/instance.go new file mode 100644 index 0000000..24de9e9 --- /dev/null +++ b/driver/controller/subgraph/instance.go @@ -0,0 +1,787 @@ +package subgraph + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strconv" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/subgraph/common" + "sentioxyz/sentio-core/driver/subgraph/ethereum" + "sentioxyz/sentio-core/driver/subgraph/manifest" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/mr-tron/base58/base58" + "github.com/pkg/errors" +) + +func (c *HandlerController) newInstance(ctx context.Context) (*instance, error) { + inst := &instance{ + mods: make(map[string]*wasm.Instance[CtxData]), + handlerCtrl: c, + } + + hashBelong := make(map[string][]string) + + _ = c.manifest.TravelDataSourcesAndTemplates(func(ds *manifest.DataSource, name string) error { + hash := ds.Mapping.File.GetIpfsHash() + m, has := inst.mods[hash] + if !has { + m = wasm.NewInstance[CtxData]( + fmt.Sprintf("%s/%s", name, hash), + []byte(ds.Mapping.File.GetContent()), + c.memHardLimit, + ) + inst.importFunctions(m) + if c.debugTrace { + m.SetDebugLevel(wasm.DebugLevelTrace) + } + } + switch ds.Kind { + case "ethereum/contract", "ethereum": + for _, evh := range ds.Mapping.EventHandlers { + m.MustExportFunction(evh.Handler, (func(*ethereum.Event))(nil)) + } + for _, cah := range ds.Mapping.CallHandlers { + m.MustExportFunction(cah.Handler, (func(*ethereum.Call))(nil)) + } + for _, blh := range ds.Mapping.BlockHandlers { + m.MustExportFunction(blh.Handler, (func(*ethereum.Block))(nil)) + } + case "file/ipfs": + m.MustExportFunction(ds.Mapping.Handler, (func(*wasm.ByteArray))(nil)) + } + inst.mods[hash] = m + hashBelong[hash] = append(hashBelong[hash], name) + return nil + }) + + // init modules + _, logger := log.FromContext(ctx) + for hash, m := range inst.mods { + if err := m.Init(logger); err != nil { + return nil, errors.Wrapf(err, "init wasm instance with ipfs hash %q for %v failed", hash, hashBelong[hash]) + } + } + + return inst, nil +} + +type CtxData struct { + task *task + checkpointCtrl controller.CheckpointController + + // This is the data source of the calling handler. + // Normally this should be equal to task.dataSource, when the entry handler of the task calling a file template + // handler, this will be a dynamic data source created from the file template + dataSource *manifest.DataSource +} + +func (c CtxData) String() string { + return "" +} + +type instance struct { + mods map[string]*wasm.Instance[CtxData] + handlerCtrl *HandlerController +} + +const ( + LogLevelCritical = 0 + LogLevelError = 1 + LogLevelWarning = 2 + LogLevelInfo = 3 + LogLevelDebug = 4 +) + +func (inst *instance) importFunctions(m *wasm.Instance[CtxData]) { + // TODO more functions + + // /** + // * Special function for ENS name lookups, not meant for general purpose use. + // * This function will only be useful if the graph-node instance has additional + // * data loaded ** + // */ + //export declare namespace ens { + // function nameByHash(hash: string): string | null; + // } + + m.MustImportFunction("env", "abort", + func(ctx *wasm.CallContext[CtxData], msg *wasm.String, filename *wasm.String, lineNum wasm.I32, colNum wasm.I32) { + errMsg := "" + if msg != nil && msg.String() != "" { + errMsg = msg.String() + } + abortErr := fmt.Errorf("abort at %s:%d:%d while calling %s, %s", + filename, lineNum, colNum, ctx.DumpCallStack(), errMsg) + ctx.Logger().UserVisible().Error(abortErr.Error()) + panic(abortErr) + }). + MustImportFunction("wasi_snapshot_preview1", "fd_write", + func(ctx *wasm.CallContext[CtxData], _, _, _, _ wasm.I32) wasm.U16 { + ctx.Logger().UserVisible().Warnf("Calling %s => wasi_snapshot_preview1.fd_write was ignored. "+ + "If you want to print log, please use log in @graphprotocol/graph-ts", ctx.DumpCallStack()) + return 0 + }). + //export declare namespace typeConversion { + // function bytesToString(bytes: Uint8Array): string + // function bytesToHex(bytes: Uint8Array): string + // function bigIntToString(bigInt: Uint8Array): string + // function bigIntToHex(bigInt: Uint8Array): string + // function stringToH160(s: string): Bytes + // function bytesToBase58(n: Uint8Array): string + //} + MustImportFunction("conversion", "typeConversion.bytesToString", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildStringFromBytes(arg.Data) + }). + MustImportFunction("conversion", "typeConversion.bytesToHex", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + return wasm.BuildString(arg.ToHex()) + }). + MustImportFunction("conversion", "typeConversion.bigIntToString", + func(_ *wasm.CallContext[CtxData], arg *common.BigInt) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildString(arg.String()) + }). + MustImportFunction("conversion", "typeConversion.bigIntToHex", + func(_ *wasm.CallContext[CtxData], arg *common.BigInt) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildString(arg.ToHex()) + }). + MustImportFunction("conversion", "typeConversion.stringToH160", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *wasm.ByteArray { + arr, buildErr := wasm.BuildByteArrayFromHex(arg.String()) + if buildErr != nil { + panic(buildErr) + } + return arr + }). + MustImportFunction("conversion", "typeConversion.bytesToBase58", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + if arg == nil { + return wasm.BuildString(base58.Encode(nil)) + } + return wasm.BuildString(base58.Encode(arg.Data)) + }). + //export declare namespace bigInt { + // function plus(x: BigInt, y: BigInt): BigInt + // function minus(x: BigInt, y: BigInt): BigInt + // function times(x: BigInt, y: BigInt): BigInt + // function dividedBy(x: BigInt, y: BigInt): BigInt + // function dividedByDecimal(x: BigInt, y: BigDecimal): BigDecimal + // function mod(x: BigInt, y: BigInt): BigInt + // function pow(x: BigInt, exp: u8): BigInt + // function fromString(s: string): BigInt + // function bitOr(x: BigInt, y: BigInt): BigInt + // function bitAnd(x: BigInt, y: BigInt): BigInt + // function leftShift(x: BigInt, bits: u8): BigInt + // function rightShift(x: BigInt, bits: u8): BigInt + //} + MustImportFunction("numbers", "bigInt.plus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Plus(y) + }). + MustImportFunction("numbers", "bigInt.minus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Minus(y) + }). + MustImportFunction("numbers", "bigInt.times", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Times(y) + }). + MustImportFunction("numbers", "bigInt.dividedBy", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.DividedBy(y) + }). + MustImportFunction("numbers", "bigInt.dividedByDecimal", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y *common.BigDecimal) *common.BigDecimal { + return common.BuildBigDecimalFromBigInt(x, 0).DividedBy(y) + }). + MustImportFunction("numbers", "bigInt.mod", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Mod(y) + }). + MustImportFunction("numbers", "bigInt.pow", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, exp wasm.U8) *common.BigInt { + return x.Pow(exp) + }). + MustImportFunction("numbers", "bigInt.fromString", + func(_ *wasm.CallContext[CtxData], x *wasm.String) *common.BigInt { + return common.MustBuildBigInt(x.String()) + }). + MustImportFunction("numbers", "bigInt.bitOr", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.BitOr(y) + }). + MustImportFunction("numbers", "bigInt.bitAnd", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.BitAnd(y) + }). + MustImportFunction("numbers", "bigInt.leftShift", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y wasm.U8) *common.BigInt { + return x.LeftShift(y) + }). + MustImportFunction("numbers", "bigInt.rightShift", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y wasm.U8) *common.BigInt { + return x.RightShift(y) + }). + //export declare namespace bigDecimal { + // function plus(x: BigDecimal, y: BigDecimal): BigDecimal + // function minus(x: BigDecimal, y: BigDecimal): BigDecimal + // function times(x: BigDecimal, y: BigDecimal): BigDecimal + // function dividedBy(x: BigDecimal, y: BigDecimal): BigDecimal + // function equals(x: BigDecimal, y: BigDecimal): boolean + // function toString(bigDecimal: BigDecimal): string + // function fromString(s: string): BigDecimal + //} + MustImportFunction("numbers", "bigDecimal.plus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Plus(y) + }). + MustImportFunction("numbers", "bigDecimal.minus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Minus(y) + }). + MustImportFunction("numbers", "bigDecimal.times", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Times(y) + }). + MustImportFunction("numbers", "bigDecimal.dividedBy", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.DividedBy(y) + }). + MustImportFunction("numbers", "bigDecimal.equals", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) wasm.Bool { + return wasm.Bool(x.Equals(y)) + }). + MustImportFunction("numbers", "bigDecimal.toString", + func(_ *wasm.CallContext[CtxData], arg *common.BigDecimal) *wasm.String { + return wasm.BuildString(arg.String()) + }). + MustImportFunction("numbers", "bigDecimal.fromString", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *common.BigDecimal { + return common.MustBuildBigDecimalFromString(arg.String()) + }). + //export declare namespace json { + // function fromBytes(data: Bytes): JSONValue; + // function try_fromBytes(data: Bytes): Result; + // function toI64(decimal: string): i64; + // function toU64(decimal: string): u64; + // function toF64(decimal: string): f64; + // function toBigInt(decimal: string): BigInt; + //} + MustImportFunction("json", "json.fromBytes", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *common.JSONValue { + val := &common.JSONValue{} + if err := val.FromBytes(arg.Data); err != nil { + panic(err) + } + return val + }). + MustImportFunction("json", "json.try_fromBytes", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *common.Result[*common.JSONValue, wasm.Bool] { + result := &common.Result[*common.JSONValue, wasm.Bool]{} + var val common.JSONValue + if err := val.FromBytes(arg.Data); err != nil { + result.Error = &common.Wrapped[wasm.Bool]{Inner: true} + } else { + result.Value = &common.Wrapped[*common.JSONValue]{Inner: &val} + } + return result + }). + MustImportFunction("json", "json.toI64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.I64 { + v, err := strconv.ParseInt(arg.String(), 0, 64) + if err != nil { + panic(err) + } + return wasm.I64(v) + }). + MustImportFunction("json", "json.toU64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.U64 { + v, err := strconv.ParseUint(arg.String(), 0, 64) + if err != nil { + panic(err) + } + return wasm.U64(v) + }). + MustImportFunction("json", "json.toF64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.F64 { + v, err := strconv.ParseFloat(arg.String(), 64) + if err != nil { + panic(err) + } + return wasm.F64(v) + }). + MustImportFunction("json", "json.toBigInt", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *common.BigInt { + return common.MustBuildBigInt(arg.String()) + }). + //export declare namespace crypto { + // function keccak256(input: ByteArray): ByteArray + //} + MustImportFunction("index", "crypto.keccak256", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.ByteArray { + return &wasm.ByteArray{Data: crypto.Keccak256(arg.Data)} + }). + //export declare namespace ethereum { + // function call(call: SmartContractCall): Array | null + // function encode(token: Value): Bytes | null + // function decode(types: String, data: Bytes): Value | null + // } + MustImportFunction("ethereum", "ethereum.call", inst.EthCall). + MustImportFunction("ethereum", "ethereum.encode", inst.EthEncode). + MustImportFunction("ethereum", "ethereum.decode", inst.EthDecode). + //export declare namespace dataSource { + // function create(name: string, params: Array): void + // function createWithContext( + // name: string, + // params: Array, + // context: DataSourceContext, + // ): void + // + // // Properties of the data source that fired the event. + // function address(): Address + // function network(): string + // function context(): DataSourceContext + // } + MustImportFunction("datasource", "dataSource.create", inst.CreateDataSource). + MustImportFunction("datasource", "dataSource.createWithContext", inst.CreateDataSourceWithCtx). + MustImportFunction("datasource", "dataSource.network", inst.GetNetwork). + MustImportFunction("datasource", "dataSource.address", + func(ctx *wasm.CallContext[CtxData]) *common.Address { + return common.MustBuildAddressFromString(ctx.TopParams().Data.dataSource.Source.Address) + }). + MustImportFunction("datasource", "dataSource.context", + func(ctx *wasm.CallContext[CtxData]) *common.Entity { + ctxText := ctx.TopParams().Data.dataSource.Context + if ctxText == "" { + return &common.Entity{Properties: &wasm.ObjectArray[*common.EntityProperty]{}} // empty context + } + var ctxEntity common.Entity + if err := json.Unmarshal([]byte(ctxText), &ctxEntity); err != nil { + panic(err) + } + return &ctxEntity + }). + //export declare namespace ipfs { + // function cat(hash: string): Bytes | null + // function map(hash: string, callback: string, userData: Value, flags: string[]): void + //} + MustImportFunction("index", "ipfs.cat", + func(ctx *wasm.CallContext[CtxData], hash *wasm.String) *wasm.ByteArray { + r, err := inst.handlerCtrl.ipfsShell.Cat(hash.String()) + if err != nil { + ctx.Logger().UserVisible().Warnf("ipfs cat %q failed: %v", hash.String(), err) + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(err, "ipfs cat %q failed", hash.String()))) + } + cnt, readErr := io.ReadAll(r) + if readErr != nil { + ctx.Logger().UserVisible().Warnf("ipfs cat %q failed: read failed: %v", hash.String(), readErr) + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(err, "ipfs cat %q failed", hash.String()))) + } + return &wasm.ByteArray{Data: cnt} + }). + MustImportFunction("index", "ipfs.map", + func( + _ *wasm.CallContext[CtxData], + hash *wasm.String, + callback *wasm.String, + userData *common.Value, + flags *wasm.ObjectArray[*wasm.String], + ) { + panic(errors.Errorf("ipfs.map is not supported")) + }). + //export declare namespace store { + // function get(entity: string, id: string): Entity | null + // function get_in_block(entity: string, id: string): Entity | null; + // function loadRelated(entity: string, id: string, field: string): Array; + // function set(entity: string, id: string, data: Entity): void + // function remove(entity: string, id: string): void + // } + MustImportFunction("index", "store.set", inst.SetEntity). + MustImportFunction("index", "store.get", inst.GetEntity). + MustImportFunction("index", "store.get_in_block", inst.GetEntityInBlock). + MustImportFunction("index", "store.loadRelated", inst.GetRelatedEntities). + MustImportFunction("index", "store.remove", inst.RemoveEntity). + //export declare namespace log { + // // Host export for logging, providing basic logging functionality + // export function log(level: Level, msg: string): void + //} + MustImportFunction("index", "log.log", + func(ctx *wasm.CallContext[CtxData], level wasm.I32, msg *wasm.String) { + logger := ctx.Logger().UserVisible() + switch level { + case LogLevelCritical: + logger.Fatal(msg.String()) + case LogLevelError: + logger.Error(msg.String()) + case LogLevelWarning: + logger.Warn(msg.String()) + case LogLevelInfo: + logger.Info(msg.String()) + case LogLevelDebug: + logger.Debug(msg.String()) + default: + panic(errors.Errorf("invalid log level %d with msg %q", level, msg.String())) + } + }) +} + +func (inst *instance) getEntity( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, + inBlock bool, +) (entity *common.Entity) { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id := argName.String(), argID.String() + entityType := ckc.GetEntityOrInterfaceType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeGetUnknownEntity, errors.Errorf("get unknown entity %q", name))) + } + var box *persistent.EntityBox + var extErr *controller.ExternalError + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + if inBlock { + box, extErr = ckc.GetEntityInBlock(ctxEx, entityType, id, tk.GetBlockNumber()) + } else { + box, extErr = ckc.GetEntity(ctxEx, entityType, id, tk.GetBlockNumber()) + } + if extErr != nil { + panic(extErr.Wrapf("get entity %s/%s failed", name, id)) + } + if box != nil && box.Data != nil { + entity = &common.Entity{} + entity.FromGoType(box.Data, entityType) + } + return entity +} + +func (inst *instance) GetEntity( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, +) (entity *common.Entity) { + return inst.getEntity(ctx, argName, argID, false) +} + +func (inst *instance) GetEntityInBlock( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, +) (entity *common.Entity) { + return inst.getEntity(ctx, argName, argID, true) +} + +func (inst *instance) GetRelatedEntities( + ctx *wasm.CallContext[CtxData], + argName, argID, argField *wasm.String, +) *wasm.ObjectArray[*common.Entity] { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id, field := argName.String(), argID.String(), argField.String() + entityType := ckc.GetEntityType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("list related with unknown entity %q", name))) + } + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + boxes, target, extErr := ckc.ListRelated(ctxEx, entityType, id, field, tk.GetBlockNumber()) + if extErr != nil { + panic(extErr) + } + arr := &wasm.ObjectArray[*common.Entity]{Data: make([]*common.Entity, len(boxes))} + for i, box := range boxes { + if box != nil && box.Data != nil { + arr.Data[i] = &common.Entity{} + arr.Data[i].FromGoType(box.Data, target) + } + } + return arr +} + +func (inst *instance) SetEntity( + ctx *wasm.CallContext[CtxData], + argName, argID *wasm.String, + entity *common.Entity, +) { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id := argName.String(), argID.String() + entityType := ckc.GetEntityType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("set unknown entity %q", name))) + } + + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: tk.GetBlockNumber(), + GenBlockTime: tk.GetBlockTime(), + GenBlockHash: tk.GetBlockHash(), + }} + if entity != nil { + box.Data = entity.ToGoType() + box.FillLostFields(make(map[string]any), entityType) + } + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + if extErr := ckc.SetEntity(ctxEx, entityType, box); extErr != nil { + panic(extErr.Wrapf("set entity %s/%s %s failed", name, id, box.String())) + } + subtype := "upsert" + if entity == nil { + subtype = "delete" + } + controller.N.DataEmitted(ctxEx, tk.taskInfo(), "entity", subtype, name, 1) +} + +func (inst *instance) RemoveEntity(ctx *wasm.CallContext[CtxData], name *wasm.String, id *wasm.String) { + inst.SetEntity(ctx, name, id, nil) +} + +func (inst *instance) GetNetwork(ctx *wasm.CallContext[CtxData]) *wasm.String { + return wasm.BuildString(inst.handlerCtrl.chainID()) +} + +func (inst *instance) EthCall( + ctx *wasm.CallContext[CtxData], + call *ethereum.SmartContractCall, +) *wasm.ObjectArray[*ethereum.Value] { + start := time.Now() + top := ctx.TopParams() + tk, ds := top.Data.task, top.Data.dataSource + contractName, methodSig := call.ContractName.String(), call.FunctionSignature.String() + contractABI := ds.GetABIByName(contractName) + if contractABI == nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Errorf("contract %s is not found in data source %s", contractName, ds.Name))) + } + methodABI := contractABI.FindMethodBySig(methodSig) + if methodABI == nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Errorf("method with signature %q is not found in contract %s in data source %s", + contractName, methodSig, ds.Name))) + } + logger := ctx.Logger().With("ethCall", map[string]any{ + "contractName": call.ContractName.String(), + "contractAddr": call.ContractAddress.String(), + "functionName": call.FunctionName.String(), + "functionSignature": call.FunctionSignature.String(), + }) + ret, err := ethereum.EthCall( + ctx, + logger, + inst.handlerCtrl.client, + call.ContractAddress.Data, + methodABI, + call.FunctionParams, + tk.GetBlockNumber()) + + controller.N.SubgraphRPCDone(ctx, tk.taskInfoForCall(), err == nil, time.Since(start)) + + if err != nil { + if errors.Is(err, ethereum.ErrEthCallDataFormatErr) { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Wrapf(err, "calling %s.%s in data source %s failed", contractName, methodSig, ds.Name))) + } + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallFailed, + errors.Wrapf(err, "calling %s.%s in data source %s failed", contractName, methodSig, ds.Name))) + } + return ret +} + +func (inst *instance) EthEncode(ctx *wasm.CallContext[CtxData], value *ethereum.Value) *wasm.ByteArray { + logger := ctx.Logger().With("value", value.String()) + b, err := ethereum.Encode(value) + if err != nil { + logger.Warne(err, "ethereum.encode failed") + return nil + } + result := &wasm.ByteArray{Data: b} + logger.Debugw("ethereum.encode succeed", "result", result.String()) + return result +} + +func (inst *instance) EthDecode(ctx *wasm.CallContext[CtxData], types *wasm.String, data *wasm.ByteArray) *ethereum.Value { + logger := ctx.Logger().With("types", types.String(), "data", data.String()) + val, err := ethereum.Decode(types.String(), data.Data) + if err != nil { + logger.Warne(err, "ethereum.decode failed") + return nil + } + logger.Debugw("ethereum.decode succeed", "value", val.String()) + return val +} + +func (inst *instance) CreateDataSource(ctx *wasm.CallContext[CtxData], tplName *wasm.String, params *wasm.ObjectArray[*wasm.String]) { + inst.CreateDataSourceWithCtx(ctx, tplName, params, nil) +} + +func (inst *instance) CreateDataSourceWithCtx( + ctx *wasm.CallContext[CtxData], + tplName *wasm.String, + params *wasm.ObjectArray[*wasm.String], + ctxEntity *common.Entity, +) { + top := ctx.TopParams() + tk := top.Data.task + // find template + templateID, tpl := tk.handlerCtrl.manifest.FindTemplateByName(tplName.String()) + if templateID < 0 { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, errors.Errorf( + "create data source by template with name %q failed: template not found", tplName.String()))) + } + createFailedText := fmt.Sprintf("create data source by template #%d/%s failed", templateID, tplName.String()) + // need one param as contract address or file hash + if len(params.Data) == 0 { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, + errors.Errorf("%s: params is empty", createFailedText))) + } + // build context text + var ctxStr string + if ctxEntity != nil { + ctxBytes, err := json.Marshal(ctxEntity) + if err != nil { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, + errors.Wrapf(err, "%s: marshal data source ctx %s failed", createFailedText, ctxEntity.String()))) + } + ctxStr = string(ctxBytes) + } + + // file template, ipfs cat file and call file handler + if tpl.Kind == "file/ipfs" { + handlerFullName := fmt.Sprintf("%s/%s", tplName.String(), tpl.Mapping.Handler) + file := params.Data[0].String() + r, catErr := tk.handlerCtrl.ipfsShell.Cat(file) + if catErr != nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(catErr, "%s: ipfs cat %q failed", createFailedText, file))) + } + cnt, readErr := io.ReadAll(r) + if readErr != nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(catErr, "%s: ipfs cat %q failed", createFailedText, file))) + } + logger := ctx.Logger().UserVisible() + logger.Infof("will call file handler %s with ipfs file hash %q", handlerFullName, file) + err := inst.CallHandler( + ctx, + wasm.CallParams[CtxData]{ + ExportFuncName: tpl.Mapping.Handler, + Logger: top.Logger, + Data: CtxData{ + dataSource: tpl.NewDataSource("", manifest.BuildBigIntFromUint(tk.GetBlockNumber()), ctxStr), + task: tk, + checkpointCtrl: top.Data.checkpointCtrl, + }, + }, + &wasm.ByteArray{Data: cnt}) + if err != nil { + logger.Errorfe(err, "call file handler %s failed", handlerFullName) + panic(err) + } + return + } + + // contract template, save it + newTpl := controller.TemplateInstance{ + TemplateID: int32(templateID), + Address: params.Data[0].String(), + TemplateName: ctxStr, + BlockRange: controller.BlockRange{StartBlock: tk.GetBlockNumber()}, + } + extErr := top.Data.checkpointCtrl.NewTemplateInstance(ctx, tk, []controller.TemplateInstance{newTpl}) + if extErr != nil { + panic(extErr) + } +} + +func (inst *instance) Reset(ctx context.Context) error { + _, logger := log.FromContext(ctx) + for _, mod := range inst.mods { + if err := mod.Reset(logger); err != nil { + return err + } + } + return nil +} + +const ( + callTimeUsedWarningLimit = time.Second + callHandlerMaxDeep = 10 +) + +func (inst *instance) CallHandler( + ctx *wasm.CallContext[CtxData], + params wasm.CallParams[CtxData], + args ...any, +) (extErr *controller.ExternalError) { + tk, ds := params.Data.task, params.Data.dataSource + hash := ds.Mapping.File.GetIpfsHash() + mod, has := inst.mods[hash] + if !has { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("mod not found for data source %q with ipfs hash %q", ds.Name, hash)) + } + + if deep := ctx.CallStackDeep(); deep >= callHandlerMaxDeep { + return controller.NewExternalError(controller.ErrCodeWasmStackOverFlow, + errors.Errorf("call stack deep %d over flow: %s => %s::%s", + deep, ctx.DumpCallStack(), mod.Name(), params.ExportFuncName)) + } + + // Actually call the event handler. + _, report, err := mod.CallExportFunction(ctx, params, args...) + + // process call error + var errCallingImportFunc *wasm.ErrCallingImportFunc + if err != nil && errors.As(err, &errCallingImportFunc) { + if !errors.As(errCallingImportFunc.Err, &extErr) { + extErr = controller.NewExternalError(controller.ErrCodeCallWasmExportFunctionFailed, errCallingImportFunc.Err) + } + } else if errors.Is(err, wasm.ErrPrepareCallExportFunc) || errors.Is(err, wasm.ErrPanic) { + extErr = controller.NewExternalError(controller.ErrCodeCallWasmExportFunctionFailed, err) + } else if err != nil { + extErr = controller.NewExternalError(controller.ErrCodeWasmError, err) + } + + // print logs + if err != nil { + tk.errLogger().With("report", report).Errorfe(err, "called handler") + } else if report.TimeUsed > callTimeUsedWarningLimit { + tk.logger.With("report", report).Warnf("called handler") + } else { + tk.logger.With("report", report).Infof("called handler") + } + + // report metrics + controller.N.SubgraphTaskDone(ctx, tk.taskInfoForCall(), err == nil, + report.TimeUsed, report.ImportFuncCallUsed, report.MemoryUsed) + + return extErr +} + +func (inst *instance) Snapshot() any { + return utils.MapMapNoError(inst.mods, func(mod *wasm.Instance[CtxData]) any { + return mod.Snapshot() + }) +} diff --git a/driver/controller/subgraph/task.go b/driver/controller/subgraph/task.go new file mode 100644 index 0000000..a72cc3d --- /dev/null +++ b/driver/controller/subgraph/task.go @@ -0,0 +1,93 @@ +package subgraph + +import ( + "context" + "fmt" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +type task struct { + controller.BlockHeader + taskData + + handlerCtrl *HandlerController + instance *instance + + index controller.TaskIndex + timer *timer.Timer + logger *log.SentioLogger +} + +func (t *task) taskInfo() controller.TaskInfo { + return controller.TaskInfo{ + Processor: t.handlerCtrl.processor, + ChainID: t.handlerCtrl.chainID(), + Handler: t.handlerID.Name, + Category: t.handlerID.Type, + DataSource: t.handlerID.DataSource, + } +} + +func (t *task) GetHandlerID() controller.HandlerID { + return t.handlerID +} + +func (t *task) Init(ctx context.Context, index controller.TaskIndex, progressbar controller.ProgressBar) { + t.handlerCtrl.waiter.NewResource(index.Global) + t.instance = t.handlerCtrl.instance // select one + t.index = index + t.timer = timer.NewTimer() + _, t.logger = log.FromContext(ctx, + "block", controller.GetBlockSummary(t), + "latest", controller.GetBlockSummary(progressbar.LatestBlock), + "index", index, + "handler", t.handlerID.String()) +} + +func (t *task) errLogger() *log.SentioLogger { + return t.logger.With("callHandlerArg", utils.MustJSONMarshal(t.callHandlerParam)) +} + +// taskInfoForCall is like taskInfo but uses the resolved data +// source name (set when the handler is actually invoked). +func (t *task) taskInfoForCall() controller.TaskInfo { + h := t.taskInfo() + h.DataSource = t.dataSource.Name + return h +} + +func (t *task) Summary() string { + return fmt.Sprintf("#%d binding data %d/%d for handler %s in block %s", + t.index.Global, t.index.InBlock, t.index.TotalInBlock, t.handlerID, controller.GetBlockSummary(t)) +} + +func (t *task) Exec(ctx context.Context, checkpointCtrl controller.CheckpointController) *controller.ExternalError { + err := t.handlerCtrl.waiter.Wait(ctx, func(u uint64) bool { + return u < t.index.Global + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("waiting all previous tasks finish failed: %v", err)) + } + extErr := t.instance.CallHandler( + wasm.NewCallContext[CtxData](ctx), + wasm.CallParams[CtxData]{ + ExportFuncName: t.handlerID.Name, + Logger: t.logger, + Data: CtxData{dataSource: t.dataSource, task: t, checkpointCtrl: checkpointCtrl}, + }, + t.callHandlerParam, + ) + if extErr != nil { + return extErr.Wrapf("process %s failed", t.Summary()) + } + t.handlerCtrl.waiter.ResourceReady(t.index.Global) + return nil +} diff --git a/driver/controller/timeseries.go b/driver/controller/timeseries.go new file mode 100644 index 0000000..1a15338 --- /dev/null +++ b/driver/controller/timeseries.go @@ -0,0 +1,50 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/timeseries" +) + +type TimeSeriesController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, + ) (stat map[timeseries.MetaType]map[string]int, err *ExternalError) + + Insert(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) + + Snapshot() any +} + +type EmptyTimeSeriesController struct{} + +func (c EmptyTimeSeriesController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyTimeSeriesController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyTimeSeriesController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[timeseries.MetaType]map[string]int, *ExternalError) { + return nil, nil +} + +func (c EmptyTimeSeriesController) Insert(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) { + if len(data) > 0 { + panic("do not support time series data") + } +} + +func (c EmptyTimeSeriesController) Snapshot() any { + return nil +} diff --git a/driver/controller/webhook.go b/driver/controller/webhook.go new file mode 100644 index 0000000..8670c31 --- /dev/null +++ b/driver/controller/webhook.go @@ -0,0 +1,55 @@ +package controller + +import ( + "context" + "time" +) + +type WebhookController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit(ctx context.Context, blockNumber uint64, blockTime time.Time) (stat map[string]int, err *ExternalError) + Insert(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) + Snapshot() any +} + +type WebhookMessage struct { + Name string + Channel string + BlockTime time.Time + Payload string +} + +func StatisticWebhookMessages(msgs []WebhookMessage, stat map[string]int) { + for _, msg := range msgs { + stat[msg.Name] += 1 + } +} + +type EmptyWebhookController struct{} + +func (c EmptyWebhookController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyWebhookController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyWebhookController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, *ExternalError) { + return nil, nil +} + +func (c EmptyWebhookController) Insert(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) { + if len(messages) > 0 { + panic("do not support webhook data") + } +} + +func (c EmptyWebhookController) Snapshot() any { + return nil +} diff --git a/go.mod b/go.mod index 0d100da..6f45af4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.3 require ( cloud.google.com/go/bigquery v1.67.0 + cloud.google.com/go/pubsub v1.49.0 github.com/BurntSushi/toml v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.42.0 github.com/DmitriyVTitov/size v1.5.0 @@ -151,6 +152,7 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.etcd.io/bbolt v1.3.5 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opencensus.io v0.24.0 // indirect go.uber.org/ratelimit v0.3.1 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect @@ -256,7 +258,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.51.0 // indirect - golang.org/x/sync v0.20.0 // indirect + golang.org/x/sync v0.20.0 golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index 8b7ad0f..1e2da30 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= @@ -14,10 +15,14 @@ cloud.google.com/go/datacatalog v1.26.0 h1:eFgygb3DTufTWWUB8ARk+dSuXz+aefNJXTlkW cloud.google.com/go/datacatalog v1.26.0/go.mod h1:bLN2HLBAwB3kLTFT5ZKLHVPj/weNz6bR0c7nYp0LE14= cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/kms v1.21.2 h1:c/PRUSMNQ8zXrc1sdAUnsenWWaNXN+PzTXfXOcSFdoE= +cloud.google.com/go/kms v1.21.2/go.mod h1:8wkMtHV/9Z8mLXEXr1GK7xPSBdi6knuLXIhqjuWcI6w= cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/pubsub v1.49.0 h1:5054IkbslnrMCgA2MAEPcsN3Ky+AyMpEZcii/DoySPo= +cloud.google.com/go/pubsub v1.49.0/go.mod h1:K1FswTWP+C1tI/nfi3HQecoVeFvL4HUOB1tdaNXKhUY= cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= @@ -104,12 +109,15 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= @@ -184,9 +192,13 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= @@ -257,10 +269,20 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= @@ -270,7 +292,13 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -282,6 +310,7 @@ github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9 github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= @@ -530,6 +559,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= @@ -682,11 +712,15 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.einride.tech/aip v0.68.1 h1:16/AfSxcQISGN5z9C5lM+0mLYXihrHbQ1onvYTr93aQ= +go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= @@ -756,8 +790,12 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -765,12 +803,16 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -779,9 +821,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -791,6 +835,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -831,7 +876,10 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -849,14 +897,33 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.233.0 h1:iGZfjXAJiUFSSaekVB7LzXl6tRfEKhUN7FkZN++07tI= google.golang.org/api v0.233.0/go.mod h1:TCIVLLlcwunlMpZIhIp7Ltk77W+vUSdUKAAIlbxY44c= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -895,6 +962,8 @@ gorm.io/gorm v1.31.0 h1:0VlycGreVhK7RF/Bwt51Fk8v0xLiiiFdbGDPIZQ7mJY= gorm.io/gorm v1.31.0/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= From 28a9d95ac754649026e0f355ce20c8f38fe5828d Mon Sep 17 00:00:00 2001 From: jupiterv2 Date: Thu, 18 Jun 2026 17:32:22 +0800 Subject: [PATCH 5/5] refactor(driver): rename controller config ConfigV2 to ChainConfig Now that the streaming controller lives in sentio-core, the "V2" suffix (which only disambiguated it from the legacy sentio chain.Config) is no longer meaningful here: - ConfigV2 -> ChainConfig - LoadChainsConfigV2 -> LoadChainsConfig - NewCustomizedChainConfigV2 -> NewCustomizedChainConfig Also drop the `chain` import alias on driver/controller/config across all consumers and use the package's own name (config.ChainConfig etc.). Co-Authored-By: Claude Opus 4.8 (1M context) --- driver/controller/config/config.go | 16 ++++++++-------- driver/controller/standard/aptos/handler.go | 4 ++-- driver/controller/standard/evm/handler.go | 4 ++-- driver/controller/standard/fuel/handler.go | 4 ++-- driver/controller/standard/handler.go | 6 +++--- driver/controller/standard/sol/handler.go | 4 ++-- driver/controller/standard/sui/agents.go | 4 ++-- driver/controller/standard/sui/grpc/handler.go | 4 ++-- driver/controller/standard/sui/handler.go | 4 ++-- driver/controller/startup/startup.go | 8 ++++---- driver/controller/startup/subgraph.go | 6 +++--- driver/controller/subgraph/handler.go | 6 +++--- 12 files changed, 35 insertions(+), 35 deletions(-) diff --git a/driver/controller/config/config.go b/driver/controller/config/config.go index b209804..5fc5492 100644 --- a/driver/controller/config/config.go +++ b/driver/controller/config/config.go @@ -11,10 +11,10 @@ import ( "sentioxyz/sentio-core/service/processor/models" ) -// ConfigV2 is the per-chain configuration consumed by the streaming +// ChainConfig is the per-chain configuration consumed by the streaming // (driver v3/v4) controller. The legacy driver v2 configuration (chain.Config) // stays in the sentio repository. -type ConfigV2 struct { +type ChainConfig struct { ChainID string Endpoint string StartBlockOverride int64 @@ -28,11 +28,11 @@ type ConfigV2 struct { // applied on top of the chains config file before it is parsed. const PatchChainsConfigEnv = "CHAIN_CONFIG_JSON_PATCH" -func LoadChainsConfigV2( +func LoadChainsConfig( path string, patchEnv string, networkOverrides []models.NetworkOverride, -) (map[string]*ConfigV2, error) { +) (map[string]*ChainConfig, error) { var file []byte var err error if file, err = os.ReadFile(path); err != nil { @@ -46,19 +46,19 @@ func LoadChainsConfigV2( return nil, fmt.Errorf("patch chain config from env %s failed: %w", patchEnv, err) } } - var chainsConfig map[string]*ConfigV2 + var chainsConfig map[string]*ChainConfig if err = json.Unmarshal(file, &chainsConfig); err != nil { return nil, err } for _, no := range networkOverrides { - chainsConfig[no.Chain] = &ConfigV2{ChainID: no.Chain, Endpoint: no.Host, IsCustomizedEndpoint: true} + chainsConfig[no.Chain] = &ChainConfig{ChainID: no.Chain, Endpoint: no.Host, IsCustomizedEndpoint: true} log.Infof("will use customized host %q in chain %s", no.Host, no.Chain) } return chainsConfig, nil } -func NewCustomizedChainConfigV2(chainID, endpoint string) *ConfigV2 { - return &ConfigV2{ +func NewCustomizedChainConfig(chainID, endpoint string) *ChainConfig { + return &ChainConfig{ ChainID: chainID, Endpoint: endpoint, } diff --git a/driver/controller/standard/aptos/handler.go b/driver/controller/standard/aptos/handler.go index 708bf7d..4ab9818 100644 --- a/driver/controller/standard/aptos/handler.go +++ b/driver/controller/standard/aptos/handler.go @@ -14,7 +14,7 @@ import ( "sentioxyz/sentio-core/common/set" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/aptos" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -34,7 +34,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client aptos.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/standard/evm/handler.go b/driver/controller/standard/evm/handler.go index a357d50..16aa0bc 100644 --- a/driver/controller/standard/evm/handler.go +++ b/driver/controller/standard/evm/handler.go @@ -9,7 +9,7 @@ import ( "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/evm" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -33,7 +33,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client evm.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/standard/fuel/handler.go b/driver/controller/standard/fuel/handler.go index d4faee7..1454157 100644 --- a/driver/controller/standard/fuel/handler.go +++ b/driver/controller/standard/fuel/handler.go @@ -12,7 +12,7 @@ import ( chainFuel "sentioxyz/sentio-core/chain/fuel" "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/fuel" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -32,7 +32,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client fuel.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/standard/handler.go b/driver/controller/standard/handler.go index ef4987a..8fe65cc 100644 --- a/driver/controller/standard/handler.go +++ b/driver/controller/standard/handler.go @@ -12,7 +12,7 @@ import ( "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/timeseries" "sentioxyz/sentio-core/processor/protos" @@ -51,7 +51,7 @@ func (c HandlerConfig) String() string { type BaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]] struct { Processor *models.Processor InitResult *protos.InitResponse - ChainConfig *chain.ConfigV2 + ChainConfig *config.ChainConfig Client CLI Config HandlerConfig // result of SetTemplates @@ -71,7 +71,7 @@ type BaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA func NewBaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]]( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client CLI, processorClients []protos.ProcessorV3Client, ) *BaseHandlerController[CLI, BKD, HA] { diff --git a/driver/controller/standard/sol/handler.go b/driver/controller/standard/sol/handler.go index 46dba1c..bf78f13 100644 --- a/driver/controller/standard/sol/handler.go +++ b/driver/controller/standard/sol/handler.go @@ -11,7 +11,7 @@ import ( "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/sol" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -27,7 +27,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client sol.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/standard/sui/agents.go b/driver/controller/standard/sui/agents.go index de5436c..fb234a7 100644 --- a/driver/controller/standard/sui/agents.go +++ b/driver/controller/standard/sui/agents.go @@ -10,7 +10,7 @@ import ( "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data/sui" "sentioxyz/sentio-core/driver/controller/standard" "sentioxyz/sentio-core/processor" @@ -28,7 +28,7 @@ import ( func BuildSuiAgents( ctx context.Context, config standard.HandlerConfig, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, sdkVersion string, client sui.Client, first uint64, diff --git a/driver/controller/standard/sui/grpc/handler.go b/driver/controller/standard/sui/grpc/handler.go index 88d316c..2ba1d54 100644 --- a/driver/controller/standard/sui/grpc/handler.go +++ b/driver/controller/standard/sui/grpc/handler.go @@ -8,7 +8,7 @@ import ( "sentioxyz/sentio-core/common/errgroup" "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" suidata "sentioxyz/sentio-core/driver/controller/data/sui" suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" @@ -34,7 +34,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client suidata.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/standard/sui/handler.go b/driver/controller/standard/sui/handler.go index 30a005c..e82c603 100644 --- a/driver/controller/standard/sui/handler.go +++ b/driver/controller/standard/sui/handler.go @@ -8,7 +8,7 @@ import ( "sentioxyz/sentio-core/common/errgroup" "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/sui" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -32,7 +32,7 @@ type HandlerController struct { func NewHandlerController( processor *models.Processor, initResult *protos.InitResponse, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client sui.Client, processorClients []protos.ProcessorV3Client, ) *HandlerController { diff --git a/driver/controller/startup/startup.go b/driver/controller/startup/startup.go index afd292d..f0b721e 100644 --- a/driver/controller/startup/startup.go +++ b/driver/controller/startup/startup.go @@ -24,7 +24,7 @@ import ( "sentioxyz/sentio-core/common/tracker" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" entitychs "sentioxyz/sentio-core/driver/entity/clickhouse" "sentioxyz/sentio-core/driver/entity/persistent" "sentioxyz/sentio-core/driver/entity/schema" @@ -50,7 +50,7 @@ type baseStartupController struct { registryClient protosregistry.DatabaseRegistryServiceClient processor *models.Processor - chainConfigs map[string]*chain.ConfigV2 + chainConfigs map[string]*config.ChainConfig pubSubTopic *pubsub.Topic @@ -131,8 +131,8 @@ func (c *baseStartupController) connectToDBRegistryService(ctx context.Context) func (c *baseStartupController) loadChainsConfig(ctx context.Context) (err error) { _, logger := log.FromContext(ctx) - c.chainConfigs, err = chain.LoadChainsConfigV2( - c.config.ChainConfigFile, chain.PatchChainsConfigEnv, c.processor.NetworkOverrides) + c.chainConfigs, err = config.LoadChainsConfig( + c.config.ChainConfigFile, config.PatchChainsConfigEnv, c.processor.NetworkOverrides) if err == nil { logger.Info("loaded chain config") } diff --git a/driver/controller/startup/subgraph.go b/driver/controller/startup/subgraph.go index 3f2e8ae..b4e8d9c 100644 --- a/driver/controller/startup/subgraph.go +++ b/driver/controller/startup/subgraph.go @@ -7,7 +7,7 @@ import ( evmchain "sentioxyz/sentio-core/chain/evm" "sentioxyz/sentio-core/common/chains" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data/evm" "sentioxyz/sentio-core/driver/controller/subgraph" "sentioxyz/sentio-core/driver/exitcode" @@ -43,9 +43,9 @@ func (c *subgraphStartupController) buildMainControllers(ctx context.Context) ( // chainID and chainConfig and client chainID, endpoint, _ := manifest.GetChainID(mf.GetNetwork(), false) var client evm.Client - var chainConfig *chain.ConfigV2 + var chainConfig *config.ChainConfig if chainID == manifest.CustomizedChainID { - chainConfig = chain.NewCustomizedChainConfigV2(manifest.CustomizedChainID, endpoint) + chainConfig = config.NewCustomizedChainConfig(manifest.CustomizedChainID, endpoint) } else if chains.IsEVMChains(chainID) { var has bool chainConfig, has = c.chainConfigs[chainID] diff --git a/driver/controller/subgraph/handler.go b/driver/controller/subgraph/handler.go index e09f856..efe19e8 100644 --- a/driver/controller/subgraph/handler.go +++ b/driver/controller/subgraph/handler.go @@ -12,7 +12,7 @@ import ( "sentioxyz/sentio-core/common/log" "sentioxyz/sentio-core/common/utils" "sentioxyz/sentio-core/driver/controller" - chain "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/config" "sentioxyz/sentio-core/driver/controller/data" "sentioxyz/sentio-core/driver/controller/data/evm" "sentioxyz/sentio-core/driver/controller/fetcher" @@ -38,7 +38,7 @@ const ( type HandlerController struct { processor *models.Processor - chainConfig *chain.ConfigV2 + chainConfig *config.ChainConfig client evm.Client ipfsShell *shell.Shell manifest *manifest.Manifest @@ -58,7 +58,7 @@ type HandlerController struct { func NewHandlerController( ctx context.Context, processor *models.Processor, - chainConfig *chain.ConfigV2, + chainConfig *config.ChainConfig, client evm.Client, ipfsShell *shell.Shell, manifest *manifest.Manifest,