diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5cebd15..46e4b94 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -2,10 +2,10 @@ name: Build and Push Docker Image on: push: - branches: [ main ] - tags: [ 'v*' ] + branches: [main] + tags: ["v*"] pull_request: - branches: [ main ] + branches: [main] env: REGISTRY: docker.io @@ -40,16 +40,16 @@ jobs: id: meta run: | TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.sha.outputs.short_sha }}" - + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" fi - + if [[ "${{ github.ref }}" == refs/tags/* ]]; then VERSION=${GITHUB_REF#refs/tags/} TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$VERSION" fi - + echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: Build and push Docker image @@ -59,4 +59,5 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + cache-to: type=gha,mode=max + diff --git a/api/api.go b/api/api.go index bd39e1f..90f9294 100644 --- a/api/api.go +++ b/api/api.go @@ -7,8 +7,8 @@ import ( "os" "runtime/debug" - "github.com/Arkiv-Network/query-api/query" "github.com/Arkiv-Network/query-api/sqlstore" + "github.com/Arkiv-Network/sqlite-store/query" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -49,7 +49,7 @@ func (api *arkivAPI) Query( // TODO log query plan - response, err := api.store.QueryEntities(ctx, req, op, "postgresql") + response, err := api.store.QueryEntities(ctx, req, op) if err != nil { api.log.Warn("error executing query RPC", "error", err) diff --git a/flake.lock b/flake.lock index 0361d7c..522a084 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1764517877, - "narHash": "sha256-xLPjeWHfxEJtZmosOmLaT25Vb2rbktbbE7ShRtAm8h0=", - "rev": "2d293cbfa5a793b4c50d17c05ef9e385b90edf6c", + "lastModified": 1767379071, + "narHash": "sha256-3xDI4xtzovwqE/eAxCwmXxUqBg6Yoam2L1u0IwRNhW4=", + "rev": "fb7944c166a3b630f177938e478f0378e64ce108", "type": "tarball", - "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre904649.2d293cbfa5a7/nixexprs.tar.xz?lastModified=1764517877&rev=2d293cbfa5a793b4c50d17c05ef9e385b90edf6c" + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre921484.fb7944c166a3/nixexprs.tar.xz?lastModified=1767379071&rev=fb7944c166a3b630f177938e478f0378e64ce108" }, "original": { "type": "tarball", diff --git a/go.mod b/go.mod index 7e6d3f5..0607fee 100644 --- a/go.mod +++ b/go.mod @@ -3,22 +3,21 @@ module github.com/Arkiv-Network/query-api go 1.25.4 require ( - github.com/alecthomas/participle/v2 v2.1.4 + github.com/Arkiv-Network/sqlite-store v0.0.22 github.com/ethereum/go-ethereum v1.16.7 github.com/jackc/pgx/v5 v5.7.6 github.com/prometheus/client_golang v1.23.2 - github.com/stretchr/testify v1.11.1 github.com/urfave/cli/v2 v2.27.7 - golang.org/x/sync v0.16.0 + golang.org/x/sync v0.18.0 ) require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/StackExchange/wmi v1.2.1 // indirect + github.com/alecthomas/participle/v2 v2.1.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect @@ -27,7 +26,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -37,9 +35,8 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect google.golang.org/protobuf v1.36.8 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 15b98bd..7d37b90 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Arkiv-Network/sqlite-store v0.0.22 h1:3//zRpx6bakkRE4k1Z/ZzSoPAMGZZPeNgfjBnenkP+U= +github.com/Arkiv-Network/sqlite-store v0.0.22/go.mod h1:E2qiyqwX5IJlsPTmlTCy4dXCwDjFdiOhRxRAasw19SY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= @@ -25,8 +27,8 @@ github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= @@ -66,8 +68,9 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -101,18 +104,18 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/query/cursor.go b/query/cursor.go deleted file mode 100644 index 5290084..0000000 --- a/query/cursor.go +++ /dev/null @@ -1,131 +0,0 @@ -package query - -import ( - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "slices" -) - -func (opts *QueryOptions) EncodeCursor(cursor *Cursor) (string, error) { - bs, err := json.Marshal(cursor) - if err != nil { - return "", fmt.Errorf("error marshalling cursor: %w", err) - } - opts.Log.Info("encode cursor", "cursor", string(bs)) - encodedCursor := make([]any, 0, len(cursor.ColumnValues)*3+1) - - encodedCursor = append(encodedCursor, cursor.BlockNumber) - - for _, c := range cursor.ColumnValues { - columnIx, err := opts.GetColumnIndex(c.ColumnName) - if err != nil { - return "", fmt.Errorf("could not find column index: %w", err) - } - descending := uint64(0) - if c.Descending { - descending = 1 - } - encodedCursor = append(encodedCursor, - uint64(columnIx), c.Value, descending, - ) - } - - s, err := json.Marshal(encodedCursor) - if err != nil { - return "", fmt.Errorf("could not marshal cursor: %w", err) - } - opts.Log.Info("Encoded cursor", "cursor", string(s)) - - hexCursor := hex.EncodeToString([]byte(s)) - opts.Log.Info("Hex encoded cursor", "cursor", hexCursor) - - return hexCursor, nil -} - -func (opts *QueryOptions) DecodeCursor(cursorStr string) (*Cursor, error) { - if len(cursorStr) == 0 { - return nil, nil - } - - bs, err := hex.DecodeString(cursorStr) - if err != nil { - return nil, fmt.Errorf("could not decode cursor: %w", err) - } - - cursor := Cursor{} - - encoded := make([]any, 0) - err = json.Unmarshal(bs, &encoded) - if err != nil { - return nil, fmt.Errorf("could not unmarshal cursor: %w (%s)", err, string(bs)) - } - - firstValue, ok := encoded[0].(float64) - if !ok { - return nil, fmt.Errorf("invalid block number: %d", encoded[0]) - } - blockNumber := uint64(firstValue) - cursor.BlockNumber = blockNumber - - cursor.ColumnValues = make([]CursorValue, 0, len(encoded)-1) - - for c := range slices.Chunk(encoded[1:], 3) { - if len(c) != 3 { - return nil, fmt.Errorf("invalid length of cursor array: %d", len(c)) - } - - firstValue, ok := c[0].(float64) - if !ok { - return nil, fmt.Errorf("unknown column index: %d", c[0]) - } - thirdValue, ok := c[2].(float64) - if !ok { - return nil, fmt.Errorf("unknown value for descending: %d", c[3]) - } - - columnIx := int(firstValue) - if columnIx >= len(opts.Columns) { - return nil, fmt.Errorf("unknown column index: %d", columnIx) - } - - descendingInt := int(thirdValue) - descending := false - switch descendingInt { - case 0: - descending = false - case 1: - descending = true - default: - return nil, fmt.Errorf("unknown value for descending: %d", descendingInt) - } - - value := c[1] - if opts.Columns[columnIx].IsBytes { - encoded, ok := value.(string) - if !ok { - return nil, fmt.Errorf("failed to decode cursor, byte column is not a string") - } - decoded, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - return nil, fmt.Errorf("failed to decode cursor: %w", err) - } - value = decoded - } - - cursor.ColumnValues = append(cursor.ColumnValues, CursorValue{ - ColumnName: opts.Columns[columnIx].Name, - Value: value, - Descending: descending, - }) - } - - jsonCursor, err := json.Marshal(cursor) - if err != nil { - return nil, err - } - opts.Log.Info("Decoded cursor", "cursor", string(jsonCursor)) - - return &cursor, nil -} diff --git a/query/eval_test.go b/query/eval_test.go deleted file mode 100644 index 609a181..0000000 --- a/query/eval_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package query - -import ( - "fmt" - "log/slog" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/require" -) - -var queryOptions = &QueryOptions{} -var log *slog.Logger = slog.Default() - -func TestEqualExpr(t *testing.T) { - expr, err := Parse("name = \"test\"", log) - require.NoError(t, err) - - res, err := expr.Evaluate(queryOptions) - require.NoError(t, err) - - block := uint64(0) - - require.ElementsMatch(t, - []any{ - "name", - "test", - block, block, - }, - res.Args, - ) - - // Query for a key with special characters - expr, err = Parse("déçevant = \"non\"", log) - require.NoError(t, err) - - res, err = expr.Evaluate(queryOptions) - require.NoError(t, err) - - require.ElementsMatch(t, - []any{ - "déçevant", - "non", - block, block, - }, - res.Args, - ) - - expr, err = Parse("بروح = \"ايوة\"", log) - require.NoError(t, err) - - res, err = expr.Evaluate(queryOptions) - require.NoError(t, err) - - require.ElementsMatch(t, - []any{ - "بروح", - "ايوة", - block, block, - }, - res.Args, - ) - - // But symbols should fail - _, err = Parse("foo@ = \"bar\"", log) - require.Error(t, err) -} - -func TestNumericEqualExpr(t *testing.T) { - expr, err := Parse("age = 123", log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestAndExpr(t *testing.T) { - expr, err := Parse(`age = 123 && name = "abc"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestOrExpr(t *testing.T) { - expr, err := Parse(`age = 123 || name = "abc"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestParenthesesExpr(t *testing.T) { - expr, err := Parse(`(name = 123 || name2 = "abc") && name3 = "def" || (name4 = 456 && name5 = 567)`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestOwner(t *testing.T) { - owner := common.HexToAddress("0x1") - - expr, err := Parse(fmt.Sprintf(`(age = 123 || name = "abc") && $owner = %s`, owner), log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestGlob(t *testing.T) { - expr, err := Parse(`age ~ "abc"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestNegation(t *testing.T) { - expr, err := Parse( - `!(name < 123 || !(name2 = "abc" && name2 != "bcd")) && !(name3 = "def") || name4 = 456`, - log, - ) - - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestAndExpr_MultipleTerms(t *testing.T) { - expr, err := Parse(`a = 1 && b = "x" && c = 2 && d = "y"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestOrExpr_MultipleTerms(t *testing.T) { - expr, err := Parse(`a = 1 || b = "x" || c = 2 || d = "y"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestMixedAndOr_NoParens(t *testing.T) { - expr, err := Parse(`a = 1 && b = "x" || c = 2 && d = "y"`, log) - require.NoError(t, err) - - expr.Evaluate(queryOptions) -} - -func TestSorting(t *testing.T) { - expr, err := Parse(`a = 1`, log) - require.NoError(t, err) - - _, err = expr.Evaluate(&QueryOptions{ - OrderByAnnotations: []OrderByAnnotation{ - { - Name: "foo", - Type: "string", - }, - { - Name: "bar", - Type: "numeric", - }, - }, - }) - require.NoError(t, err) -} diff --git a/query/join_method.go b/query/join_method.go deleted file mode 100644 index 7f2089e..0000000 --- a/query/join_method.go +++ /dev/null @@ -1,548 +0,0 @@ -package query - -import ( - "fmt" - "strings" -) - -type AttrJoin struct { - Table string - Alias string -} - -func (t *TopLevel) Evaluate2(options *QueryOptions, sqlDialect string) (*SelectQuery, error) { - - switch sqlDialect { - case "sqlite", "postgresql": - // Pass - default: - return nil, fmt.Errorf("unrecognised SQL dialect: %s", sqlDialect) - } - - qb := strings.Builder{} - args := []any{} - - b := QueryBuilder{ - options: *options, - queryBuilder: &qb, - args: args, - needsComma: false, - needsWhere: true, - sqlDialect: sqlDialect, - } - - b.queryBuilder.WriteString(strings.Join( - []string{ - "SELECT", - b.options.columnString(), - "FROM payloads AS e", - }, - " ", - )) - - if t.Expression != nil { - attrJoins := make(map[string]AttrJoin) - t.Expression.addJoins(attrJoins) - - for attrName, join := range attrJoins { - attrPlaceholder := b.pushArgument(attrName) - fmt.Fprintf( - b.queryBuilder, - " INNER JOIN %[1]s AS %[2]s ON e.entity_key = %[2]s.entity_key AND e.from_block = %[2]s.from_block AND %[2]s.key = %[3]s", - join.Table, - join.Alias, - attrPlaceholder, - ) - } - } - - if b.options.IncludeData != nil { - if b.options.IncludeData.Owner { - fmt.Fprintf(b.queryBuilder, - " LEFT JOIN string_attributes AS ownerAttrs"+ - " ON e.entity_key = ownerAttrs.entity_key"+ - " AND e.from_block = ownerAttrs.from_block"+ - " AND ownerAttrs.key = '%s'", - OwnerAttributeKey, - ) - } - if b.options.IncludeData.Expiration { - fmt.Fprintf(b.queryBuilder, - " LEFT JOIN numeric_attributes AS expirationAttrs"+ - " ON e.entity_key = expirationAttrs.entity_key"+ - " AND e.from_block = expirationAttrs.from_block"+ - " AND expirationAttrs.key = '%s'", - ExpirationAttributeKey, - ) - } - if b.options.IncludeData.CreatedAtBlock { - fmt.Fprintf(b.queryBuilder, - " LEFT JOIN numeric_attributes AS createdAtBlockAttrs"+ - " ON e.entity_key = createdAtBlockAttrs.entity_key"+ - " AND e.from_block = createdAtBlockAttrs.from_block"+ - " AND createdAtBlockAttrs.key = '%s'", - CreatedAtBlockKey, - ) - } - if b.options.IncludeData.LastModifiedAtBlock || - options.IncludeData.TransactionIndexInBlock || - options.IncludeData.OperationIndexInTransaction { - fmt.Fprintf(b.queryBuilder, - " LEFT JOIN numeric_attributes AS sequenceAttrs"+ - " ON e.entity_key = sequenceAttrs.entity_key"+ - " AND e.from_block = sequenceAttrs.from_block"+ - " AND sequenceAttrs.key = '%s'", - SequenceAttributeKey, - ) - } - } - - for i, orderBy := range b.options.OrderByAnnotations { - tableName := "" - switch orderBy.Type { - case "string": - tableName = "string_attributes" - case "numeric": - tableName = "numeric_attributes" - default: - return nil, fmt.Errorf("a type of either 'string' or 'numeric' needs to be provided for the annotation '%s'", orderBy.Name) - } - - sortingTable := fmt.Sprintf("arkiv_annotation_sorting%d", i) - - keyPlaceholder := b.pushArgument(orderBy.Name) - - fmt.Fprintf(b.queryBuilder, - " LEFT JOIN %[1]s AS %s"+ - " ON %[2]s.entity_key = e.entity_key"+ - " AND %[2]s.from_block = e.from_block"+ - " AND %[2]s.key = %[3]s", - - tableName, - sortingTable, - keyPlaceholder, - ) - } - - err := b.addPaginationArguments() - if err != nil { - return nil, fmt.Errorf("error adding the pagination condition: %w", err) - } - - if b.needsWhere { - b.queryBuilder.WriteString(" WHERE ") - b.needsWhere = false - } else { - b.queryBuilder.WriteString(" AND ") - } - - blockArg := b.pushArgument(b.options.AtBlock) - fmt.Fprintf(b.queryBuilder, "%s BETWEEN e.from_block AND e.to_block - 1", blockArg) - - if t.Expression != nil { - if b.needsWhere { - b.queryBuilder.WriteString(" WHERE ") - b.needsWhere = false - } else { - b.queryBuilder.WriteString(" AND ") - } - - t.Expression.pushWhereConditions(&b) - } - - b.queryBuilder.WriteString(" ORDER BY ") - - orderColumns := make([]string, 0, len(b.options.OrderBy)) - for _, o := range b.options.OrderBy { - suffix := "" - if o.Descending { - suffix = " DESC" - } - orderColumns = append(orderColumns, o.Column.Name+suffix) - } - b.queryBuilder.WriteString(strings.Join(orderColumns, ", ")) - - fmt.Fprintf(b.queryBuilder, " LIMIT %d", QueryResultCountLimit) - - return &SelectQuery{ - Query: b.queryBuilder.String(), - Args: b.args, - }, nil -} - -func (e *Expression) addJoins(j map[string]AttrJoin) { - e.Or.addJoins(j) -} - -func (e *Expression) pushWhereConditions(b *QueryBuilder) { - b.queryBuilder.WriteString("(") - e.Or.pushWhereConditions(b) - b.queryBuilder.WriteString(")") -} - -func (e *OrExpression) addJoins(j map[string]AttrJoin) { - e.Left.addJoins(j) - for _, r := range e.Right { - r.addJoins(j) - } -} - -func (e *OrExpression) pushWhereConditions(b *QueryBuilder) { - e.Left.pushWhereConditions(b) - for _, r := range e.Right { - b.queryBuilder.WriteString(" OR ") - r.pushWhereConditions(b) - } -} - -func (e *OrRHS) addJoins(j map[string]AttrJoin) { - e.Expr.addJoins(j) -} - -func (e *OrRHS) pushWhereConditions(b *QueryBuilder) { - e.Expr.pushWhereConditions(b) -} - -func (e *AndExpression) addJoins(j map[string]AttrJoin) { - e.Left.addJoins(j) - for _, r := range e.Right { - r.addJoins(j) - } -} - -func (e *AndExpression) pushWhereConditions(b *QueryBuilder) { - e.Left.pushWhereConditions(b) - for _, r := range e.Right { - b.queryBuilder.WriteString(" AND ") - r.pushWhereConditions(b) - } -} - -func (e *AndRHS) addJoins(j map[string]AttrJoin) { - e.Expr.addJoins(j) -} - -func (e *AndRHS) pushWhereConditions(b *QueryBuilder) { - e.Expr.pushWhereConditions(b) -} - -func (e *EqualExpr) addJoins(j map[string]AttrJoin) { - if e.Paren != nil { - e.Paren.addJoins(j) - return - } - - if e.LessThan != nil { - e.LessThan.addJoins(j) - return - } - - if e.LessOrEqualThan != nil { - e.LessOrEqualThan.addJoins(j) - return - } - - if e.GreaterThan != nil { - e.GreaterThan.addJoins(j) - return - } - - if e.GreaterOrEqualThan != nil { - e.GreaterOrEqualThan.addJoins(j) - return - } - - if e.Glob != nil { - e.Glob.addJoins(j) - return - } - - if e.Assign != nil { - e.Assign.addJoins(j) - return - } - - if e.Inclusion != nil { - e.Inclusion.addJoins(j) - return - } - - panic("This should not happen!") -} - -func (e *EqualExpr) pushWhereConditions(b *QueryBuilder) { - if e.Paren != nil { - e.Paren.pushWhereConditions(b) - return - } - - if e.LessThan != nil { - e.LessThan.pushWhereConditions(b) - return - } - - if e.LessOrEqualThan != nil { - e.LessOrEqualThan.pushWhereConditions(b) - return - } - - if e.GreaterThan != nil { - e.GreaterThan.pushWhereConditions(b) - return - } - - if e.GreaterOrEqualThan != nil { - e.GreaterOrEqualThan.pushWhereConditions(b) - return - } - - if e.Glob != nil { - e.Glob.pushWhereConditions(b) - return - } - - if e.Assign != nil { - e.Assign.pushWhereConditions(b) - return - } - - if e.Inclusion != nil { - e.Inclusion.pushWhereConditions(b) - return - } - - panic("This should not happen!") -} - -func (e *Paren) addJoins(j map[string]AttrJoin) { - e.Nested.addJoins(j) -} - -func (e *Paren) pushWhereConditions(b *QueryBuilder) { - // We already surround every expr with parenthesis, so no need to add more - e.Nested.pushWhereConditions(b) -} - -func (e *LessThan) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Value.Number != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *LessThan) pushWhereConditions(b *QueryBuilder) { - argName := "" - if e.Value.String != nil { - argName = b.pushArgument(*e.Value.String) - } else { - argName = b.pushArgument(*e.Value.Number) - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value < %s", - attributeTableAlias(e.Var), - argName, - ) -} - -func (e *LessOrEqualThan) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Value.Number != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *LessOrEqualThan) pushWhereConditions(b *QueryBuilder) { - argName := "" - if e.Value.String != nil { - argName = b.pushArgument(*e.Value.String) - } else { - argName = b.pushArgument(*e.Value.Number) - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value <= %s", - attributeTableAlias(e.Var), - argName, - ) -} - -func (e *GreaterThan) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Value.Number != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *GreaterThan) pushWhereConditions(b *QueryBuilder) { - argName := "" - if e.Value.String != nil { - argName = b.pushArgument(*e.Value.String) - } else { - argName = b.pushArgument(*e.Value.Number) - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value > %s", - attributeTableAlias(e.Var), - argName, - ) -} - -func (e *GreaterOrEqualThan) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Value.Number != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *GreaterOrEqualThan) pushWhereConditions(b *QueryBuilder) { - argName := "" - if e.Value.String != nil { - argName = b.pushArgument(*e.Value.String) - } else { - argName = b.pushArgument(*e.Value.Number) - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value >= %s", - attributeTableAlias(e.Var), - argName, - ) -} - -func (e *Glob) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *Glob) pushWhereConditions(b *QueryBuilder) { - argName := b.pushArgument(e.Value) - - op := "" - if b.sqlDialect == "postgresql" { - op = "~" - if e.IsNot { - op = "!~" - } - } else { - op = "GLOB" - if e.IsNot { - op = "NOT GLOB" - } - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value %s %s", - attributeTableAlias(e.Var), - op, - argName, - ) -} - -func (e *Equality) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Value.Number != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *Equality) pushWhereConditions(b *QueryBuilder) { - argName := "" - if e.Value.String != nil { - argName = b.pushArgument(*e.Value.String) - } else { - argName = b.pushArgument(*e.Value.Number) - } - - op := "=" - if e.IsNot { - op = "!=" - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value %s %s", - attributeTableAlias(e.Var), - op, - argName, - ) -} - -func (e *Inclusion) addJoins(j map[string]AttrJoin) { - tableName := "string_attributes" - if e.Values.Numbers != nil { - tableName = "numeric_attributes" - } - - j[e.Var] = AttrJoin{ - Table: tableName, - Alias: attributeTableAlias(e.Var), - } -} - -func (e *Inclusion) pushWhereConditions(b *QueryBuilder) { - - args := []any{} - if e.Values.Strings != nil { - for _, s := range e.Values.Strings { - args = append(args, s) - } - } else { - for _, s := range e.Values.Numbers { - args = append(args, s) - } - } - - argNames := []string{} - for _, arg := range args { - argNames = append(argNames, b.pushArgument(arg)) - } - - op := "IN" - if e.IsNot { - op = "NOT IN" - } - - fmt.Fprintf( - b.queryBuilder, - "%s.value %s (%s)", - attributeTableAlias(e.Var), - op, - strings.Join(argNames, ", "), - ) -} diff --git a/query/language.go b/query/language.go deleted file mode 100644 index c56a77c..0000000 --- a/query/language.go +++ /dev/null @@ -1,563 +0,0 @@ -package query - -import ( - "log/slog" - "strings" - - "github.com/alecthomas/participle/v2" - "github.com/alecthomas/participle/v2/lexer" -) - -const AnnotationIdentRegex string = `[\p{L}_][\p{L}\p{N}_]*` - -// Define the lexer with distinct tokens for each operator and parentheses. -var lex = lexer.MustSimple([]lexer.SimpleRule{ - {Name: "Whitespace", Pattern: `[ \t\n\r]+`}, - {Name: "LParen", Pattern: `\(`}, - {Name: "RParen", Pattern: `\)`}, - {Name: "And", Pattern: `&&`}, - {Name: "Or", Pattern: `\|\|`}, - {Name: "Neq", Pattern: `!=`}, - {Name: "Eq", Pattern: `=`}, - {Name: "Geqt", Pattern: `>=`}, - {Name: "Leqt", Pattern: `<=`}, - {Name: "Gt", Pattern: `>`}, - {Name: "Lt", Pattern: `<`}, - {Name: "NotGlob", Pattern: `!~`}, - {Name: "Glob", Pattern: `~`}, - {Name: "Not", Pattern: `!`}, - {Name: "EntityKey", Pattern: `0x[a-fA-F0-9]{64}`}, - {Name: "Address", Pattern: `0x[a-fA-F0-9]{40}`}, - {Name: "String", Pattern: `"(?:[^"\\]|\\.)*"`}, - {Name: "Number", Pattern: `[0-9]+`}, - {Name: "Ident", Pattern: AnnotationIdentRegex}, - // Meta-annotations, should start with $ - {Name: "Owner", Pattern: `\$owner`}, - {Name: "Creator", Pattern: `\$creator`}, - {Name: "Key", Pattern: `\$key`}, - {Name: "Expiration", Pattern: `\$expiration`}, - {Name: "Sequence", Pattern: `\$sequence`}, - {Name: "All", Pattern: `\$all`}, - {Name: "Star", Pattern: `\*`}, -}) - -type TopLevel struct { - Expression *Expression `parser:"@@ | All | Star"` -} - -func (t *TopLevel) Normalise() *TopLevel { - if t.Expression != nil { - return &TopLevel{ - Expression: t.Expression.Normalise(), - } - } - return t -} - -// Expression is the top-level rule. -type Expression struct { - Or OrExpression `parser:"@@"` -} - -func (e *Expression) Normalise() *Expression { - normalised := e.Or.Normalise() - // Remove unneeded OR+AND nodes that both only contain a single child - // when that child is a parenthesised expression - if len(normalised.Right) == 0 && len(normalised.Left.Right) == 0 && normalised.Left.Left.Paren != nil { - // This has already been normalised by the call above, so any negation has - // been pushed into the leaf expressions and we can safely strip away the - // parentheses - return &normalised.Left.Left.Paren.Nested - } - return &Expression{ - Or: *normalised, - } -} - -func (e *Expression) invert() *Expression { - - newLeft := e.Or.invert() - - if len(newLeft.Right) == 0 { - // By construction, this will always be a Paren - if newLeft.Left.Paren == nil { - panic("This should never happen!") - } - return &newLeft.Left.Paren.Nested - } - - return &Expression{ - Or: OrExpression{ - Left: *newLeft, - }, - } -} - -// OrExpression handles expressions connected with ||. -type OrExpression struct { - Left AndExpression `parser:"@@"` - Right []*OrRHS `parser:"@@*"` -} - -func (e *OrExpression) Normalise() *OrExpression { - var newRight []*OrRHS = nil - - if e.Right != nil { - newRight = make([]*OrRHS, 0, len(e.Right)) - for _, rhs := range e.Right { - newRight = append(newRight, rhs.Normalise()) - } - } - - return &OrExpression{ - Left: *e.Left.Normalise(), - Right: newRight, - } -} - -func (e *OrExpression) invert() *AndExpression { - newLeft := EqualExpr{ - Paren: &Paren{ - IsNot: false, - Nested: Expression{ - Or: *e.Left.invert(), - }, - }, - } - - var newRight []*AndRHS = nil - - if e.Right != nil { - newRight = make([]*AndRHS, 0, len(e.Right)) - for _, rhs := range e.Right { - newRight = append(newRight, rhs.invert()) - } - } - - return &AndExpression{ - Left: newLeft, - Right: newRight, - } -} - -// OrRHS represents the right-hand side of an OR. -type OrRHS struct { - Expr AndExpression `parser:"(Or | 'OR' | 'or') @@"` -} - -func (e *OrRHS) Normalise() *OrRHS { - return &OrRHS{ - Expr: *e.Expr.Normalise(), - } -} - -func (e *OrRHS) invert() *AndRHS { - return &AndRHS{ - Expr: EqualExpr{ - Paren: &Paren{ - IsNot: false, - Nested: Expression{ - Or: *e.Expr.invert(), - }, - }, - }, - } -} - -// AndExpression handles expressions connected with &&. -type AndExpression struct { - Left EqualExpr `parser:"@@"` - Right []*AndRHS `parser:"@@*"` -} - -func (e *AndExpression) Normalise() *AndExpression { - var newRight []*AndRHS = nil - - if e.Right != nil { - newRight = make([]*AndRHS, 0, len(e.Right)) - for _, rhs := range e.Right { - newRight = append(newRight, rhs.Normalise()) - } - } - - return &AndExpression{ - Left: *e.Left.Normalise(), - Right: newRight, - } -} - -func (e *AndExpression) invert() *OrExpression { - newLeft := AndExpression{ - Left: *e.Left.invert(), - } - - var newRight []*OrRHS = nil - - if e.Right != nil { - newRight = make([]*OrRHS, 0, len(e.Right)) - for _, rhs := range e.Right { - newRight = append(newRight, rhs.invert()) - } - } - - return &OrExpression{ - Left: newLeft, - Right: newRight, - } -} - -// AndRHS represents the right-hand side of an AND. -type AndRHS struct { - Expr EqualExpr `parser:"(And | 'AND' | 'and') @@"` -} - -func (e *AndRHS) Normalise() *AndRHS { - return &AndRHS{ - Expr: *e.Expr.Normalise(), - } -} - -func (e *AndRHS) invert() *OrRHS { - return &OrRHS{ - Expr: AndExpression{ - Left: *e.Expr.invert(), - }, - } -} - -// EqualExpr can be either an equality or a parenthesized expression. -type EqualExpr struct { - Paren *Paren `parser:" @@"` - Assign *Equality `parser:"| @@"` - Inclusion *Inclusion `parser:"| @@"` - - LessThan *LessThan `parser:"| @@"` - LessOrEqualThan *LessOrEqualThan `parser:"| @@"` - GreaterThan *GreaterThan `parser:"| @@"` - GreaterOrEqualThan *GreaterOrEqualThan `parser:"| @@"` - Glob *Glob `parser:"| @@"` -} - -func (e *EqualExpr) Normalise() *EqualExpr { - - if e.Paren != nil { - p := e.Paren.Normalise() - - // Remove parentheses that only contain a single nested expression - // (i.e. no OR or AND with multiple children) - if len(p.Nested.Or.Right) == 0 && len(p.Nested.Or.Left.Right) == 0 { - // This expression should already be properly normalised, we don't need to - // call Normalise again here - return &p.Nested.Or.Left.Left - } else { - return &EqualExpr{Paren: p} - } - } - - if e.LessThan != nil { - return &EqualExpr{LessThan: e.LessThan.Normalise()} - } - - if e.LessOrEqualThan != nil { - return &EqualExpr{LessOrEqualThan: e.LessOrEqualThan.Normalise()} - } - - if e.GreaterThan != nil { - return &EqualExpr{GreaterThan: e.GreaterThan.Normalise()} - } - - if e.GreaterOrEqualThan != nil { - return &EqualExpr{GreaterOrEqualThan: e.GreaterOrEqualThan.Normalise()} - } - - if e.Glob != nil { - return &EqualExpr{Glob: e.Glob.Normalise()} - } - - if e.Assign != nil { - return &EqualExpr{Assign: e.Assign.Normalise()} - } - - if e.Inclusion != nil { - return &EqualExpr{Inclusion: e.Inclusion.Normalise()} - } - - panic("This should not happen!") -} - -func (e *EqualExpr) invert() *EqualExpr { - if e.Paren != nil { - return &EqualExpr{Paren: e.Paren.invert()} - } - - if e.LessThan != nil { - return &EqualExpr{GreaterOrEqualThan: e.LessThan.invert()} - } - - if e.LessOrEqualThan != nil { - return &EqualExpr{GreaterThan: e.LessOrEqualThan.invert()} - } - - if e.GreaterThan != nil { - return &EqualExpr{LessOrEqualThan: e.GreaterThan.invert()} - } - - if e.GreaterOrEqualThan != nil { - return &EqualExpr{LessThan: e.GreaterOrEqualThan.invert()} - } - - if e.Glob != nil { - return &EqualExpr{Glob: e.Glob.invert()} - } - - if e.Assign != nil { - return &EqualExpr{Assign: e.Assign.invert()} - } - - if e.Inclusion != nil { - return &EqualExpr{Inclusion: e.Inclusion.invert()} - } - - panic("This should not happen!") -} - -type Paren struct { - IsNot bool `parser:"@(Not | 'NOT' | 'not')?"` - Nested Expression `parser:"LParen @@ RParen"` -} - -func (e *Paren) Normalise() *Paren { - nested := e.Nested - - if e.IsNot { - nested = *nested.invert() - } - - return &Paren{ - IsNot: false, - Nested: *nested.Normalise(), - } -} - -func (e *Paren) invert() *Paren { - return &Paren{ - IsNot: !e.IsNot, - Nested: e.Nested, - } -} - -type Glob struct { - Var string `parser:"@Ident"` - IsNot bool `parser:"((Glob | @NotGlob) | (@('NOT' | 'not')? ('GLOB' | 'glob')))"` - Value string `parser:"@String"` -} - -func (e *Glob) Normalise() *Glob { - // TODO do we need to change casing here too? - return e -} - -func (e *Glob) invert() *Glob { - return &Glob{ - Var: e.Var, - IsNot: !e.IsNot, - Value: e.Value, - } -} - -type LessThan struct { - Var string `parser:"@Ident Lt"` - Value Value `parser:"@@"` -} - -func (e *LessThan) Normalise() *LessThan { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - val := strings.ToLower(*e.Value.String) - return &LessThan{ - Var: e.Var, - Value: Value{ - String: &val, - }, - } - default: - return e - } -} - -func (e *LessThan) invert() *GreaterOrEqualThan { - return &GreaterOrEqualThan{ - Var: e.Var, - Value: e.Value, - } -} - -type LessOrEqualThan struct { - Var string `parser:"@Ident Leqt"` - Value Value `parser:"@@"` -} - -func (e *LessOrEqualThan) Normalise() *LessOrEqualThan { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - val := strings.ToLower(*e.Value.String) - return &LessOrEqualThan{ - Var: e.Var, - Value: Value{ - String: &val, - }, - } - default: - return e - } -} - -func (e *LessOrEqualThan) invert() *GreaterThan { - return &GreaterThan{ - Var: e.Var, - Value: e.Value, - } -} - -type GreaterThan struct { - Var string `parser:"@Ident Gt"` - Value Value `parser:"@@"` -} - -func (e *GreaterThan) Normalise() *GreaterThan { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - val := strings.ToLower(*e.Value.String) - return &GreaterThan{ - Var: e.Var, - Value: Value{ - String: &val, - }, - } - default: - return e - } -} - -func (e *GreaterThan) invert() *LessOrEqualThan { - return &LessOrEqualThan{ - Var: e.Var, - Value: e.Value, - } -} - -type GreaterOrEqualThan struct { - Var string `parser:"@Ident Geqt"` - Value Value `parser:"@@"` -} - -func (e *GreaterOrEqualThan) Normalise() *GreaterOrEqualThan { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - val := strings.ToLower(*e.Value.String) - return &GreaterOrEqualThan{ - Var: e.Var, - Value: Value{ - String: &val, - }, - } - default: - return e - } -} - -func (e *GreaterOrEqualThan) invert() *LessThan { - return &LessThan{ - Var: e.Var, - Value: e.Value, - } -} - -// Equality represents a simple equality (e.g. name = 123). -type Equality struct { - Var string `parser:"@(Ident | Key | Owner | Creator | Expiration | Sequence)"` - IsNot bool `parser:"(Eq | @Neq)"` - Value Value `parser:"@@"` -} - -func (e *Equality) Normalise() *Equality { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - val := strings.ToLower(*e.Value.String) - return &Equality{ - Var: e.Var, - Value: Value{ - String: &val, - }, - } - default: - return e - } -} - -func (e *Equality) invert() *Equality { - return &Equality{ - Var: e.Var, - IsNot: !e.IsNot, - Value: e.Value, - } -} - -type Inclusion struct { - Var string `parser:"@(Ident | Key | Owner | Creator | Expiration | Sequence)"` - IsNot bool `parser:"(@('NOT'|'not')? ('IN'|'in'))"` - Values Values `parser:"@@"` -} - -func (e *Inclusion) Normalise() *Inclusion { - switch e.Var { - case KeyAttributeKey, OwnerAttributeKey, CreatorAttributeKey: - vals := make([]string, 0, len(e.Values.Strings)) - for _, val := range e.Values.Strings { - vals = append(vals, strings.ToLower(val)) - } - return &Inclusion{ - Var: e.Var, - Values: Values{ - Strings: vals, - }, - } - default: - return e - } -} - -func (e *Inclusion) invert() *Inclusion { - return &Inclusion{ - Var: e.Var, - IsNot: !e.IsNot, - Values: e.Values, - } -} - -// Value is a literal value (a number or a string). -type Value struct { - String *string `parser:" (@String | @EntityKey | @Address)"` - Number *uint64 `parser:"| @Number"` -} - -type Values struct { - Strings []string `parser:" '(' (@String | @EntityKey | @Address)+ ')'"` - Numbers []uint64 `parser:"| '(' @Number+ ')'"` -} - -var Parser = participle.MustBuild[TopLevel]( - participle.Lexer(lex), - participle.Elide("Whitespace"), - participle.Unquote("String"), -) - -func Parse(s string, log *slog.Logger) (*TopLevel, error) { - log.Info("parsing query", "query", s) - - v, err := Parser.ParseString("", s) - if err != nil { - return nil, err - } - return v.Normalise(), err -} diff --git a/query/language_test.go b/query/language_test.go deleted file mode 100644 index 05675af..0000000 --- a/query/language_test.go +++ /dev/null @@ -1,734 +0,0 @@ -package query - -import ( - "fmt" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/require" -) - -func pointerOf[T any](v T) *T { - return &v -} - -func TestParse(t *testing.T) { - t.Run("quoted string", func(t *testing.T) { - v, err := Parse(`name = "test\"2"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: false, - Value: Value{ - String: pointerOf("test\"2"), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("empty query", func(t *testing.T) { - _, err := Parse(``, log) - require.Error(t, err) - }) - - t.Run("all", func(t *testing.T) { - v, err := Parse(`$all`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: nil, - }, - v, - ) - }) - - t.Run("number", func(t *testing.T) { - v, err := Parse(`name = 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: false, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("not parentheses", func(t *testing.T) { - v, err := Parse(`!(name = 123 || name = 456)`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: true, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - Right: []*AndRHS{ - { - EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: true, - Value: Value{ - Number: pointerOf(uint64(456)), - }, - }, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("not number", func(t *testing.T) { - v, err := Parse(`!(name = 123)`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: true, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("not equal", func(t *testing.T) { - v, err := Parse(`name != 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: true, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("lessthan", func(t *testing.T) { - v, err := Parse(`name < 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - LessThan: &LessThan{ - Var: "name", - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - - v, err = Parse(`name < "123"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - LessThan: &LessThan{ - Var: "name", - Value: Value{ - String: pointerOf("123"), - }, - }, - }, - }, - }, - }, - }, - v, - ) - - v, err = Parse(`!(name < 123)`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - GreaterOrEqualThan: &GreaterOrEqualThan{ - Var: "name", - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("lessthanequal", func(t *testing.T) { - v, err := Parse(`name <= 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - LessOrEqualThan: &LessOrEqualThan{ - Var: "name", - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - - v, err = Parse(`name <= "123"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - LessOrEqualThan: &LessOrEqualThan{ - Var: "name", - Value: Value{ - String: pointerOf("123"), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("greaterthan", func(t *testing.T) { - v, err := Parse(`name > 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - GreaterThan: &GreaterThan{ - Var: "name", - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - - v, err = Parse(`name > "123"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - GreaterThan: &GreaterThan{ - Var: "name", - Value: Value{ - String: pointerOf("123"), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("greaterthanequal", func(t *testing.T) { - v, err := Parse(`name >= 123`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - GreaterOrEqualThan: &GreaterOrEqualThan{ - Var: "name", - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - }, - }, - }, - v, - ) - - v, err = Parse(`name >= "123"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - GreaterOrEqualThan: &GreaterOrEqualThan{ - Var: "name", - Value: Value{ - String: pointerOf("123"), - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("glob", func(t *testing.T) { - v, err := Parse(`name ~ "foo"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Glob: &Glob{ - Var: "name", - IsNot: false, - Value: "foo", - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("owner", func(t *testing.T) { - owner := common.HexToAddress("0x1").Hex() - v, err := Parse(fmt.Sprintf(`$owner = %s`, owner), log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "$owner", - IsNot: false, - Value: Value{ - String: &owner, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("owner quoted", func(t *testing.T) { - owner := common.HexToAddress("0x1").Hex() - v, err := Parse(fmt.Sprintf(`$owner = "%s"`, owner), log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "$owner", - IsNot: false, - Value: Value{ - String: &owner, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("not owner", func(t *testing.T) { - owner := common.HexToAddress("0x1").Hex() - v, err := Parse(fmt.Sprintf(`$owner != %s`, owner), log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "$owner", - IsNot: true, - Value: Value{ - String: &owner, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("glob", func(t *testing.T) { - v, err := Parse(`name ~ "foo"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Glob: &Glob{ - Var: "name", - IsNot: false, - Value: "foo", - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("not glob", func(t *testing.T) { - v, err := Parse(`name !~ "foo"`, log) - require.NoError(t, err) - - require.Equal( - t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Glob: &Glob{ - Var: "name", - IsNot: true, - Value: "foo", - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("and", func(t *testing.T) { - v, err := Parse(`(name = 123 && name2 = "abc")`, log) - require.NoError(t, err) - - require.Equal(t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: false, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - Right: []*AndRHS{ - { - Expr: EqualExpr{ - Assign: &Equality{ - Var: "name2", - IsNot: false, - Value: Value{ - String: pointerOf("abc"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("or", func(t *testing.T) { - v, err := Parse(`name = 123 || name2 = "abc"`, log) - require.NoError(t, err) - - require.Equal(t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: false, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - Right: []*OrRHS{ - { - Expr: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name2", - IsNot: false, - Value: Value{ - String: pointerOf("abc"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("parentheses", func(t *testing.T) { - v, err := Parse(`(name = 123 || name2 = "abc") && (name3 = "def") || name4 = 456`, log) - require.NoError(t, err) - - require.Equal(t, - &TopLevel{ - Expression: &Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Paren: &Paren{ - Nested: Expression{ - Or: OrExpression{ - Left: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name", - IsNot: false, - Value: Value{ - Number: pointerOf(uint64(123)), - }, - }, - }, - }, - Right: []*OrRHS{ - { - Expr: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name2", - IsNot: false, - Value: Value{ - String: pointerOf("abc"), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - Right: []*AndRHS{ - { - Expr: EqualExpr{ - Assign: &Equality{ - Var: "name3", - IsNot: false, - Value: Value{ - String: pointerOf("def"), - }, - }, - }, - }, - }, - }, - Right: []*OrRHS{ - { - Expr: AndExpression{ - Left: EqualExpr{ - Assign: &Equality{ - Var: "name4", - IsNot: false, - Value: Value{ - Number: pointerOf(uint64(456)), - }, - }, - }, - }, - }, - }, - }, - }, - }, - v, - ) - }) - - t.Run("invalid expression", func(t *testing.T) { - _, err := Parse(`key = 8e`, log) - require.Error(t, err, `1:8: unexpected token "e"`) - }) - - t.Run("invalid expression", func(t *testing.T) { - _, err := Parse(`key = 8e`, log) - require.Error(t, err, `1:8: unexpected token "e"`) - }) - -} diff --git a/query/query.go b/query/query.go deleted file mode 100644 index 8319138..0000000 --- a/query/query.go +++ /dev/null @@ -1,111 +0,0 @@ -package query - -import ( - "fmt" - "hash/fnv" - "strings" -) - -type SelectQuery struct { - Query string - Args []any -} - -type QueryBuilder struct { - queryBuilder *strings.Builder - args []any - argsCount uint32 - tableCounter uint32 - needsComma bool - needsWhere bool - options QueryOptions - sqlDialect string -} - -func attributeTableAlias(name string) string { - h := fnv.New32a() - h.Write([]byte(name)) - - return fmt.Sprintf("arkiv_attr_%d", h.Sum32()) -} - -func (b *QueryBuilder) nextTableName() string { - b.tableCounter = b.tableCounter + 1 - return fmt.Sprintf("table_%d", b.tableCounter) -} - -func (b *QueryBuilder) pushArgument(arg any) string { - b.args = append(b.args, arg) - b.argsCount += 1 - return fmt.Sprintf("$%d", b.argsCount) -} - -func (b *QueryBuilder) writeComma() { - if b.needsComma { - b.queryBuilder.WriteString(", ") - } else { - b.needsComma = true - } -} - -func (b *QueryBuilder) addPaginationArguments() error { - paginationConditions := []string{} - - if len(b.options.Cursor) > 0 { - // Pre-allocate argument counters so that we don't need to duplicate them below - args := make([]string, 0, len(b.options.Cursor)) - for _, val := range b.options.Cursor { - args = append(args, b.pushArgument(val.Value)) - } - - for i := range b.options.Cursor { - subcondition := []string{} - for j, from := range b.options.Cursor { - if j > i { - break - } - var operator string - if j < i { - operator = "=" - } else if from.Descending { - operator = "<" - } else { - operator = ">" - } - - arg := args[j] - - columnIx, err := b.options.GetColumnIndex(from.ColumnName) - if err != nil { - return fmt.Errorf("error getting column index: %w", err) - } - column := b.options.Columns[columnIx] - - subcondition = append( - subcondition, - fmt.Sprintf("%s %s %s", column.QualifiedName, operator, arg), - ) - } - - paginationConditions = append( - paginationConditions, - fmt.Sprintf("(%s)", strings.Join(subcondition, " AND ")), - ) - } - - paginationCondition := strings.Join(paginationConditions, " OR ") - - if b.needsWhere { - b.queryBuilder.WriteString(" WHERE ") - b.needsWhere = false - } else { - b.queryBuilder.WriteString(" AND ") - } - - b.queryBuilder.WriteString("(") - b.queryBuilder.WriteString(paginationCondition) - b.queryBuilder.WriteString(")") - } - - return nil -} diff --git a/query/query_options.go b/query/query_options.go deleted file mode 100644 index 3228681..0000000 --- a/query/query_options.go +++ /dev/null @@ -1,201 +0,0 @@ -package query - -import ( - "cmp" - "fmt" - "log/slog" - "slices" - "strings" -) - -const QueryResultCountLimit uint64 = 200 - -// ResponseSize is 256 bytes for the overhead of the 'envelope' around the entity data -// and the separator characters in between -const ResponseSize int = 256 - -// MaxResponseSize is 512 kb -const MaxResponseSize int = 512 * 1024 * 1024 - -type Column struct { - Name string - QualifiedName string - // If this is a byte column, we need to decode it when we get it from the json-encoded cursor - IsBytes bool -} - -func (c Column) selector() string { - return fmt.Sprintf("%s AS %s", c.QualifiedName, c.Name) -} - -func (c Column) Compare(other Column) int { - return cmp.Compare(c.Name, other.Name) -} - -type OrderBy struct { - Column Column - Descending bool -} - -type QueryOptions struct { - AtBlock uint64 - IncludeData *IncludeData - Columns []Column - OrderBy []OrderBy - OrderByAnnotations []OrderByAnnotation - Cursor []CursorValue - - // Cache the sorted list of unique columns to fetch - allColumnsSorted []string - orderByColumns []OrderBy - - Log *slog.Logger -} - -func NewQueryOptions(log *slog.Logger, latestHead uint64, options *InternalQueryOptions) (*QueryOptions, error) { - queryOptions := QueryOptions{ - Log: log, - OrderByAnnotations: options.OrderBy, - IncludeData: options.IncludeData, - } - - queryOptions.Columns = []Column{} - - // We always need the primary key of the payloads table because of sorting - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "from_block", - QualifiedName: "e.from_block", - }) - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "entity_key", - QualifiedName: "e.entity_key", - IsBytes: true, - }) - - if options.IncludeData.Payload { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "payload", - QualifiedName: "e.payload", - }) - } - if options.IncludeData.ContentType { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "content_type", - QualifiedName: "e.content_type", - }) - } - if options.IncludeData.Attributes { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "string_attributes", - QualifiedName: "e.string_attributes", - }) - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "numeric_attributes", - QualifiedName: "e.numeric_attributes", - }) - } - - for i := range options.OrderBy { - name := fmt.Sprintf("arkiv_annotation_sorting%d_value", i) - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: name, - QualifiedName: fmt.Sprintf("arkiv_annotation_sorting%d.value", i), - }) - } - - if options.IncludeData.Owner { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "owner", - QualifiedName: "ownerAttrs.Value", - }) - } - if options.IncludeData.Expiration { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "expires_at", - QualifiedName: "expirationAttrs.Value", - }) - } - if options.IncludeData.CreatedAtBlock { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "created_at_block", - QualifiedName: "createdAtBlockAttrs.Value", - }) - } - if options.IncludeData.LastModifiedAtBlock || - options.IncludeData.TransactionIndexInBlock || - options.IncludeData.OperationIndexInTransaction { - queryOptions.Columns = append(queryOptions.Columns, Column{ - Name: "sequence", - QualifiedName: "sequenceAttrs.Value", - }) - } - - // Sort so that we can use binary search later - slices.SortFunc(queryOptions.Columns, Column.Compare) - - queryOptions.OrderBy = []OrderBy{} - - for i, o := range queryOptions.OrderByAnnotations { - queryOptions.OrderBy = append(queryOptions.OrderBy, OrderBy{ - Column: Column{ - Name: fmt.Sprintf("arkiv_annotation_sorting%d_value", i), - QualifiedName: fmt.Sprintf("arkiv_annotation_sorting%d.value", i), - }, - Descending: o.Descending, - }) - } - queryOptions.OrderBy = append(queryOptions.OrderBy, OrderBy{ - Column: Column{ - Name: "from_block", - QualifiedName: "e.from_block", - }, - }) - queryOptions.OrderBy = append(queryOptions.OrderBy, OrderBy{ - Column: Column{ - Name: "entity_key", - QualifiedName: "e.entity_key", - IsBytes: true, - }, - }) - - queryOptions.AtBlock = latestHead - - if len(options.Cursor) != 0 { - cursor, err := queryOptions.DecodeCursor(options.Cursor) - if err != nil { - return nil, err - } - queryOptions.AtBlock = cursor.BlockNumber - queryOptions.Cursor = cursor.ColumnValues - } - - if options.AtBlock != nil { - queryOptions.AtBlock = *options.AtBlock - } - - return &queryOptions, nil -} - -func (opts *QueryOptions) GetColumnIndex(column string) (int, error) { - ix, found := slices.BinarySearchFunc(opts.Columns, column, func(a Column, b string) int { - return cmp.Compare(a.Name, b) - }) - - if !found { - return -1, fmt.Errorf("unknown column %s", column) - } - return ix, nil -} - -func (opts *QueryOptions) columnString() string { - if len(opts.Columns) == 0 { - return "1" - } - - columns := make([]string, 0, len(opts.Columns)) - for _, c := range opts.Columns { - columns = append(columns, c.selector()) - } - - return strings.Join(columns, ", ") -} diff --git a/query/tables_method.go b/query/tables_method.go deleted file mode 100644 index da18490..0000000 --- a/query/tables_method.go +++ /dev/null @@ -1,459 +0,0 @@ -package query - -import ( - "fmt" - "strings" -) - -func (b *QueryBuilder) createLeafQuery(query string) string { - tableName := b.nextTableName() - b.writeComma() - b.queryBuilder.WriteString(tableName) - b.queryBuilder.WriteString(" AS (") - b.queryBuilder.WriteString(query) - b.queryBuilder.WriteString(")") - - return tableName -} - -func (t *TopLevel) Evaluate(options *QueryOptions) (*SelectQuery, error) { - tableBuilder := strings.Builder{} - args := []any{} - - builder := QueryBuilder{ - options: *options, - queryBuilder: &tableBuilder, - args: args, - needsComma: false, - needsWhere: true, - } - - if t.Expression != nil { - builder.queryBuilder.WriteString(strings.Join( - []string{ - " SELECT", - builder.options.columnString(), - "FROM", - t.Expression.Evaluate(&builder), - "AS keys INNER JOIN payloads AS e ON keys.entity_key = e.entity_key AND keys.from_block = e.from_block", - }, - " ", - )) - } else { - builder.queryBuilder.WriteString(strings.Join( - []string{ - "SELECT", - builder.options.columnString(), - "FROM payloads AS e", - }, - " ", - )) - } - - if builder.options.IncludeData != nil { - if builder.options.IncludeData.Owner { - fmt.Fprintf(builder.queryBuilder, - " INNER JOIN string_attributes AS ownerAttrs"+ - " ON e.entity_key = ownerAttrs.entity_key"+ - " AND e.from_block = ownerAttrs.from_block"+ - " AND ownerAttrs.key = '%s'", - OwnerAttributeKey, - ) - } - if builder.options.IncludeData.Expiration { - fmt.Fprintf(builder.queryBuilder, - " INNER JOIN numeric_attributes AS expirationAttrs"+ - " ON e.entity_key = expirationAttrs.entity_key"+ - " AND e.from_block = expirationAttrs.from_block"+ - " AND expirationAttrs.key = '%s'", - ExpirationAttributeKey, - ) - } - if builder.options.IncludeData.CreatedAtBlock { - fmt.Fprintf(builder.queryBuilder, - " INNER JOIN numeric_attributes AS createdAtBlockAttrs"+ - " ON e.entity_key = createdAtBlockAttrs.entity_key"+ - " AND e.from_block = createdAtBlockAttrs.from_block"+ - " AND createdAtBlockAttrs.key = '%s'", - CreatedAtBlockKey, - ) - } - if builder.options.IncludeData.LastModifiedAtBlock || - options.IncludeData.TransactionIndexInBlock || - options.IncludeData.OperationIndexInTransaction { - fmt.Fprintf(builder.queryBuilder, - " INNER JOIN numeric_attributes AS sequenceAttrs"+ - " ON e.entity_key = sequenceAttrs.entity_key"+ - " AND e.from_block = sequenceAttrs.from_block"+ - " AND sequenceAttrs.key = '%s'", - SequenceAttributeKey, - ) - } - } - - for i, orderBy := range builder.options.OrderByAnnotations { - tableName := "" - switch orderBy.Type { - case "string": - tableName = "string_attributes" - case "numeric": - tableName = "numeric_attributes" - default: - return nil, fmt.Errorf("a type of either 'string' or 'numeric' needs to be provided for the annotation '%s'", orderBy.Name) - } - - sortingTable := fmt.Sprintf("arkiv_annotation_sorting%d", i) - - keyPlaceholder := builder.pushArgument(orderBy.Name) - - fmt.Fprintf(builder.queryBuilder, - " LEFT JOIN %[1]s AS %s"+ - " ON %[2]s.entity_key = e.entity_key"+ - " AND %[2]s.from_block = e.from_block"+ - " AND %[2]s.key = %[3]s", - - tableName, - sortingTable, - keyPlaceholder, - ) - } - - err := builder.addPaginationArguments() - if err != nil { - return nil, fmt.Errorf("error adding the pagination condition: %w", err) - } - - if builder.needsWhere { - builder.queryBuilder.WriteString(" WHERE ") - builder.needsWhere = false - } else { - builder.queryBuilder.WriteString(" AND ") - } - - blockArg := builder.pushArgument(builder.options.AtBlock) - fmt.Fprintf(builder.queryBuilder, "%s BETWEEN e.from_block AND e.to_block", blockArg) - - builder.queryBuilder.WriteString(" ORDER BY ") - - orderColumns := make([]string, 0, len(builder.options.OrderBy)) - for _, o := range builder.options.OrderBy { - suffix := "" - if o.Descending { - suffix = " DESC" - } - orderColumns = append(orderColumns, o.Column.Name+suffix) - } - builder.queryBuilder.WriteString(strings.Join(orderColumns, ", ")) - - fmt.Fprintf(builder.queryBuilder, " LIMIT %d", QueryResultCountLimit) - - return &SelectQuery{ - Query: builder.queryBuilder.String(), - Args: builder.args, - }, nil -} - -func (e *Expression) Evaluate(builder *QueryBuilder) string { - builder.queryBuilder.WriteString("WITH ") - prevTable := e.Or.Evaluate(builder) - - builder.writeComma() - nextTable := builder.nextTableName() - - builder.queryBuilder.WriteString(nextTable) - builder.queryBuilder.WriteString(" AS (") - builder.queryBuilder.WriteString("SELECT DISTINCT * FROM ") - builder.queryBuilder.WriteString(prevTable) - builder.queryBuilder.WriteString(")") - - return nextTable -} - -func (e *OrExpression) Evaluate(b *QueryBuilder) string { - leftTable := e.Left.Evaluate(b) - tableName := leftTable - - for _, rhs := range e.Right { - rightTable := rhs.Evaluate(b) - tableName = b.nextTableName() - - b.writeComma() - - b.queryBuilder.WriteString(tableName) - b.queryBuilder.WriteString(" AS (") - b.queryBuilder.WriteString("SELECT * FROM ") - b.queryBuilder.WriteString(leftTable) - b.queryBuilder.WriteString(" UNION ") - b.queryBuilder.WriteString("SELECT * FROM ") - b.queryBuilder.WriteString(rightTable) - b.queryBuilder.WriteString(")") - - // Carry forward the cumulative result of the UNION - leftTable = tableName - } - - return tableName -} - -func (e *OrRHS) Evaluate(b *QueryBuilder) string { - return e.Expr.Evaluate(b) -} - -func (e *AndExpression) Evaluate(b *QueryBuilder) string { - leftTable := e.Left.Evaluate(b) - tableName := leftTable - - for _, rhs := range e.Right { - rightTable := rhs.Evaluate(b) - tableName = b.nextTableName() - - b.writeComma() - - b.queryBuilder.WriteString(tableName) - b.queryBuilder.WriteString(" AS (") - b.queryBuilder.WriteString("SELECT * FROM ") - b.queryBuilder.WriteString(leftTable) - b.queryBuilder.WriteString(" INTERSECT ") - b.queryBuilder.WriteString("SELECT * FROM ") - b.queryBuilder.WriteString(rightTable) - b.queryBuilder.WriteString(")") - - // Carry forward the cumulative result of the INTERSECT - leftTable = tableName - } - - return tableName -} - -func (e *AndRHS) Evaluate(b *QueryBuilder) string { - return e.Expr.Evaluate(b) -} - -func (e *EqualExpr) Evaluate(b *QueryBuilder) string { - if e.Paren != nil { - return e.Paren.Evaluate(b) - } - - if e.LessThan != nil { - return e.LessThan.Evaluate(b) - } - - if e.LessOrEqualThan != nil { - return e.LessOrEqualThan.Evaluate(b) - } - - if e.GreaterThan != nil { - return e.GreaterThan.Evaluate(b) - } - - if e.GreaterOrEqualThan != nil { - return e.GreaterOrEqualThan.Evaluate(b) - } - - if e.Glob != nil { - return e.Glob.Evaluate(b) - } - - if e.Assign != nil { - return e.Assign.Evaluate(b) - } - - if e.Inclusion != nil { - return e.Inclusion.Evaluate(b) - } - - panic("This should not happen!") -} - -func (e *Paren) Evaluate(b *QueryBuilder) string { - expr := e.Nested - // If we have a negation, we will push it down into the expression - if e.IsNot { - expr = *e.Nested.invert() - } - // We don't have to do anything here regarding precedence, the parsing order - // is already taking care of precedence since the nested OR node will create a subquery - return expr.Or.Evaluate(b) -} - -func (b *QueryBuilder) createAnnotationQuery( - attributeType string, - whereClause string, -) string { - - tableName := "string_attributes" - if attributeType == "numeric" { - tableName = "numeric_attributes" - } - - blockArg := b.pushArgument(b.options.AtBlock) - - return b.createLeafQuery( - strings.Join( - []string{ - "SELECT e.entity_key, e.from_block FROM", - tableName, - "AS a", - "INNER JOIN payloads AS e", - "ON a.entity_key = e.entity_key", - "AND a.from_block = e.from_block", - fmt.Sprintf("AND %s BETWEEN e.from_block AND e.to_block - 1", blockArg), - "WHERE", - whereClause, - }, - " ", - ), - ) -} - -func (e *Glob) Evaluate(b *QueryBuilder) string { - varArg := b.pushArgument(e.Var) - valArg := b.pushArgument(e.Value) - - op := "~" - if e.IsNot { - op = "!~" - } - - return b.createAnnotationQuery( - "string", - fmt.Sprintf("key = %s AND value %s %s", varArg, op, valArg), - ) -} - -func (e *LessThan) Evaluate(b *QueryBuilder) string { - attrType := "string" - varArg := b.pushArgument(e.Var) - valArg := "" - - if e.Value.String != nil { - valArg = b.pushArgument(*e.Value.String) - } else { - attrType = "numeric" - valArg = b.pushArgument(*e.Value.Number) - } - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("key = %s AND value < %s", varArg, valArg), - ) -} - -func (e *LessOrEqualThan) Evaluate(b *QueryBuilder) string { - attrType := "string" - varArg := b.pushArgument(e.Var) - valArg := "" - - if e.Value.String != nil { - valArg = b.pushArgument(*e.Value.String) - } else { - attrType = "numeric" - valArg = b.pushArgument(*e.Value.Number) - } - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("key = %s AND value <= %s", varArg, valArg), - ) -} - -func (e *GreaterThan) Evaluate(b *QueryBuilder) string { - attrType := "string" - varArg := b.pushArgument(e.Var) - valArg := "" - - if e.Value.String != nil { - valArg = b.pushArgument(*e.Value.String) - } else { - attrType = "numeric" - valArg = b.pushArgument(*e.Value.Number) - } - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("key = %s AND value > %s", varArg, valArg), - ) -} - -func (e *GreaterOrEqualThan) Evaluate(b *QueryBuilder) string { - attrType := "string" - varArg := b.pushArgument(e.Var) - valArg := "" - - if e.Value.String != nil { - valArg = b.pushArgument(*e.Value.String) - } else { - attrType = "numeric" - valArg = b.pushArgument(*e.Value.Number) - } - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("key = %s AND value >= %s", varArg, valArg), - ) -} - -func (e *Equality) Evaluate(b *QueryBuilder) string { - attrType := "string" - varArg := b.pushArgument(e.Var) - valArg := "" - - op := "=" - if e.IsNot { - op = "!=" - } - - if e.Value.String != nil { - valArg = b.pushArgument(*e.Value.String) - } else { - attrType = "numeric" - valArg = b.pushArgument(*e.Value.Number) - } - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("key = %s AND value %s %s", varArg, op, valArg), - ) -} - -func (e *Inclusion) Evaluate(b *QueryBuilder) string { - var values []string - attrType := "string" - if len(e.Values.Strings) > 0 { - - values = make([]string, 0, len(e.Values.Strings)) - for _, value := range e.Values.Strings { - if e.Var == OwnerAttributeKey || - e.Var == CreatorAttributeKey || - e.Var == KeyAttributeKey { - values = append(values, b.pushArgument(strings.ToLower(value))) - } else { - values = append(values, b.pushArgument(value)) - } - } - - } else { - attrType = "numeric" - values = make([]string, 0, len(e.Values.Numbers)+1) - values = append(values, e.Var) - for _, value := range e.Values.Numbers { - values = append(values, b.pushArgument(value)) - } - } - - paramStr := strings.Join(values, ", ") - - condition := fmt.Sprintf("a.value IN (%s)", paramStr) - if e.IsNot { - condition = fmt.Sprintf("a.value NOT IN (%s)", paramStr) - } - - keyArg := b.pushArgument(e.Var) - - return b.createAnnotationQuery( - attrType, - fmt.Sprintf("a.key = %s AND %s", keyArg, - condition, - ), - ) -} diff --git a/query/types.go b/query/types.go deleted file mode 100644 index 3a8487f..0000000 --- a/query/types.go +++ /dev/null @@ -1,124 +0,0 @@ -package query - -import ( - "encoding/json" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - -var KeyAttributeKey = "$key" -var CreatorAttributeKey = "$creator" -var OwnerAttributeKey = "$owner" -var ExpirationAttributeKey = "$expiration" -var CreatedAtBlockKey = "$createdAtBlock" -var SequenceAttributeKey = "$sequence" - -type OrderByAnnotation struct { - Name string `json:"name"` - Type string `json:"type"` - Descending bool `json:"desc"` -} - -type QueryResponse struct { - Data []json.RawMessage `json:"data"` - BlockNumber uint64 `json:"blockNumber"` - Cursor *string `json:"cursor,omitempty"` -} - -type Cursor struct { - BlockNumber uint64 `json:"blockNumber"` - ColumnValues []CursorValue `json:"columnValues"` -} - -type CursorValue struct { - ColumnName string `json:"columnName"` - Value any `json:"value"` - Descending bool `json:"desc"` -} - -type StringAnnotation struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type NumericAnnotation struct { - Key string `json:"key"` - Value uint64 `json:"value"` -} - -type EntityData struct { - Key *common.Hash `json:"key,omitempty"` - Value hexutil.Bytes `json:"value,omitempty"` - ContentType *string `json:"contentType,omitempty"` - ExpiresAt *uint64 `json:"expiresAt,omitempty"` - Owner *common.Address `json:"owner,omitempty"` - CreatedAtBlock *uint64 `json:"createdAtBlock,omitempty"` - LastModifiedAtBlock *uint64 `json:"lastModifiedAtBlock,omitempty"` - TransactionIndexInBlock *uint64 `json:"transactionIndexInBlock,omitempty"` - OperationIndexInTransaction *uint64 `json:"operationIndexInTransaction,omitempty"` - - StringAttributes []StringAnnotation `json:"stringAttributes,omitempty"` - NumericAttributes []NumericAnnotation `json:"numericAttributes,omitempty"` -} - -type IncludeData struct { - Key bool `json:"key"` - Attributes bool `json:"attributes"` - SyntheticAttributes bool `json:"syntheticAttributes"` - Payload bool `json:"payload"` - ContentType bool `json:"contentType"` - Expiration bool `json:"expiration"` - Owner bool `json:"owner"` - CreatedAtBlock bool `json:"createdAtBlock"` - LastModifiedAtBlock bool `json:"lastModifiedAtBlock"` - TransactionIndexInBlock bool `json:"transactionIndexInBlock"` - OperationIndexInTransaction bool `json:"operationIndexInTransaction"` -} - -type Options struct { - AtBlock *uint64 `json:"atBlock"` - IncludeData *IncludeData `json:"includeData"` - OrderBy []OrderByAnnotation `json:"orderBy"` - ResultsPerPage uint64 `json:"resultsPerPage"` - Cursor string `json:"cursor"` -} - -func (options *Options) ToInternalQueryOptions() (*InternalQueryOptions, error) { - defaultIncludeData := &IncludeData{ - Key: true, - Expiration: true, - Owner: true, - Payload: true, - ContentType: true, - Attributes: true, - } - switch { - case options == nil: - return &InternalQueryOptions{ - IncludeData: defaultIncludeData, - }, nil - case options.IncludeData == nil: - return &InternalQueryOptions{ - IncludeData: defaultIncludeData, - OrderBy: options.OrderBy, - AtBlock: options.AtBlock, - Cursor: options.Cursor, - }, nil - default: - iq := InternalQueryOptions{ - OrderBy: options.OrderBy, - AtBlock: options.AtBlock, - Cursor: options.Cursor, - IncludeData: options.IncludeData, - } - return &iq, nil - } -} - -type InternalQueryOptions struct { - AtBlock *uint64 `json:"atBlock"` - IncludeData *IncludeData `json:"includeData"` - OrderBy []OrderByAnnotation `json:"orderBy"` - Cursor string `json:"cursor"` -} diff --git a/sqlstore/join_method.go b/sqlstore/join_method.go new file mode 100644 index 0000000..90c622d --- /dev/null +++ b/sqlstore/join_method.go @@ -0,0 +1,395 @@ +package sqlstore + +import ( + "fmt" + "strings" + + "github.com/Arkiv-Network/sqlite-store/query" +) + +type JoinEvaluator struct{} + +var _ query.QueryEvaluator = JoinEvaluator{} + +type AttrJoin struct { + Table string + Alias string +} + +func (e JoinEvaluator) EvaluateAST(a *query.AST, options *query.QueryOptions) (*query.SelectQuery, error) { + b := QueryBuilder{ + options: *options, + queryBuilder: &strings.Builder{}, + args: []any{}, + needsWhere: true, + } + + b.queryBuilder.WriteString(strings.Join( + []string{ + "SELECT", + b.options.ColumnString(), + "FROM payloads AS e", + }, + " ", + )) + + if a.Expr != nil { + attrJoins := make(map[string]AttrJoin) + e.addJoinsOr(&a.Expr.Or, attrJoins) + + for attrName, join := range attrJoins { + attrPlaceholder := b.PushArgument(attrName) + fmt.Fprintf( + b.queryBuilder, + " INNER JOIN %[1]s AS %[2]s ON e.entity_key = %[2]s.entity_key AND e.from_block = %[2]s.from_block AND %[2]s.key = %[3]s", + join.Table, + join.Alias, + attrPlaceholder, + ) + } + } + + for i, orderBy := range b.options.OrderByAnnotations { + tableName := "" + switch orderBy.Type { + case "string": + tableName = "string_attributes" + case "numeric": + tableName = "numeric_attributes" + default: + return nil, fmt.Errorf("a type of either 'string' or 'numeric' needs to be provided for the annotation '%s'", orderBy.Name) + } + + sortingTable := fmt.Sprintf("arkiv_annotation_sorting%d", i) + + keyPlaceholder := b.PushArgument(orderBy.Name) + + fmt.Fprintf(b.queryBuilder, + " LEFT JOIN %[1]s AS %s"+ + " ON %[2]s.entity_key = e.entity_key"+ + " AND %[2]s.from_block = e.from_block"+ + " AND %[2]s.key = %[3]s", + + tableName, + sortingTable, + keyPlaceholder, + ) + } + + err := query.AddPaginationArguments(&b) + if err != nil { + return nil, fmt.Errorf("error adding the pagination condition: %w", err) + } + + if b.needsWhere { + b.queryBuilder.WriteString(" WHERE ") + b.needsWhere = false + } else { + b.queryBuilder.WriteString(" AND ") + } + + blockArg := b.PushArgument(b.options.AtBlock) + fmt.Fprintf(b.queryBuilder, "%s BETWEEN e.from_block AND e.to_block - 1", blockArg) + + if a.Expr != nil { + if b.needsWhere { + b.queryBuilder.WriteString(" WHERE ") + b.needsWhere = false + } else { + b.queryBuilder.WriteString(" AND ") + } + + e.pushWhereConditionsExpr(a.Expr, &b) + } + + b.queryBuilder.WriteString(" ORDER BY ") + + orderColumns := make([]string, 0, len(b.options.OrderBy)) + for _, o := range b.options.OrderBy { + suffix := "" + if o.Descending { + suffix = " DESC" + } + orderColumns = append(orderColumns, o.Column.Name+suffix) + } + b.queryBuilder.WriteString(strings.Join(orderColumns, ", ")) + + fmt.Fprintf(b.queryBuilder, " LIMIT %d", query.QueryResultCountLimit) + + return &query.SelectQuery{ + Query: b.queryBuilder.String(), + Args: b.args, + }, nil +} + +func (e JoinEvaluator) pushWhereConditionsExpr(expr *query.ASTExpr, b *QueryBuilder) { + b.queryBuilder.WriteString("(") + e.pushWhereConditionsOr(&expr.Or, b) + b.queryBuilder.WriteString(")") +} + +func (e JoinEvaluator) addJoinsOr(expr *query.ASTOr, j map[string]AttrJoin) { + for _, r := range expr.Terms { + e.addJoinsAnd(&r, j) + } +} + +func (e JoinEvaluator) pushWhereConditionsOr(expr *query.ASTOr, b *QueryBuilder) { + e.pushWhereConditionsAnd(&expr.Terms[0], b) + for _, r := range expr.Terms[1:] { + b.queryBuilder.WriteString(" OR ") + e.pushWhereConditionsAnd(&r, b) + } +} + +func (e JoinEvaluator) addJoinsAnd(expr *query.ASTAnd, j map[string]AttrJoin) { + e.addJoinsTerm(&expr.Terms[0], j) + for _, r := range expr.Terms[1:] { + e.addJoinsTerm(&r, j) + } +} + +func (e JoinEvaluator) pushWhereConditionsAnd(expr *query.ASTAnd, b *QueryBuilder) { + e.pushWhereConditionsTerm(&expr.Terms[0], b) + for _, r := range expr.Terms[1:] { + b.queryBuilder.WriteString(" AND ") + e.pushWhereConditionsTerm(&r, b) + } +} + +func (JoinEvaluator) addJoinsTerm(e *query.ASTTerm, j map[string]AttrJoin) { + if e.LessThan != nil { + tableName := "string_attributes" + if e.LessThan.Value.Number != nil { + tableName = "numeric_attributes" + } + + j[e.LessThan.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.LessThan.Var), + } + return + } + + if e.LessOrEqualThan != nil { + tableName := "string_attributes" + if e.LessOrEqualThan.Value.Number != nil { + tableName = "numeric_attributes" + } + + j[e.LessOrEqualThan.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.LessOrEqualThan.Var), + } + return + } + + if e.GreaterThan != nil { + tableName := "string_attributes" + if e.GreaterThan.Value.Number != nil { + tableName = "numeric_attributes" + } + + j[e.GreaterThan.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.GreaterThan.Var), + } + return + } + + if e.GreaterOrEqualThan != nil { + tableName := "string_attributes" + if e.GreaterOrEqualThan.Value.Number != nil { + tableName = "numeric_attributes" + } + + j[e.GreaterOrEqualThan.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.GreaterOrEqualThan.Var), + } + return + } + + if e.Glob != nil { + tableName := "string_attributes" + + j[e.Glob.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.Glob.Var), + } + return + } + + if e.Assign != nil { + tableName := "string_attributes" + if e.Assign.Value.Number != nil { + tableName = "numeric_attributes" + } + + j[e.Assign.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.Assign.Var), + } + return + } + + if e.Inclusion != nil { + tableName := "string_attributes" + if e.Inclusion.Values.Numbers != nil { + tableName = "numeric_attributes" + } + + j[e.Inclusion.Var] = AttrJoin{ + Table: tableName, + Alias: attributeTableAlias(e.Inclusion.Var), + } + return + } + + panic("This should not happen!") +} + +func (JoinEvaluator) pushWhereConditionsTerm(e *query.ASTTerm, b *QueryBuilder) { + if e.LessThan != nil { + argName := "" + if e.LessThan.Value.String != nil { + argName = b.PushArgument(*e.LessThan.Value.String) + } else { + argName = b.PushArgument(*e.LessThan.Value.Number) + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value < %s", + attributeTableAlias(e.LessThan.Var), + argName, + ) + return + } + + if e.LessOrEqualThan != nil { + argName := "" + if e.LessOrEqualThan.Value.String != nil { + argName = b.PushArgument(*e.LessOrEqualThan.Value.String) + } else { + argName = b.PushArgument(*e.LessOrEqualThan.Value.Number) + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value <= %s", + attributeTableAlias(e.LessOrEqualThan.Var), + argName, + ) + return + } + + if e.GreaterThan != nil { + argName := "" + if e.GreaterThan.Value.String != nil { + argName = b.PushArgument(*e.GreaterThan.Value.String) + } else { + argName = b.PushArgument(*e.GreaterThan.Value.Number) + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value > %s", + attributeTableAlias(e.GreaterThan.Var), + argName, + ) + return + } + + if e.GreaterOrEqualThan != nil { + argName := "" + if e.GreaterOrEqualThan.Value.String != nil { + argName = b.PushArgument(*e.GreaterOrEqualThan.Value.String) + } else { + argName = b.PushArgument(*e.GreaterOrEqualThan.Value.Number) + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value >= %s", + attributeTableAlias(e.GreaterOrEqualThan.Var), + argName, + ) + return + } + + if e.Glob != nil { + argName := b.PushArgument(e.Glob.Value) + + op := "~" + if e.Glob.IsNot { + op = "!~" + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value %s %s", + attributeTableAlias(e.Glob.Var), + op, + argName, + ) + return + } + + if e.Assign != nil { + argName := "" + if e.Assign.Value.String != nil { + argName = b.PushArgument(*e.Assign.Value.String) + } else { + argName = b.PushArgument(*e.Assign.Value.Number) + } + + op := "=" + if e.Assign.IsNot { + op = "!=" + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value %s %s", + attributeTableAlias(e.Assign.Var), + op, + argName, + ) + return + } + + if e.Inclusion != nil { + args := []any{} + if e.Inclusion.Values.Strings != nil { + for _, s := range e.Inclusion.Values.Strings { + args = append(args, s) + } + } else { + for _, s := range e.Inclusion.Values.Numbers { + args = append(args, s) + } + } + + argNames := []string{} + for _, arg := range args { + argNames = append(argNames, b.PushArgument(arg)) + } + + op := "IN" + if e.Inclusion.IsNot { + op = "NOT IN" + } + + fmt.Fprintf( + b.queryBuilder, + "%s.value %s (%s)", + attributeTableAlias(e.Inclusion.Var), + op, + strings.Join(argNames, ", "), + ) + return + } + + panic("This should not happen!") +} diff --git a/sqlstore/query.go b/sqlstore/query.go new file mode 100644 index 0000000..71214b4 --- /dev/null +++ b/sqlstore/query.go @@ -0,0 +1,50 @@ +package sqlstore + +import ( + "fmt" + "hash/fnv" + "strings" + + "github.com/Arkiv-Network/sqlite-store/query" +) + +type QueryBuilder struct { + queryBuilder *strings.Builder + args []any + argsCount uint32 + tableCounter uint32 + needsWhere bool + options query.QueryOptions +} + +var _ query.Builder = &QueryBuilder{} + +func attributeTableAlias(name string) string { + h := fnv.New32a() + h.Write([]byte(name)) + + return fmt.Sprintf("arkiv_attr_%d", h.Sum32()) +} + +func (b *QueryBuilder) PushArgument(arg any) string { + b.args = append(b.args, arg) + b.argsCount += 1 + return fmt.Sprintf("$%d", b.argsCount) +} + +func (b *QueryBuilder) GetOptions() *query.QueryOptions { + return &b.options +} + +func (b *QueryBuilder) WriteWhereClause(condition string) { + if b.needsWhere { + b.queryBuilder.WriteString(" WHERE ") + b.needsWhere = false + } else { + b.queryBuilder.WriteString(" AND ") + } + + b.queryBuilder.WriteString("(") + b.queryBuilder.WriteString(condition) + b.queryBuilder.WriteString(")") +} diff --git a/sqlstore/queryoptions.go b/sqlstore/queryoptions.go new file mode 100644 index 0000000..7ba93a6 --- /dev/null +++ b/sqlstore/queryoptions.go @@ -0,0 +1,137 @@ +package sqlstore + +import ( + "fmt" + "log/slog" + "slices" + + "github.com/Arkiv-Network/sqlite-store/query" +) + +func NewQueryOptions(log *slog.Logger, latestHead uint64, options *query.InternalQueryOptions) (*query.QueryOptions, error) { + queryOptions := query.QueryOptions{ + Log: log, + OrderByAnnotations: options.OrderBy, + IncludeData: options.IncludeData, + } + + queryOptions.Columns = []query.Column{} + + // We always need the primary key of the payloads table because of sorting + queryOptions.Columns = append(queryOptions.Columns, + query.Column{ + Name: "from_block", + QualifiedName: "e.from_block", + }, + query.Column{ + Name: "entity_key", + QualifiedName: "e.entity_key", + IsBytes: true, + }, + ) + + if options.IncludeData.Payload { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "payload", + QualifiedName: "e.payload", + }) + } + if options.IncludeData.ContentType { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "content_type", + QualifiedName: "e.content_type", + }) + } + if options.IncludeData.Attributes { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "string_attributes", + QualifiedName: "e.string_attributes", + }) + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "numeric_attributes", + QualifiedName: "e.numeric_attributes", + }) + } + + for i := range options.OrderBy { + name := fmt.Sprintf("arkiv_annotation_sorting%d_value", i) + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: name, + QualifiedName: fmt.Sprintf("arkiv_annotation_sorting%d.value", i), + }) + } + + if options.IncludeData.Owner { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "owner", + QualifiedName: fmt.Sprintf("e.string_attributes ->> '%s'", query.OwnerAttributeKey), + }) + } + if options.IncludeData.Expiration { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "expires_at", + QualifiedName: fmt.Sprintf("e.numeric_attributes ->> '%s'", query.ExpirationAttributeKey), + }) + } + if options.IncludeData.CreatedAtBlock { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "created_at_block", + QualifiedName: fmt.Sprintf("e.numeric_attributes ->> '%s'", query.CreatedAtBlockKey), + }) + } + if options.IncludeData.LastModifiedAtBlock || + options.IncludeData.TransactionIndexInBlock || + options.IncludeData.OperationIndexInTransaction { + queryOptions.Columns = append(queryOptions.Columns, query.Column{ + Name: "sequence", + QualifiedName: fmt.Sprintf("e.numeric_attributes ->> '%s'", query.SequenceAttributeKey), + }) + } + + // Sort so that we can use binary search later + slices.SortFunc(queryOptions.Columns, query.Column.Compare) + + queryOptions.OrderBy = []query.OrderBy{} + + for i, o := range queryOptions.OrderByAnnotations { + queryOptions.OrderBy = append(queryOptions.OrderBy, query.OrderBy{ + Column: query.Column{ + Name: fmt.Sprintf("arkiv_annotation_sorting%d_value", i), + QualifiedName: fmt.Sprintf("arkiv_annotation_sorting%d.value", i), + }, + Descending: o.Descending, + }) + } + queryOptions.OrderBy = append(queryOptions.OrderBy, + query.OrderBy{ + Column: query.Column{ + Name: "from_block", + QualifiedName: "e.from_block", + }, + }, + query.OrderBy{ + Column: query.Column{ + Name: "entity_key", + QualifiedName: "e.entity_key", + IsBytes: true, + }, + }, + ) + + queryOptions.AtBlock = latestHead + + if len(options.Cursor) != 0 { + cursor, err := queryOptions.DecodeCursor(options.Cursor) + if err != nil { + return nil, err + } + queryOptions.AtBlock = cursor.BlockNumber + queryOptions.Cursor = cursor.ColumnValues + } + + if options.AtBlock != nil { + queryOptions.AtBlock = *options.AtBlock + } + + return &queryOptions, nil +} diff --git a/sqlstore/sqlstore.go b/sqlstore/sqlstore.go index 2781f28..eb43c47 100644 --- a/sqlstore/sqlstore.go +++ b/sqlstore/sqlstore.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/Arkiv-Network/query-api/query" + "github.com/Arkiv-Network/sqlite-store/query" "github.com/ethereum/go-ethereum/common" ) @@ -78,7 +78,6 @@ func (s *SQLStore) QueryEntities( ctx context.Context, req string, op *query.Options, - sqlDialect string, ) (*query.QueryResponse, error) { if op != nil { @@ -104,14 +103,14 @@ func (s *SQLStore) QueryEntities( return nil, err } - queryOptions, err := query.NewQueryOptions(s.log, latestHead, options) + queryOptions, err := NewQueryOptions(s.log, latestHead, options) if err != nil { return nil, err } s.log.Info("final query options", "options", queryOptions) - evaluatedQuery, err := expr.Evaluate2(queryOptions, sqlDialect) + evaluatedQuery, err := JoinEvaluator{}.EvaluateAST(expr, queryOptions) if err != nil { return nil, err }