diff --git a/package.json b/package.json index 8add7c7..1680af6 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "@aws-sdk/client-s3": "^3.2.0", "bunyan": "^1.8.15", "merge-stream": "^2.0.0", - "node-sql-parser": "^1.11.0", + "pgsql-ast-parser": "^6.3.2", "sqlite3": "^5.0.0" }, "devDependencies": { diff --git a/pgsql-parser.d.ts b/pgsql-parser.d.ts new file mode 100644 index 0000000..937a5a5 --- /dev/null +++ b/pgsql-parser.d.ts @@ -0,0 +1 @@ +declare module "pgsql-parser"; diff --git a/src/utils/sql-query.helper.test.ts b/src/utils/sql-query.helper.test.ts index 3739ba9..7d44d3e 100644 --- a/src/utils/sql-query.helper.test.ts +++ b/src/utils/sql-query.helper.test.ts @@ -35,12 +35,7 @@ describe("it getting db and table from SQL clause", () => { ); }); it("Single table FROM only", () => { - expect(() => getTableAndDbAndExpr("SELECT * FROM db.t AS t, db2.t2 as t2")).toThrowError( - "Only single table sources supported for now", - ); - }); - it("No FROM DUAL", () => { - expect(() => getTableAndDbAndExpr("SELECT * FROM DUAL")).toThrowError("DUAL not supported"); + expect(() => getTableAndDbAndExpr("SELECT * FROM db.t AS t, db2.t2 as t2")).toThrowError(); }); it("Both db and table must be given", () => { expect(() => getTableAndDbAndExpr("SELECT * FROM t")).toThrowError("Both db and table needed"); @@ -102,8 +97,8 @@ describe("ensuring JSON table queries work too", () => { }); }); -describe("S3 Select SQL clauses", () => { - it("changes partition column filters to TRUE (partition filtering has already been done)", () => { +describe.only("S3 Select SQL clauses", () => { + it.only("changes partition column filters to TRUE (partition filtering has already been done)", () => { const sql = "SELECT * FROM s3Object WHERE part=0"; const expected = "SELECT * FROM s3Object WHERE TRUE"; expect(getNonPartsSQL(sql, ["part"])).toEqual(expected); diff --git a/src/utils/sql-query.helper.ts b/src/utils/sql-query.helper.ts index 8219b51..7009a55 100644 --- a/src/utils/sql-query.helper.ts +++ b/src/utils/sql-query.helper.ts @@ -1,4 +1,4 @@ -import { AST, From, Parser, Select } from "node-sql-parser"; +import { astMapper, FromTable, parse, parseFirst, SelectFromStatement, Statement, toSql } from "pgsql-ast-parser"; /* * ## How to filter S3 Keys based on table partition keys that map to "folders" on S3 @@ -13,26 +13,20 @@ import { AST, From, Parser, Select } from "node-sql-parser"; * to be applied. Then run the full query on S3 Select over the S3 Objects. */ -const nodeSqlParserOpts = { - database: "Mysql", -}; - -export function getTableAndDbFromAST(ast: AST | AST[]): [string | null, string] { - if (Array.isArray(ast)) throw new Error("Multiple queries not supported"); +export function getTableAndDbFromAST(ast: Statement): [string | undefined, string] { if (ast.type !== "select") throw new Error("Only SELECT queries are supported"); if (!ast.from) throw new Error("Only SELECT queries with FROM are supported"); if (ast.from.length !== 1) throw new Error("Only single table sources supported for now"); const from = ast.from[0]; - if (Object.prototype.hasOwnProperty.call(from, "type")) throw new Error("DUAL not supported"); - const { db, table } = from; - return [db, table]; + if (from.type !== "table") throw new Error("Only FROM table supported"); + const { schema, name: table } = from; + return [schema, table]; } export function getPlainSQLAndExpr(sql: string): [string, string] { const regex = /FROM (\w+)(\.*)(\w*)(\S*)\s*(.*)$/im; const matches = sql.match(regex); const expr = matches && matches.length >= 5 ? matches[4] : ""; - //const rest = matches && matches.length >= 6 ? matches[5] : ""; if (expr.trim() === ";") throw new Error("Multiple queries not supported (;)"); if (expr.trim()[0] === ".") throw new Error("Can not use format FROM a.b.c"); const plainSql = sql.replace(regex, `FROM $1$2$3 $5`).trim(); @@ -41,79 +35,57 @@ export function getPlainSQLAndExpr(sql: string): [string, string] { export function getTableAndDbAndExpr(sql: string): [string, string, string] { const [plainSql, expr] = getPlainSQLAndExpr(sql); - const parser = new Parser(); - const ast = parser.astify(plainSql, nodeSqlParserOpts); - const [db, table] = getTableAndDbFromAST(ast); + const ast = parse(plainSql); + if (ast.length !== 1) throw new Error("Multiple queries not supported"); + const [db, table] = getTableAndDbFromAST(ast[0]); if (!db || !table) throw new Error("Both db and table needed"); return [db, table, expr]; } -export function getSQLWhereAST(sql: string): AST { - const [plainSql] = getPlainSQLAndExpr(sql); - const parser = new Parser(); - const ast = parser.astify(plainSql, nodeSqlParserOpts); - getTableAndDbFromAST(ast); - return ({ ...ast, where: newWhere }, nodeSqlParserOpts).replace(/`/g, ""); + const ast = parseFirst(plainSql); + return toSql.statement({ ...ast, where: selStm?.where }); } export function getSQLLimit(sql: string): number { @@ -127,9 +99,9 @@ export function setSQLLimit(sql: string, limit: number): string { } export function getPartsOnlySQLWhereString(expression: string, partitionColumns: string[]): string { - return getSQLWhereStringFromAST(makePartitionSpecificAST(getSQLWhereAST(expression), partitionColumns)); + return getSQLWhereStringFromAST(makePartitionSpecificAST(expression, partitionColumns)); } export function getNonPartsSQL(expression: string, partitionColumns: string[]): string { - return replaceWhereInSQL(expression, makeSelectSpecificAST(getSQLWhereAST(expression), partitionColumns)); + return replaceWhereInSQL(expression, makeSelectSpecificAST(expression, partitionColumns)); } diff --git a/tsconfig.json b/tsconfig.json index 766139c..0c4f865 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,6 +31,6 @@ "inlineSources": true, "forceConsistentCasingInFileNames": true }, - "include": ["src/**/*.ts", "jest.config.js"], + "include": ["src/**/*.ts", "jest.config.js", "pgsql-parser.d.ts"], "exclude": ["**/*.test.ts"] } diff --git a/yarn.lock b/yarn.lock index ebeecd6..0cb0a90 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1756,11 +1756,6 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" -big-integer@^1.6.48: - version "1.6.48" - resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e" - integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w== - binary-extensions@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" @@ -2010,6 +2005,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +commander@^2.19.0: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" @@ -2216,6 +2216,11 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" +discontinuous-range@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a" + integrity sha1-44Mx8IRLukm5qctxx3FYWqsbxlo= + doctrine@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" @@ -3976,6 +3981,11 @@ moment@^2.19.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== +moo@^0.5.0, moo@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4" + integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w== + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -4032,6 +4042,16 @@ ncp@~2.0.0: resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M= +nearley@^2.19.5: + version "2.20.1" + resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474" + integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ== + dependencies: + commander "^2.19.0" + moo "^0.5.0" + railroad-diagrams "^1.0.0" + randexp "0.4.6" + needle@^2.2.1: version "2.5.2" resolved "https://registry.yarnpkg.com/needle/-/needle-2.5.2.tgz#cf1a8fce382b5a280108bba90a14993c00e4010a" @@ -4107,13 +4127,6 @@ node-pre-gyp@^0.11.0: semver "^5.3.0" tar "^4" -node-sql-parser@^1.11.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/node-sql-parser/-/node-sql-parser-1.11.0.tgz#7f4a497f54befe341cbc0d9b916877471e182b76" - integrity sha512-lvXZ0fPh3J0bQ2/GHyVEqZ8k732SdLInTahkoUaPYYnYH0zp0XSprjyXkbt6TbA0ujLl1lp+wGnlJFINRWJ/ww== - dependencies: - big-integer "^1.6.48" - "nopt@2 || 3": version "3.0.6" resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" @@ -4418,6 +4431,14 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +pgsql-ast-parser@^6.3.2: + version "6.3.2" + resolved "https://registry.yarnpkg.com/pgsql-ast-parser/-/pgsql-ast-parser-6.3.2.tgz#c79cb54614e57647d9f6bee9f0fdce1d7548eda5" + integrity sha512-25G57vKnOdAgkFuBz7TC0/Qg3dtJtyu+UrSIzpTEFny9/C4Ol8W30RPWge+h42ag/maqhaUVzWfTk8BWiIheUA== + dependencies: + moo "^0.5.1" + nearley "^2.19.5" + picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" @@ -4539,6 +4560,19 @@ querystring@0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= +railroad-diagrams@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e" + integrity sha1-635iZ1SN3t+4mcG5Dlc3RVnN234= + +randexp@0.4.6: + version "0.4.6" + resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3" + integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ== + dependencies: + discontinuous-range "1.0.0" + ret "~0.1.10" + rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"