Consider switching to pgsql-parser SQL parser

package.json

@@ -45,7 +45,7 @@
     "@aws-sdk/client-s3": "^3.2.0",
     "bunyan": "^1.8.15",
     "merge-stream": "^2.0.0",
-    "node-sql-parser": "^1.11.0",
+    "pgsql-ast-parser": "^6.3.2",
     "sqlite3": "^5.0.0"
   },
   "devDependencies": {

pgsql-parser.d.ts

@@ -0,0 +1 @@
+declare module "pgsql-parser";

src/utils/sql-query.helper.test.ts

@@ -35,12 +35,7 @@ describe("it getting db and table from SQL clause", () => {
     );
   });
   it("Single table FROM only", () => {
-    expect(() => getTableAndDbAndExpr("SELECT * FROM db.t AS t, db2.t2 as t2")).toThrowError(
-      "Only single table sources supported for now",
-    );
-  });
-  it("No FROM DUAL", () => {
-    expect(() => getTableAndDbAndExpr("SELECT * FROM DUAL")).toThrowError("DUAL not supported");
+    expect(() => getTableAndDbAndExpr("SELECT * FROM db.t AS t, db2.t2 as t2")).toThrowError();
   });
   it("Both db and table must be given", () => {
     expect(() => getTableAndDbAndExpr("SELECT * FROM t")).toThrowError("Both db and table needed");
@@ -102,8 +97,8 @@ describe("ensuring JSON table queries work too", () => {
   });
 });
 
-describe("S3 Select SQL clauses", () => {
-  it("changes partition column filters to TRUE (partition filtering has already been done)", () => {
+describe.only("S3 Select SQL clauses", () => {
+  it.only("changes partition column filters to TRUE (partition filtering has already been done)", () => {
     const sql = "SELECT * FROM s3Object WHERE part=0";
     const expected = "SELECT * FROM s3Object WHERE TRUE";
     expect(getNonPartsSQL(sql, ["part"])).toEqual(expected);

src/utils/sql-query.helper.ts

@@ -1,4 +1,4 @@
-import { AST, From, Parser, Select } from "node-sql-parser";
+import { astMapper, FromTable, parse, parseFirst, SelectFromStatement, Statement, toSql } from "pgsql-ast-parser";
 
 /*
  * ## How to filter S3 Keys based on table partition keys that map to "folders" on S3
@@ -13,26 +13,20 @@ import { AST, From, Parser, Select } from "node-sql-parser";
  * to be applied. Then run the full query on S3 Select over the S3 Objects.
  */
 
-const nodeSqlParserOpts = {
-  database: "Mysql",
-};
-
-export function getTableAndDbFromAST(ast: AST | AST[]): [string | null, string] {
-  if (Array.isArray(ast)) throw new Error("Multiple queries not supported");
+export function getTableAndDbFromAST(ast: Statement): [string | undefined, string] {
   if (ast.type !== "select") throw new Error("Only SELECT queries are supported");
   if (!ast.from) throw new Error("Only SELECT queries with FROM are supported");
   if (ast.from.length !== 1) throw new Error("Only single table sources supported for now");
   const from = ast.from[0];
-  if (Object.prototype.hasOwnProperty.call(from, "type")) throw new Error("DUAL not supported");
-  const { db, table } = <From>from;
-  return [db, table];
+  if (from.type !== "table") throw new Error("Only FROM table supported");
+  const { schema, name: table } = <FromTable>from;
+  return [schema, table];
 }
 
 export function getPlainSQLAndExpr(sql: string): [string, string] {
   const regex = /FROM (\w+)(\.*)(\w*)(\S*)\s*(.*)$/im;
   const matches = sql.match(regex);
   const expr = matches && matches.length >= 5 ? matches[4] : "";
-  //const rest = matches && matches.length >= 6 ? matches[5] : "";
   if (expr.trim() === ";") throw new Error("Multiple queries not supported (;)");
   if (expr.trim()[0] === ".") throw new Error("Can not use format FROM a.b.c");
   const plainSql = sql.replace(regex, `FROM $1$2$3 $5`).trim();
@@ -41,79 +35,57 @@ export function getPlainSQLAndExpr(sql: string): [string, string] {
 
 export function getTableAndDbAndExpr(sql: string): [string, string, string] {
   const [plainSql, expr] = getPlainSQLAndExpr(sql);
-  const parser = new Parser();
-  const ast = parser.astify(plainSql, nodeSqlParserOpts);
-  const [db, table] = getTableAndDbFromAST(ast);
+  const ast = parse(plainSql);
+  if (ast.length !== 1) throw new Error("Multiple queries not supported");
+  const [db, table] = getTableAndDbFromAST(ast[0]);
   if (!db || !table) throw new Error("Both db and table needed");
   return [db, table, expr];
 }
 
-export function getSQLWhereAST(sql: string): AST {
-  const [plainSql] = getPlainSQLAndExpr(sql);
-  const parser = new Parser();
-  const ast = parser.astify(plainSql, nodeSqlParserOpts);
-  getTableAndDbFromAST(ast);
-  return (<Select>ast).where;
-}
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function partFilter(partCols: string[], column: any): boolean {
+function partFilter(partCols: string[], column: string): boolean {
   return partCols.some(c => c === column);
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function nonPartFilter(partCols: string[], column: any): boolean {
+function nonPartFilter(partCols: string[], column: string): boolean {
   return partCols.every(c => c !== column);
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function filterParts(ast: any, partCols: string[], filter: (partCols: string[], column: any) => boolean): any {
-  const nonFiltering = { type: "bool", value: true };
-  if (ast.type === "column_ref") return ast;
-  if (ast.left?.type === "column_ref" && filter(partCols, ast.left.column)) return nonFiltering;
-  if (ast.right?.type === "column_ref" && filter(partCols, ast.right.column)) return nonFiltering;
-  if (!ast.left || !ast.right) return ast;
-  return { ...ast, left: filterParts(ast.left, partCols, filter), right: filterParts(ast.right, partCols, filter) };
+function filterParts(
+  sql: string,
+  partCols: string[],
+  filter: (partCols: string[], column: string) => boolean,
+): SelectFromStatement | null | undefined {
+  return <SelectFromStatement>astMapper(_map => ({
+    ref: c => (filter(partCols, c.name) ? null : c),
+  })).statement(parseFirst(sql));
 }
 
-export function makePartitionSpecificAST(ast: AST | undefined, partitionColumns: string[]): AST | undefined {
-  if (!ast) return;
-  return filterParts(ast, partitionColumns, nonPartFilter);
+export function makePartitionSpecificAST(
+  sql: string,
+  partitionColumns: string[],
+): SelectFromStatement | undefined | null {
+  return filterParts(sql, partitionColumns, nonPartFilter);
 }
 
-export function makeSelectSpecificAST(ast: AST | undefined, partitionColumns: string[]): AST | undefined {
-  if (!ast) return;
-  return filterParts(ast, partitionColumns, partFilter);
+export function makeSelectSpecificAST(sql: string, partitionColumns: string[]): SelectFromStatement | undefined | null {
+  return filterParts(sql, partitionColumns, partFilter);
 }
 
-export function getSQLWhereStringFromAST(where: AST | undefined): string {
-  const parser = new Parser();
-  return parser
-    .sqlify(
-      {
-        where,
-        with: null,
-        type: "select",
-        options: null,
-        distinct: null,
-        columns: "*",
-        from: [{ db: null, table: "s3Object", as: null }],
-        groupby: null,
-        having: null,
-        orderby: null,
-        limit: null,
-      },
-      nodeSqlParserOpts,
-    )
+export function getSQLWhereStringFromAST(selStmt: SelectFromStatement | undefined | null): string {
+  return toSql
+    .statement({
+      where: selStmt?.where,
+      type: "select",
+      from: [{ type: "table", name: "s3Object" }],
+    })
     .substring(25);
 }
 
-function replaceWhereInSQL(sql: string, newWhere: AST | undefined): string {
-  if (!newWhere) return sql;
+function replaceWhereInSQL(sql: string, selStm: SelectFromStatement | undefined | null): string {
+  if (!selStm) return sql;
   const [plainSql] = getPlainSQLAndExpr(sql);
-  const parser = new Parser();
-  const ast = parser.astify(plainSql, nodeSqlParserOpts);
-  return parser.sqlify(<Select>{ ...ast, where: newWhere }, nodeSqlParserOpts).replace(/`/g, "");
+  const ast = parseFirst(plainSql);
+  return toSql.statement(<SelectFromStatement>{ ...ast, where: selStm?.where });
 }
 
 export function getSQLLimit(sql: string): number {
@@ -127,9 +99,9 @@ export function setSQLLimit(sql: string, limit: number): string {
 }
 
 export function getPartsOnlySQLWhereString(expression: string, partitionColumns: string[]): string {
-  return getSQLWhereStringFromAST(makePartitionSpecificAST(getSQLWhereAST(expression), partitionColumns));
+  return getSQLWhereStringFromAST(makePartitionSpecificAST(expression, partitionColumns));
 }
 
 export function getNonPartsSQL(expression: string, partitionColumns: string[]): string {
-  return replaceWhereInSQL(expression, makeSelectSpecificAST(getSQLWhereAST(expression), partitionColumns));
+  return replaceWhereInSQL(expression, makeSelectSpecificAST(expression, partitionColumns));
 }

tsconfig.json

@@ -31,6 +31,6 @@
     "inlineSources": true,
     "forceConsistentCasingInFileNames": true
   },
-  "include": ["src/**/*.ts", "jest.config.js"],
+  "include": ["src/**/*.ts", "jest.config.js", "pgsql-parser.d.ts"],
   "exclude": ["**/*.test.ts"]
 }

yarn.lock

@@ -1756,11 +1756,6 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
-big-integer@^1.6.48:
-  version "1.6.48"
-  resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e"
-  integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==
-
 binary-extensions@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9"
@@ -2010,6 +2005,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@^2.19.0:
+  version "2.20.3"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
+  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
+
 compare-versions@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62"
@@ -2216,6 +2216,11 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
+discontinuous-range@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a"
+  integrity sha1-44Mx8IRLukm5qctxx3FYWqsbxlo=
+
 doctrine@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961"
@@ -3976,6 +3981,11 @@ moment@^2.19.3:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
   integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
 
+moo@^0.5.0, moo@^0.5.1:
+  version "0.5.1"
+  resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4"
+  integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -4032,6 +4042,16 @@ ncp@~2.0.0:
   resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3"
   integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=
 
+nearley@^2.19.5:
+  version "2.20.1"
+  resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474"
+  integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==
+  dependencies:
+    commander "^2.19.0"
+    moo "^0.5.0"
+    railroad-diagrams "^1.0.0"
+    randexp "0.4.6"
+
 needle@^2.2.1:
   version "2.5.2"
   resolved "https://registry.yarnpkg.com/needle/-/needle-2.5.2.tgz#cf1a8fce382b5a280108bba90a14993c00e4010a"
@@ -4107,13 +4127,6 @@ node-pre-gyp@^0.11.0:
     semver "^5.3.0"
     tar "^4"
 
-node-sql-parser@^1.11.0:
-  version "1.11.0"
-  resolved "https://registry.yarnpkg.com/node-sql-parser/-/node-sql-parser-1.11.0.tgz#7f4a497f54befe341cbc0d9b916877471e182b76"
-  integrity sha512-lvXZ0fPh3J0bQ2/GHyVEqZ8k732SdLInTahkoUaPYYnYH0zp0XSprjyXkbt6TbA0ujLl1lp+wGnlJFINRWJ/ww==
-  dependencies:
-    big-integer "^1.6.48"
-
 "nopt@2 || 3":
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9"
@@ -4418,6 +4431,14 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
+pgsql-ast-parser@^6.3.2:
+  version "6.3.2"
+  resolved "https://registry.yarnpkg.com/pgsql-ast-parser/-/pgsql-ast-parser-6.3.2.tgz#c79cb54614e57647d9f6bee9f0fdce1d7548eda5"
+  integrity sha512-25G57vKnOdAgkFuBz7TC0/Qg3dtJtyu+UrSIzpTEFny9/C4Ol8W30RPWge+h42ag/maqhaUVzWfTk8BWiIheUA==
+  dependencies:
+    moo "^0.5.1"
+    nearley "^2.19.5"
+
 picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
@@ -4539,6 +4560,19 @@ querystring@0.2.0:
   resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
   integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
 
+railroad-diagrams@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e"
+  integrity sha1-635iZ1SN3t+4mcG5Dlc3RVnN234=
+
+randexp@0.4.6:
+  version "0.4.6"
+  resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3"
+  integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==
+  dependencies:
+    discontinuous-range "1.0.0"
+    ret "~0.1.10"
+
 rc@^1.2.7:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"