From 708ba9872e8106538f868b7ed315eddc307124fd Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 3 Dec 2023 23:36:12 +0800 Subject: [PATCH 1/7] feat(Dockerfile): use node:18 --- ChhoeTaigiPgCreate/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ChhoeTaigiPgCreate/Dockerfile b/ChhoeTaigiPgCreate/Dockerfile index 982f869..5c06ced 100644 --- a/ChhoeTaigiPgCreate/Dockerfile +++ b/ChhoeTaigiPgCreate/Dockerfile @@ -1,4 +1,4 @@ -FROM node:14 +FROM node:18 WORKDIR /usr/src/app @@ -7,5 +7,3 @@ COPY package*.json ./ RUN npm install COPY . . - -RUN export NODE_ENV=production \ No newline at end of file From 9373c0b2a1a265581caa843d961f21917202b3dc Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 3 Dec 2023 23:36:52 +0800 Subject: [PATCH 2/7] chore: move .gitignore to the top-level of the monorepo --- ChhoeTaigiPgApiServer/.gitignore => .gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ChhoeTaigiPgApiServer/.gitignore => .gitignore (100%) diff --git a/ChhoeTaigiPgApiServer/.gitignore b/.gitignore similarity index 100% rename from ChhoeTaigiPgApiServer/.gitignore rename to .gitignore From c492121b5d54ddda5224c33856f51e8480dac4cc Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 3 Dec 2023 23:40:28 +0800 Subject: [PATCH 3/7] feat(package.json): upgrade to `knex@^3` --- ChhoeTaigiPgCreate/package-lock.json | 279 +++++++++++++++------------ ChhoeTaigiPgCreate/package.json | 2 +- 2 files changed, 157 insertions(+), 124 deletions(-) diff --git a/ChhoeTaigiPgCreate/package-lock.json b/ChhoeTaigiPgCreate/package-lock.json index 72690b5..50542e4 100644 --- a/ChhoeTaigiPgCreate/package-lock.json +++ b/ChhoeTaigiPgCreate/package-lock.json @@ -13,7 +13,7 @@ "@types/knex": "^0.16.1", "csvtojson": "^2.0.10", "express": "^4.17.1", - "knex": "^0.95.6", + "knex": "^3.0.1", "pg": "^8.6.0" }, "devDependencies": { @@ -527,16 +527,16 @@ "dev": true }, "node_modules/colorette": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", - "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==" + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" }, "node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", "engines": { - "node": ">= 10" + "node": ">=14" } }, "node_modules/concat-map": { @@ -866,9 +866,20 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "engines": { + "node": ">=8.0.0" + } }, "node_modules/get-stream": { "version": "4.1.0", @@ -883,9 +894,9 @@ } }, "node_modules/getopts": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", - "integrity": "sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" }, "node_modules/glob-parent": { "version": "5.1.2", @@ -942,17 +953,6 @@ "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", "dev": true }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -971,6 +971,17 @@ "node": ">=8" } }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", @@ -1079,11 +1090,11 @@ } }, "node_modules/is-core-module": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz", - "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -1204,31 +1215,35 @@ } }, "node_modules/knex": { - "version": "0.95.6", - "resolved": "https://registry.npmjs.org/knex/-/knex-0.95.6.tgz", - "integrity": "sha512-noRcmkJl1MdicUbezrcr8OtVLcqQ/cfLIwgAx5EaxNxQOIJff88rBeyLywUScGhQNd/b78DIKKXZzLMrm6h/cw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.0.1.tgz", + "integrity": "sha512-ruASxC6xPyDklRdrcDy6a9iqK+R9cGK214aiQa+D9gX2ZnHZKv6o6JC9ZfgxILxVAul4bZ13c3tgOAHSuQ7/9g==", "dependencies": { - "colorette": "1.2.1", - "commander": "^7.1.0", - "debug": "4.3.1", + "colorette": "2.0.19", + "commander": "^10.0.0", + "debug": "4.3.4", "escalade": "^3.1.1", "esm": "^3.2.25", - "getopts": "2.2.5", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", "interpret": "^2.2.0", "lodash": "^4.17.21", - "pg-connection-string": "2.4.0", - "rechoir": "^0.7.0", + "pg-connection-string": "2.6.1", + "rechoir": "^0.8.0", "resolve-from": "^5.0.0", - "tarn": "^3.0.1", + "tarn": "^3.0.2", "tildify": "2.0.0" }, "bin": { "knex": "bin/cli.js" }, "engines": { - "node": ">=10" + "node": ">=16" }, "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, "mysql": { "optional": true }, @@ -1238,6 +1253,9 @@ "pg": { "optional": true }, + "pg-native": { + "optional": true + }, "sqlite3": { "optional": true }, @@ -1247,9 +1265,9 @@ } }, "node_modules/knex/node_modules/debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, @@ -1590,9 +1608,9 @@ } }, "node_modules/pg-connection-string": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.4.0.tgz", - "integrity": "sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==" + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" }, "node_modules/pg-int8": { "version": "1.0.1", @@ -1630,11 +1648,6 @@ "node": ">=4" } }, - "node_modules/pg/node_modules/pg-connection-string": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", - "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" - }, "node_modules/pgpass": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz", @@ -1810,14 +1823,14 @@ } }, "node_modules/rechoir": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", - "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", "dependencies": { - "resolve": "^1.9.0" + "resolve": "^1.20.0" }, "engines": { - "node": ">= 0.10" + "node": ">= 10.13.0" } }, "node_modules/registry-auth-token": { @@ -1845,12 +1858,16 @@ } }, "node_modules/resolve": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", - "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "dependencies": { - "is-core-module": "^2.2.0", - "path-parse": "^1.0.6" + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2086,10 +2103,21 @@ "node": ">=4" } }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/tarn": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.1.tgz", - "integrity": "sha512-6usSlV9KyHsspvwu2duKH+FMUhqJnAh6J5J/4MITl8s94iSUQTLkJggdiewKv4RyARQccnigV48Z+khiuVZDJw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", "engines": { "node": ">=8.0.0" } @@ -2804,14 +2832,14 @@ "dev": true }, "colorette": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", - "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==" + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" }, "commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==" + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==" }, "concat-map": { "version": "0.0.1", @@ -3067,9 +3095,14 @@ "optional": true }, "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==" }, "get-stream": { "version": "4.1.0", @@ -3081,9 +3114,9 @@ } }, "getopts": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", - "integrity": "sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" }, "glob-parent": { "version": "5.1.2", @@ -3128,14 +3161,6 @@ "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", "dev": true }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "requires": { - "function-bind": "^1.1.1" - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -3148,6 +3173,14 @@ "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", "dev": true }, + "hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "requires": { + "function-bind": "^1.1.2" + } + }, "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", @@ -3232,11 +3265,11 @@ } }, "is-core-module": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz", - "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "requires": { - "has": "^1.0.3" + "hasown": "^2.0.0" } }, "is-extglob": { @@ -3327,29 +3360,30 @@ } }, "knex": { - "version": "0.95.6", - "resolved": "https://registry.npmjs.org/knex/-/knex-0.95.6.tgz", - "integrity": "sha512-noRcmkJl1MdicUbezrcr8OtVLcqQ/cfLIwgAx5EaxNxQOIJff88rBeyLywUScGhQNd/b78DIKKXZzLMrm6h/cw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.0.1.tgz", + "integrity": "sha512-ruASxC6xPyDklRdrcDy6a9iqK+R9cGK214aiQa+D9gX2ZnHZKv6o6JC9ZfgxILxVAul4bZ13c3tgOAHSuQ7/9g==", "requires": { - "colorette": "1.2.1", - "commander": "^7.1.0", - "debug": "4.3.1", + "colorette": "2.0.19", + "commander": "^10.0.0", + "debug": "4.3.4", "escalade": "^3.1.1", "esm": "^3.2.25", - "getopts": "2.2.5", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", "interpret": "^2.2.0", "lodash": "^4.17.21", - "pg-connection-string": "2.4.0", - "rechoir": "^0.7.0", + "pg-connection-string": "2.6.1", + "rechoir": "^0.8.0", "resolve-from": "^5.0.0", - "tarn": "^3.0.1", + "tarn": "^3.0.2", "tildify": "2.0.0" }, "dependencies": { "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } @@ -3599,19 +3633,12 @@ "pg-protocol": "^1.5.0", "pg-types": "^2.1.0", "pgpass": "1.x" - }, - "dependencies": { - "pg-connection-string": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", - "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" - } } }, "pg-connection-string": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.4.0.tgz", - "integrity": "sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==" + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.1.tgz", + "integrity": "sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg==" }, "pg-int8": { "version": "1.0.1", @@ -3771,11 +3798,11 @@ } }, "rechoir": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", - "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", "requires": { - "resolve": "^1.9.0" + "resolve": "^1.20.0" } }, "registry-auth-token": { @@ -3797,12 +3824,13 @@ } }, "resolve": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", - "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "requires": { - "is-core-module": "^2.2.0", - "path-parse": "^1.0.6" + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" } }, "resolve-from": { @@ -3988,10 +4016,15 @@ "has-flag": "^3.0.0" } }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + }, "tarn": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.1.tgz", - "integrity": "sha512-6usSlV9KyHsspvwu2duKH+FMUhqJnAh6J5J/4MITl8s94iSUQTLkJggdiewKv4RyARQccnigV48Z+khiuVZDJw==" + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==" }, "term-size": { "version": "2.2.1", diff --git a/ChhoeTaigiPgCreate/package.json b/ChhoeTaigiPgCreate/package.json index d4caa5f..44337cd 100644 --- a/ChhoeTaigiPgCreate/package.json +++ b/ChhoeTaigiPgCreate/package.json @@ -9,7 +9,7 @@ "@types/knex": "^0.16.1", "csvtojson": "^2.0.10", "express": "^4.17.1", - "knex": "^0.95.6", + "knex": "^3.0.1", "pg": "^8.6.0" }, "devDependencies": { From 76acdd493ea01b21e2b74410d6ea0590bd70f9e1 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 3 Dec 2023 23:42:58 +0800 Subject: [PATCH 4/7] feat(docker-compose.override): allow local override --- docker-compose.override.yml | 10 ++++++++++ docker-compose.yml | 8 ++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 docker-compose.override.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..cdb48f6 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,10 @@ +# version: '3.8' +# services: +# taigidb: +# volumes: +# - "./ChhoeTaigiPgCreate:/usr/src/app" +# command: tail -f +# taigiapi: +# volumes: +# - "./ChhoeTaigiPgApiServer:/usr/src/app" +# command: tail -f diff --git a/docker-compose.yml b/docker-compose.yml index 96834f4..b270f6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '3.8' services: - db: + db: container_name: pg_container image: postgres restart: always @@ -11,7 +11,7 @@ services: POSTGRES_HOST_AUTH_METHOD: "trust" ports: - "5432:5432" - pgadmin: + pgadmin: container_name: pgadmin4_container image: dpage/pgadmin4 restart: always @@ -20,12 +20,12 @@ services: PGADMIN_DEFAULT_PASSWORD: root ports: - "5050:80" - taigidb: + taigidb: depends_on: - db build: ./ChhoeTaigiPgCreate command: npm run dev - taigiapi: + taigiapi: depends_on: - taigidb build: ./ChhoeTaigiPgApiServer From 82133038f0b06761311016ef1a203779b82af267 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 4 Dec 2023 01:47:24 +0800 Subject: [PATCH 5/7] feat(package.json): add `through2` to dependencies --- ChhoeTaigiPgCreate/package-lock.json | 38 +++++++++++++++++++++++++++- ChhoeTaigiPgCreate/package.json | 4 ++- ChhoeTaigiPgCreate/tsconfig.json | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ChhoeTaigiPgCreate/package-lock.json b/ChhoeTaigiPgCreate/package-lock.json index 50542e4..208d2a8 100644 --- a/ChhoeTaigiPgCreate/package-lock.json +++ b/ChhoeTaigiPgCreate/package-lock.json @@ -14,10 +14,12 @@ "csvtojson": "^2.0.10", "express": "^4.17.1", "knex": "^3.0.1", - "pg": "^8.6.0" + "pg": "^8.6.0", + "through2": "^4.0.2" }, "devDependencies": { "@types/node": "^15.6.1", + "@types/through2": "^2.0.41", "nodemon": "^2.0.7", "ts-node": "^10.0.0", "typescript": "^4.3.2" @@ -144,6 +146,15 @@ "@types/node": "*" } }, + "node_modules/@types/through2": { + "version": "2.0.41", + "resolved": "https://registry.npmjs.org/@types/through2/-/through2-2.0.41.tgz", + "integrity": "sha512-ryQ0tidWkb1O1JuYvWKyMLYEtOWDqF5mHerJzKz/gQpoAaJq2l/dsMPBF0B5BNVT34rbARYJ5/tsZwLfUi2kwQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -2134,6 +2145,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dependencies": { + "readable-stream": "3" + } + }, "node_modules/tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", @@ -2529,6 +2548,15 @@ "@types/node": "*" } }, + "@types/through2": { + "version": "2.0.41", + "resolved": "https://registry.npmjs.org/@types/through2/-/through2-2.0.41.tgz", + "integrity": "sha512-ryQ0tidWkb1O1JuYvWKyMLYEtOWDqF5mHerJzKz/gQpoAaJq2l/dsMPBF0B5BNVT34rbARYJ5/tsZwLfUi2kwQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -4032,6 +4060,14 @@ "integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==", "dev": true }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "requires": { + "readable-stream": "3" + } + }, "tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", diff --git a/ChhoeTaigiPgCreate/package.json b/ChhoeTaigiPgCreate/package.json index 44337cd..23122a7 100644 --- a/ChhoeTaigiPgCreate/package.json +++ b/ChhoeTaigiPgCreate/package.json @@ -10,10 +10,12 @@ "csvtojson": "^2.0.10", "express": "^4.17.1", "knex": "^3.0.1", - "pg": "^8.6.0" + "pg": "^8.6.0", + "through2": "^4.0.2" }, "devDependencies": { "@types/node": "^15.6.1", + "@types/through2": "^2.0.41", "nodemon": "^2.0.7", "ts-node": "^10.0.0", "typescript": "^4.3.2" diff --git a/ChhoeTaigiPgCreate/tsconfig.json b/ChhoeTaigiPgCreate/tsconfig.json index a13b784..0e75770 100644 --- a/ChhoeTaigiPgCreate/tsconfig.json +++ b/ChhoeTaigiPgCreate/tsconfig.json @@ -4,7 +4,7 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ - "target": "es2018", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */ + "target": "es2021", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */ "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation. */ // "allowJs": true, /* Allow javascript files to be compiled. */ From 5b49b73b6d11807fa35b556c6891ee5b8d4f3d6f Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 4 Dec 2023 01:48:05 +0800 Subject: [PATCH 6/7] fix(config): HOST_LOCAL is pg_container --- ChhoeTaigiPgCreate/app/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChhoeTaigiPgCreate/app/config.ts b/ChhoeTaigiPgCreate/app/config.ts index 235feb4..1256778 100644 --- a/ChhoeTaigiPgCreate/app/config.ts +++ b/ChhoeTaigiPgCreate/app/config.ts @@ -4,7 +4,7 @@ const config = { IS_ENV_PRODUCTION: process.env.NODE_ENV == 'production', HOST: '192.168.1.222', - HOST_LOCAL: '192.168.1.222', + HOST_LOCAL: 'pg_container', PG_DEFAULT_DATABASE: 'postgres', PG_USER: 'root', From beaee539c100b24ce08eb90014764f53029dfa34 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 4 Dec 2023 01:48:27 +0800 Subject: [PATCH 7/7] feat(ChhoeTaigiPgCreate): rewrite with batch inserting --- ChhoeTaigiPgCreate/app/apiV1.ts | 129 ++++++++++++++++---------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/ChhoeTaigiPgCreate/app/apiV1.ts b/ChhoeTaigiPgCreate/app/apiV1.ts index b0ef6b9..d8a6293 100644 --- a/ChhoeTaigiPgCreate/app/apiV1.ts +++ b/ChhoeTaigiPgCreate/app/apiV1.ts @@ -1,82 +1,79 @@ 'use strict'; -import knex, { Knex } from 'knex'; -import config from './config'; +import fs from 'fs'; +import through2 from 'through2'; +import csvtojson from 'csvtojson'; import dbQuery from "./db"; -import csv from 'csvtojson' -const csvFilePath = './app/ChhoeTaigiJoined.csv' -function insertheader(data: any[]) { - let table = '' - for (let index = 0; index < data.length; index++) { - table = table + data[index] + ' text'; - if (index != data.length - 1) { - table = table + ' ,' - } - } - console.log("data:" ,table ) - let createtablestr = 'CREATE TABLE taigi ('+ table + ');' - console.log('createtablestr:',createtablestr) - dbQuery.raw(createtablestr) - .then(function (result: { rows: any; }) { - if (!config.IS_ENV_PRODUCTION) { - console.log('apiV1.createDB(): result'); - } - console.log('apiV1.createDB(): result'); - }) - .catch(function (error: any) { - if (!config.IS_ENV_PRODUCTION) { - console.log(`apiV1.createDB(): errors: ${error}.`); - } +const csvFilePath = './app/ChhoeTaigiJoined.csv' - // response.status(500); - }); -} +let isDBCreated: Promise; -async function insertdata(data: string ) { - console.log("data:" ,data) // => [["1","2","3"], ["4","5","6"], ["7","8","9"]] - await dbQuery.raw(data) - .then(function (result: { rows: any; }) { - console.log(`success`); +async function createTaigiDBFromHeader(data: any[]) { + await dbQuery.schema.dropTableIfExists('taigi'); + await dbQuery.schema.createTable('taigi', function (table) { + table.increments(); + data.map(columnName => { + table.text(columnName.toString()); }) - .catch(function (error: any) { - if (!config.IS_ENV_PRODUCTION) { - console.log(`apiV1.createDB(): errors: ${error}.`); - } + }); + await dbQuery('taigi').count('id'); + console.log(`"taigi" table is ready.`); +} - // response.status(500); - }); +async function batchInsertList(rowList: Array) { + try { + await isDBCreated; + // console.log(rowList) + const idList = await dbQuery.insert( rowList, ['id']).into('taigi'); + console.log(`Batch ${idList.length}.`); + // console.log(idList.map(({ id }) => id)); + } catch (e) { + console.error(e); + } } -var rows: any[] = [] + +let once = false; function createDB() { + const csv = csvtojson(); + csv.on('header', (headers: any) => { + isDBCreated = createTaigiDBFromHeader(headers) + }); - csv({ - noheader: false, - output: "csv" - }) - .fromFile(csvFilePath) - .on('header', (headers: any) => { insertheader(headers) }) - .on('data', (data) => { - let jsonStr = data.toString('utf8') - jsonStr = jsonStr.replace(/\"/g, '\''); - jsonStr = jsonStr.replace('[', ''); - jsonStr = jsonStr.replace(']', ''); - let datainsert = 'INSERT INTO taigi VALUES (' + jsonStr + ');' - rows.push(datainsert) - console.log("num:" , rows.length) - + fs.createReadStream(csvFilePath) + .pipe(csv) + .pipe(through2({ objectMode: true }, function transform(this: any, chunk: Buffer, enc: string, callback: any) { + this.rowList = this.rowList || []; + if (this.rowList.length < 1000) { + try { + this.rowList.push(JSON.parse(chunk.toString())) + } catch (e) { + console.error(e) + } + } else { + this.push(this.rowList); + this.rowList = []; } - ).on('done',(done)=>{ - const chunkSize = 30; - console.log("done:" , rows.length) - console.log("done:" , rows) - for (let index = 0; index < rows.length; index++) { - insertdata(rows[index]) - - } - }) -}; + callback() + }, function flush(this: any, callback: any) { + this.push(this.rowList); + callback(); + })) + .pipe(through2({ objectMode: true }, function (this: any, rowList: Array, enc: string, callback: any) { + // if (once) { + // return; + // } + // once = true; + batchInsertList(rowList).then(callback) + })) + .on('finish', async () => { + const [{ count }] = await dbQuery('taigi').count({ count: '*' }); + const lastRow = await dbQuery('taigi').where('JoinedWordID', '353511').select(); + console.log(`All ${count} rows are inserted.`); + console.log(`Last row is: `, lastRow); + }); +}; export default { createDB,