From be434c89a6e955e6ec95cb4cbd163863a7d20abf Mon Sep 17 00:00:00 2001 From: lraveri Date: Tue, 23 Dec 2025 22:29:55 +0100 Subject: [PATCH 01/14] feat: migrate to postgres --- .env.example | 10 +-- README.md | 2 +- docker-compose.yml | 15 ++--- migrations/001.do.users.sql | 18 ++++- migrations/001.undo.users.sql | 2 + migrations/002.do.tasks.sql | 10 ++- migrations/002.undo.tasks.sql | 1 + migrations/004.do.roles.sql | 2 +- migrations/005.do.user_roles.sql | 2 +- package.json | 2 +- scripts/create-database.ts | 40 ++++++++--- scripts/drop-database.ts | 35 +++++++--- scripts/migrate.ts | 28 ++++---- scripts/seed-database.ts | 81 ++++++++++++----------- src/plugins/app/tasks/tasks-repository.ts | 6 +- src/plugins/external/env.ts | 32 ++++----- src/plugins/external/knex.ts | 12 ++-- test/routes/api/tasks/tasks.test.ts | 31 +++++---- test/routes/api/users/users.test.ts | 4 +- 19 files changed, 197 insertions(+), 136 deletions(-) diff --git a/.env.example b/.env.example index 997a30fb..535addc1 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,11 @@ CAN_DROP_DATABASE=0 CAN_SEED_DATABASE=0 # Database -MYSQL_HOST=localhost -MYSQL_PORT=3306 -MYSQL_DATABASE=test_db -MYSQL_USER=test_user -MYSQL_PASSWORD=test_password +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DATABASE=test_db +POSTGRES_USER=test_user +POSTGRES_PASSWORD=test_password # Server FASTIFY_CLOSE_GRACE_DELAY=1000 diff --git a/README.md b/README.md index a11732ee..96960fd8 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ npm install ``` ### Database -You can run a MySQL instance with Docker: +You can run a PostgreSQL instance with Docker: ```bash docker compose up ``` diff --git a/docker-compose.yml b/docker-compose.yml index f014b4d5..e46842c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,19 @@ services: db: - image: mysql:8.4 + image: postgres:16 environment: - MYSQL_ROOT_PASSWORD: root_password - MYSQL_DATABASE: ${MYSQL_DATABASE} - MYSQL_USER: ${MYSQL_USER} - MYSQL_PASSWORD: ${MYSQL_PASSWORD} + POSTGRES_DB: ${POSTGRES_DATABASE} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} ports: - - 3306:3306 + - 5432:5432 healthcheck: - test: ["CMD", "mysqladmin", "ping", "-u${MYSQL_USER}", "-p${MYSQL_PASSWORD}"] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DATABASE}"] interval: 10s timeout: 5s retries: 3 volumes: - - db_data:/var/lib/mysql + - db_data:/var/lib/postgresql/data volumes: db_data: diff --git a/migrations/001.do.users.sql b/migrations/001.do.users.sql index 8142b793..2bd80bd6 100644 --- a/migrations/001.do.users.sql +++ b/migrations/001.do.users.sql @@ -1,8 +1,20 @@ CREATE TABLE users ( - id INT AUTO_INCREMENT PRIMARY KEY, + id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE NOT NULL, username VARCHAR(255) NOT NULL, password VARCHAR(255) NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP ); + +CREATE OR REPLACE FUNCTION set_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER set_users_updated_at +BEFORE UPDATE ON users +FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/migrations/001.undo.users.sql b/migrations/001.undo.users.sql index c99ddcdc..6b6c9896 100644 --- a/migrations/001.undo.users.sql +++ b/migrations/001.undo.users.sql @@ -1 +1,3 @@ +DROP TRIGGER IF EXISTS set_users_updated_at ON users; +DROP FUNCTION IF EXISTS set_updated_at; DROP TABLE IF EXISTS users; diff --git a/migrations/002.do.tasks.sql b/migrations/002.do.tasks.sql index 8dd7521d..0ae7d0a4 100644 --- a/migrations/002.do.tasks.sql +++ b/migrations/002.do.tasks.sql @@ -1,12 +1,16 @@ CREATE TABLE tasks ( - id INT AUTO_INCREMENT PRIMARY KEY, + id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, author_id INT NOT NULL, assigned_user_id INT, filename VARCHAR(255), status VARCHAR(50) NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (author_id) REFERENCES users(id), FOREIGN KEY (assigned_user_id) REFERENCES users(id) ); + +CREATE TRIGGER set_tasks_updated_at +BEFORE UPDATE ON tasks +FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/migrations/002.undo.tasks.sql b/migrations/002.undo.tasks.sql index 2ff13806..eb9f4465 100644 --- a/migrations/002.undo.tasks.sql +++ b/migrations/002.undo.tasks.sql @@ -1 +1,2 @@ +DROP TRIGGER IF EXISTS set_tasks_updated_at ON tasks; DROP TABLE IF EXISTS tasks; diff --git a/migrations/004.do.roles.sql b/migrations/004.do.roles.sql index 0dbae96b..3187f195 100644 --- a/migrations/004.do.roles.sql +++ b/migrations/004.do.roles.sql @@ -1,4 +1,4 @@ CREATE TABLE roles ( - id INT AUTO_INCREMENT PRIMARY KEY, + id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL ); diff --git a/migrations/005.do.user_roles.sql b/migrations/005.do.user_roles.sql index 1ad3d932..12a853dc 100644 --- a/migrations/005.do.user_roles.sql +++ b/migrations/005.do.user_roles.sql @@ -1,5 +1,5 @@ CREATE TABLE user_roles ( - id INT AUTO_INCREMENT PRIMARY KEY, + id SERIAL PRIMARY KEY, user_id INT NOT NULL, role_id INT NOT NULL, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, diff --git a/package.json b/package.json index 53a15229..1aa95e05 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "fastify": "^5.6.0", "fastify-plugin": "^5.0.1", "knex": "^3.1.0", - "mysql2": "^3.15.0", + "pg": "^8.16.3", "postgrator": "^8.0.0", "sanitize-filename": "^1.6.3" }, diff --git a/scripts/create-database.ts b/scripts/create-database.ts index 405452b1..10110b94 100644 --- a/scripts/create-database.ts +++ b/scripts/create-database.ts @@ -1,20 +1,27 @@ -import { createConnection, Connection } from 'mysql2/promise' +import { Client } from 'pg' if (Number(process.env.CAN_CREATE_DATABASE) !== 1) { throw new Error("You can't create the database. Set `CAN_CREATE_DATABASE=1` environment variable to allow this operation.") } async function createDatabase () { - const connection = await createConnection({ - host: process.env.MYSQL_HOST, - port: Number(process.env.MYSQL_PORT), - user: process.env.MYSQL_USER, - password: process.env.MYSQL_PASSWORD + const databaseName = process.env.POSTGRES_DATABASE + if (!databaseName) { + throw new Error('Missing `POSTGRES_DATABASE` environment variable.') + } + + const connection = new Client({ + host: process.env.POSTGRES_HOST, + port: Number(process.env.POSTGRES_PORT), + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD, + database: 'postgres' }) try { - await createDB(connection) - console.log(`Database ${process.env.MYSQL_DATABASE} has been created successfully.`) + await connection.connect() + await createDB(connection, databaseName) + console.log(`Database ${databaseName} has been created successfully.`) } catch (error) { console.error('Error creating database:', error) } finally { @@ -22,9 +29,20 @@ async function createDatabase () { } } -async function createDB (connection: Connection) { - await connection.query(`CREATE DATABASE IF NOT EXISTS \`${process.env.MYSQL_DATABASE}\``) - console.log(`Database ${process.env.MYSQL_DATABASE} created or already exists.`) +async function createDB (connection: Client, databaseName: string) { + const exists = await connection.query( + 'SELECT 1 FROM pg_database WHERE datname = $1', + [databaseName] + ) + + if (exists.rowCount > 0) { + console.log(`Database ${databaseName} already exists.`) + return + } + + const safeDbName = databaseName.replace(/"/g, '""') + await connection.query(`CREATE DATABASE "${safeDbName}"`) + console.log(`Database ${databaseName} created.`) } createDatabase() diff --git a/scripts/drop-database.ts b/scripts/drop-database.ts index ccaaee70..049a6c52 100644 --- a/scripts/drop-database.ts +++ b/scripts/drop-database.ts @@ -1,20 +1,27 @@ -import { createConnection, Connection } from 'mysql2/promise' +import { Client } from 'pg' if (Number(process.env.CAN_DROP_DATABASE) !== 1) { throw new Error("You can't drop the database. Set `CAN_DROP_DATABASE=1` environment variable to allow this operation.") } async function dropDatabase () { - const connection = await createConnection({ - host: process.env.MYSQL_HOST, - port: Number(process.env.MYSQL_PORT), - user: process.env.MYSQL_USER, - password: process.env.MYSQL_PASSWORD + const databaseName = process.env.POSTGRES_DATABASE + if (!databaseName) { + throw new Error('Missing `POSTGRES_DATABASE` environment variable.') + } + + const connection = new Client({ + host: process.env.POSTGRES_HOST, + port: Number(process.env.POSTGRES_PORT), + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD, + database: 'postgres' }) try { - await dropDB(connection) - console.log(`Database ${process.env.MYSQL_DATABASE} has been dropped successfully.`) + await connection.connect() + await dropDB(connection, databaseName) + console.log(`Database ${databaseName} has been dropped successfully.`) } catch (error) { console.error('Error dropping database:', error) } finally { @@ -22,9 +29,15 @@ async function dropDatabase () { } } -async function dropDB (connection: Connection) { - await connection.query(`DROP DATABASE IF EXISTS \`${process.env.MYSQL_DATABASE}\``) - console.log(`Database ${process.env.MYSQL_DATABASE} dropped.`) +async function dropDB (connection: Client, databaseName: string) { + await connection.query( + 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = $1 AND pid <> pg_backend_pid()', + [databaseName] + ) + + const safeDbName = databaseName.replace(/"/g, '""') + await connection.query(`DROP DATABASE IF EXISTS "${safeDbName}"`) + console.log(`Database ${databaseName} dropped.`) } dropDatabase() diff --git a/scripts/migrate.ts b/scripts/migrate.ts index 068ffea4..baf34784 100644 --- a/scripts/migrate.ts +++ b/scripts/migrate.ts @@ -1,24 +1,21 @@ -import mysql, { FieldPacket } from 'mysql2/promise' +import { Client, QueryResult } from 'pg' import path from 'node:path' import fs from 'node:fs' import Postgrator from 'postgrator' -interface PostgratorResult { - rows: any; - fields: FieldPacket[]; -} +type PostgratorResult = QueryResult async function doMigration (): Promise { - const connection = await mysql.createConnection({ - multipleStatements: true, - host: process.env.MYSQL_HOST, - port: Number(process.env.MYSQL_PORT), - database: process.env.MYSQL_DATABASE, - user: process.env.MYSQL_USER, - password: process.env.MYSQL_PASSWORD + const connection = new Client({ + host: process.env.POSTGRES_HOST, + port: Number(process.env.POSTGRES_PORT), + database: process.env.POSTGRES_DATABASE, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD }) try { + await connection.connect() const migrationDir = path.join(import.meta.dirname, '../migrations') if (!fs.existsSync(migrationDir)) { @@ -29,11 +26,10 @@ async function doMigration (): Promise { const postgrator = new Postgrator({ migrationPattern: path.join(migrationDir, '*'), - driver: 'mysql', - database: process.env.MYSQL_DATABASE, + driver: 'pg', + database: process.env.POSTGRES_DATABASE, execQuery: async (query: string): Promise => { - const [rows, fields] = await connection.query(query) - return { rows, fields } + return await connection.query(query) }, schemaTable: 'schemaversion' }) diff --git a/scripts/seed-database.ts b/scripts/seed-database.ts index 077a9338..734b8479 100644 --- a/scripts/seed-database.ts +++ b/scripts/seed-database.ts @@ -1,4 +1,4 @@ -import { createConnection, Connection } from 'mysql2/promise' +import { Client } from 'pg' import { scryptHash } from '../src/plugins/app/password-manager.js' if (Number(process.env.CAN_SEED_DATABASE) !== 1) { @@ -6,16 +6,16 @@ if (Number(process.env.CAN_SEED_DATABASE) !== 1) { } async function seed () { - const connection: Connection = await createConnection({ - multipleStatements: true, - host: process.env.MYSQL_HOST, - port: Number(process.env.MYSQL_PORT), - database: process.env.MYSQL_DATABASE, - user: process.env.MYSQL_USER, - password: process.env.MYSQL_PASSWORD + const connection = new Client({ + host: process.env.POSTGRES_HOST, + port: Number(process.env.POSTGRES_PORT), + database: process.env.POSTGRES_DATABASE, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD }) try { + await connection.connect() await truncateTables(connection) await seedUsers(connection) } catch (error) { @@ -25,28 +25,24 @@ async function seed () { } } -async function truncateTables (connection: Connection) { - const [tables]: any[] = await connection.query('SHOW TABLES') +async function truncateTables (connection: Client) { + const { rows } = await connection.query( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public'" + ) - if (tables.length > 0) { - const tableNames = tables.map( - (row: Record) => row[`Tables_in_${process.env.MYSQL_DATABASE}`] - ) - const truncateQueries = tableNames - .map((tableName: string) => `TRUNCATE TABLE \`${tableName}\``) - .join('; ') - - await connection.query('SET FOREIGN_KEY_CHECKS = 0') - try { - await connection.query(truncateQueries) - console.log('All tables have been truncated successfully.') - } finally { - await connection.query('SET FOREIGN_KEY_CHECKS = 1') - } + if (rows.length === 0) { + return } + + const tableNames = rows + .map((row) => `"${String(row.tablename).replace(/"/g, '""')}"`) + .join(', ') + + await connection.query(`TRUNCATE TABLE ${tableNames} RESTART IDENTITY CASCADE`) + console.log('All tables have been truncated successfully.') } -async function seedUsers (connection: Connection) { +async function seedUsers (connection: Client) { const users = [ { username: 'basic', email: 'basic@example.com' }, { username: 'moderator', email: 'moderator@example.com' }, @@ -59,27 +55,38 @@ async function seedUsers (connection: Connection) { const rolesAccumulator: number[] = [] for (const user of users) { - const [userResult] = await connection.execute(` + const userResult = await connection.query( + ` INSERT INTO users (username, email, password) - VALUES (?, ?, ?) - `, [user.username, user.email, hash]) + VALUES ($1, $2, $3) + RETURNING id + `, + [user.username, user.email, hash] + ) - const userId = (userResult as { insertId: number }).insertId + const userId = userResult.rows[0]?.id - const [roleResult] = await connection.execute(` + const roleResult = await connection.query( + ` INSERT INTO roles (name) - VALUES (?) - `, [user.username]) + VALUES ($1) + RETURNING id + `, + [user.username] + ) - const newRoleId = (roleResult as { insertId: number }).insertId + const newRoleId = roleResult.rows[0]?.id rolesAccumulator.push(newRoleId) for (const roleId of rolesAccumulator) { - await connection.execute(` + await connection.query( + ` INSERT INTO user_roles (user_id, role_id) - VALUES (?, ?) - `, [userId, roleId]) + VALUES ($1, $2) + `, + [userId, roleId] + ) } } diff --git a/src/plugins/app/tasks/tasks-repository.ts b/src/plugins/app/tasks/tasks-repository.ts index 89e0ce62..e7929fea 100644 --- a/src/plugins/app/tasks/tasks-repository.ts +++ b/src/plugins/app/tasks/tasks-repository.ts @@ -71,8 +71,10 @@ function createRepository (fastify: FastifyInstance) { }, async create (newTask: CreateTask) { - const [id] = await knex('tasks').insert(newTask) - return id + const [row] = await knex('tasks') + .insert(newTask) + .returning('id') + return row.id }, async update (id: number, changes: UpdateTask, trx?: Knex) { diff --git a/src/plugins/external/env.ts b/src/plugins/external/env.ts index 9ec0837b..0dffbdac 100644 --- a/src/plugins/external/env.ts +++ b/src/plugins/external/env.ts @@ -4,11 +4,11 @@ declare module 'fastify' { export interface FastifyInstance { config: { PORT: number; - MYSQL_HOST: string; - MYSQL_PORT: string; - MYSQL_USER: string; - MYSQL_PASSWORD: string; - MYSQL_DATABASE: string; + POSTGRES_HOST: string; + POSTGRES_PORT: string; + POSTGRES_USER: string; + POSTGRES_PASSWORD: string; + POSTGRES_DATABASE: string; COOKIE_SECRET: string; COOKIE_NAME: string; COOKIE_SECURED: boolean; @@ -22,32 +22,32 @@ declare module 'fastify' { const schema = { type: 'object', required: [ - 'MYSQL_HOST', - 'MYSQL_PORT', - 'MYSQL_USER', - 'MYSQL_PASSWORD', - 'MYSQL_DATABASE', + 'POSTGRES_HOST', + 'POSTGRES_PORT', + 'POSTGRES_USER', + 'POSTGRES_PASSWORD', + 'POSTGRES_DATABASE', 'COOKIE_SECRET', 'COOKIE_NAME', 'COOKIE_SECURED' ], properties: { // Database - MYSQL_HOST: { + POSTGRES_HOST: { type: 'string', default: 'localhost' }, - MYSQL_PORT: { + POSTGRES_PORT: { type: 'number', - default: 3306 + default: 5432 }, - MYSQL_USER: { + POSTGRES_USER: { type: 'string' }, - MYSQL_PASSWORD: { + POSTGRES_PASSWORD: { type: 'string' }, - MYSQL_DATABASE: { + POSTGRES_DATABASE: { type: 'string' }, diff --git a/src/plugins/external/knex.ts b/src/plugins/external/knex.ts index 3ab3533d..16a84c50 100644 --- a/src/plugins/external/knex.ts +++ b/src/plugins/external/knex.ts @@ -10,13 +10,13 @@ declare module 'fastify' { export const autoConfig = (fastify: FastifyInstance) => { return { - client: 'mysql2', + client: 'pg', connection: { - host: fastify.config.MYSQL_HOST, - user: fastify.config.MYSQL_USER, - password: fastify.config.MYSQL_PASSWORD, - database: fastify.config.MYSQL_DATABASE, - port: Number(fastify.config.MYSQL_PORT) + host: fastify.config.POSTGRES_HOST, + user: fastify.config.POSTGRES_USER, + password: fastify.config.POSTGRES_PASSWORD, + database: fastify.config.POSTGRES_DATABASE, + port: Number(fastify.config.POSTGRES_PORT) }, pool: { min: 2, max: 10 } } diff --git a/test/routes/api/tasks/tasks.test.ts b/test/routes/api/tasks/tasks.test.ts index 62cff19b..74a95677 100644 --- a/test/routes/api/tasks/tasks.test.ts +++ b/test/routes/api/tasks/tasks.test.ts @@ -19,14 +19,13 @@ async function createUser ( app: FastifyInstance, userData: Partial<{ email: string; username: string; password: string }> ) { - const [id] = await app.knex('users').insert(userData) - return id + const [row] = await app.knex('users').insert(userData).returning('id') + return row.id } async function createTask (app: FastifyInstance, taskData: Partial) { - const [id] = await app.knex('tasks').insert(taskData) - - return id + const [row] = await app.knex('tasks').insert(taskData).returning('id') + return row.id } async function uploadImageForTask ( @@ -73,24 +72,32 @@ describe('Tasks api (logged user only)', () => { firstTaskId = await createTask(app, { name: 'Task 1', author_id: userId1, - status: TaskStatusEnum.New + status: TaskStatusEnum.New, + created_at: '2025-01-01T10:00:00.000Z', + updated_at: '2025-01-01T10:00:00.000Z' }) await createTask(app, { name: 'Task 2', author_id: userId1, assigned_user_id: userId2, - status: TaskStatusEnum.InProgress + status: TaskStatusEnum.InProgress, + created_at: '2025-01-01T10:01:00.000Z', + updated_at: '2025-01-01T10:01:00.000Z' }) await createTask(app, { name: 'Task 3', author_id: userId2, - status: TaskStatusEnum.Completed + status: TaskStatusEnum.Completed, + created_at: '2025-01-01T10:02:00.000Z', + updated_at: '2025-01-01T10:02:00.000Z' }) await createTask(app, { name: 'Task 4', author_id: userId1, assigned_user_id: userId1, - status: TaskStatusEnum.OnHold + status: TaskStatusEnum.OnHold, + created_at: '2025-01-01T10:03:00.000Z', + updated_at: '2025-01-01T10:03:00.000Z' }) app.close() @@ -133,9 +140,9 @@ describe('Tasks api (logged user only)', () => { assert.strictEqual(total, 4) assert.strictEqual(tasks.length, 1) - assert.strictEqual(tasks[0].name, 'Task 2') - assert.strictEqual(tasks[0].author_id, userId1) - assert.strictEqual(tasks[0].status, TaskStatusEnum.InProgress) + assert.strictEqual(tasks[0].name, 'Task 3') + assert.strictEqual(tasks[0].author_id, userId2) + assert.strictEqual(tasks[0].status, TaskStatusEnum.Completed) }) it('should filter tasks by assigned_user_id', async (t) => { diff --git a/test/routes/api/users/users.test.ts b/test/routes/api/users/users.test.ts index ebb9f844..1e1c63d9 100644 --- a/test/routes/api/users/users.test.ts +++ b/test/routes/api/users/users.test.ts @@ -5,8 +5,8 @@ import { FastifyInstance } from 'fastify' import { scryptHash } from '../../../../src/plugins/app/password-manager.js' async function createUser (app: FastifyInstance, userData: Partial<{ username: string; email: string; password: string }>) { - const [id] = await app.knex('users').insert(userData) - return id + const [row] = await app.knex('users').insert(userData).returning('id') + return row.id } async function deleteUser (app: FastifyInstance, username: string) { From 282749a3421fbf47059af25187863e97e008066e Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 11:04:16 +0100 Subject: [PATCH 02/14] feat(db): move updated_at handling to app and remove pg triggers --- migrations/001.do.users.sql | 12 ------------ migrations/001.undo.users.sql | 2 -- migrations/002.do.tasks.sql | 4 ---- migrations/002.undo.tasks.sql | 1 - src/plugins/app/tasks/tasks-repository.ts | 4 ++-- src/plugins/app/users/users-repository.ts | 2 +- test/routes/api/tasks/tasks.test.ts | 9 ++++++++- 7 files changed, 11 insertions(+), 23 deletions(-) diff --git a/migrations/001.do.users.sql b/migrations/001.do.users.sql index 2bd80bd6..f26c27c2 100644 --- a/migrations/001.do.users.sql +++ b/migrations/001.do.users.sql @@ -6,15 +6,3 @@ CREATE TABLE users ( created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP ); - -CREATE OR REPLACE FUNCTION set_updated_at() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER set_users_updated_at -BEFORE UPDATE ON users -FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/migrations/001.undo.users.sql b/migrations/001.undo.users.sql index 6b6c9896..c99ddcdc 100644 --- a/migrations/001.undo.users.sql +++ b/migrations/001.undo.users.sql @@ -1,3 +1 @@ -DROP TRIGGER IF EXISTS set_users_updated_at ON users; -DROP FUNCTION IF EXISTS set_updated_at; DROP TABLE IF EXISTS users; diff --git a/migrations/002.do.tasks.sql b/migrations/002.do.tasks.sql index 0ae7d0a4..4a47f5b8 100644 --- a/migrations/002.do.tasks.sql +++ b/migrations/002.do.tasks.sql @@ -10,7 +10,3 @@ CREATE TABLE tasks ( FOREIGN KEY (author_id) REFERENCES users(id), FOREIGN KEY (assigned_user_id) REFERENCES users(id) ); - -CREATE TRIGGER set_tasks_updated_at -BEFORE UPDATE ON tasks -FOR EACH ROW EXECUTE FUNCTION set_updated_at(); diff --git a/migrations/002.undo.tasks.sql b/migrations/002.undo.tasks.sql index eb9f4465..2ff13806 100644 --- a/migrations/002.undo.tasks.sql +++ b/migrations/002.undo.tasks.sql @@ -1,2 +1 @@ -DROP TRIGGER IF EXISTS set_tasks_updated_at ON tasks; DROP TABLE IF EXISTS tasks; diff --git a/src/plugins/app/tasks/tasks-repository.ts b/src/plugins/app/tasks/tasks-repository.ts index e7929fea..d3e1fd5b 100644 --- a/src/plugins/app/tasks/tasks-repository.ts +++ b/src/plugins/app/tasks/tasks-repository.ts @@ -80,7 +80,7 @@ function createRepository (fastify: FastifyInstance) { async update (id: number, changes: UpdateTask, trx?: Knex) { const affectedRows = await (trx ?? knex)('tasks') .where({ id }) - .update(changes) + .update({ ...changes, updated_at: knex.fn.now() }) if (affectedRows === 0) { return null @@ -92,7 +92,7 @@ function createRepository (fastify: FastifyInstance) { async deleteFilename (filename: string, value: string | null, trx: Knex) { const affectedRows = await trx('tasks') .where({ filename }) - .update({ filename: value }) + .update({ filename: value, updated_at: knex.fn.now() }) return affectedRows > 0 }, diff --git a/src/plugins/app/users/users-repository.ts b/src/plugins/app/users/users-repository.ts index c706642b..53d78e7a 100644 --- a/src/plugins/app/users/users-repository.ts +++ b/src/plugins/app/users/users-repository.ts @@ -24,7 +24,7 @@ export function createUsersRepository (fastify: FastifyInstance) { async updatePassword (email: string, hashedPassword: string) { return knex('users') - .update({ password: hashedPassword }) + .update({ password: hashedPassword, updated_at: knex.fn.now() }) .where({ email }) }, diff --git a/test/routes/api/tasks/tasks.test.ts b/test/routes/api/tasks/tasks.test.ts index 74a95677..f9c7c9c3 100644 --- a/test/routes/api/tasks/tasks.test.ts +++ b/test/routes/api/tasks/tasks.test.ts @@ -324,10 +324,12 @@ describe('Tasks api (logged user only)', () => { it('should update an existing task', async (t) => { const app = await build(t) + const originalUpdatedAt = '2025-01-01T10:00:00.000Z' const taskData = { name: 'Task to Update', author_id: 1, - status: TaskStatusEnum.New + status: TaskStatusEnum.New, + updated_at: originalUpdatedAt } const newTaskId = await createTask(app, taskData) @@ -347,6 +349,11 @@ describe('Tasks api (logged user only)', () => { .where({ id: newTaskId }) .first() assert.equal(updatedTask?.name, updatedData.name) + assert.ok(updatedTask?.updated_at) + assert.ok( + new Date(updatedTask.updated_at as unknown as string).getTime() > + new Date(originalUpdatedAt).getTime() + ) }) it('should return 404 if task is not found for update', async (t) => { From 266469b4248355707585d89cb042fe3784690d1e Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 11:27:49 +0100 Subject: [PATCH 03/14] test(tasks): simplify updated_at assertion in update test --- test/routes/api/tasks/tasks.test.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/routes/api/tasks/tasks.test.ts b/test/routes/api/tasks/tasks.test.ts index f9c7c9c3..7dfd2349 100644 --- a/test/routes/api/tasks/tasks.test.ts +++ b/test/routes/api/tasks/tasks.test.ts @@ -324,12 +324,10 @@ describe('Tasks api (logged user only)', () => { it('should update an existing task', async (t) => { const app = await build(t) - const originalUpdatedAt = '2025-01-01T10:00:00.000Z' const taskData = { name: 'Task to Update', author_id: 1, - status: TaskStatusEnum.New, - updated_at: originalUpdatedAt + status: TaskStatusEnum.New } const newTaskId = await createTask(app, taskData) @@ -350,10 +348,6 @@ describe('Tasks api (logged user only)', () => { .first() assert.equal(updatedTask?.name, updatedData.name) assert.ok(updatedTask?.updated_at) - assert.ok( - new Date(updatedTask.updated_at as unknown as string).getTime() > - new Date(originalUpdatedAt).getTime() - ) }) it('should return 404 if task is not found for update', async (t) => { From 1f661ae9d15fdbc21af20ea4f5f38f0d965655e9 Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 11:30:22 +0100 Subject: [PATCH 04/14] ci: switch workflow db service to postgres --- .github/workflows/ci.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e07f434..b8099b14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,20 +33,19 @@ jobs: node-version: [22, 24] services: - mysql: - image: mysql:8.4 + postgres: + image: postgres:16 ports: - - 3306:3306 + - 5432:5432 env: - MYSQL_ROOT_PASSWORD: root_password - MYSQL_DATABASE: test_db - MYSQL_USER: test_user - MYSQL_PASSWORD: test_password + POSTGRES_DB: test_db + POSTGRES_USER: test_user + POSTGRES_PASSWORD: test_password options: >- - --health-cmd="mysqladmin ping -u$MYSQL_USER -p$MYSQL_PASSWORD" + --health-cmd="pg_isready -U $POSTGRES_USER -d $POSTGRES_DB" --health-interval=10s --health-timeout=5s - --health-retries=3 + --health-retries=5 steps: - uses: actions/checkout@v6 @@ -77,11 +76,11 @@ jobs: - name: Test env: - MYSQL_HOST: localhost - MYSQL_PORT: 3306 - MYSQL_DATABASE: test_db - MYSQL_USER: test_user - MYSQL_PASSWORD: test_password + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: test_db + POSTGRES_USER: test_user + POSTGRES_PASSWORD: test_password # COOKIE_SECRET is dynamically generated and loaded from the environment COOKIE_NAME: 'sessid' RATE_LIMIT_MAX: 4 From d62c6dacc4fe1aab700687e8d1513a4baaeacb2e Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 11:36:36 +0100 Subject: [PATCH 05/14] feat(db): upgrade postgres to 18 and change volume mount for version 18 compatibility --- .github/workflows/ci.yml | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8099b14..12ae875f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: services: postgres: - image: postgres:16 + image: postgres:18 ports: - 5432:5432 env: diff --git a/docker-compose.yml b/docker-compose.yml index e46842c5..e743a42d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: db: - image: postgres:16 + image: postgres:18 environment: POSTGRES_DB: ${POSTGRES_DATABASE} POSTGRES_USER: ${POSTGRES_USER} @@ -13,7 +13,7 @@ services: timeout: 5s retries: 3 volumes: - - db_data:/var/lib/postgresql/data + - db_data:/var/lib/postgresql volumes: db_data: From b2ebc8e106940d48480ab61d3020595d527449b7 Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 11:46:43 +0100 Subject: [PATCH 06/14] chore(types): update env vars to Postgres --- @types/node/environment.d.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/@types/node/environment.d.ts b/@types/node/environment.d.ts index 0069d754..687b8143 100644 --- a/@types/node/environment.d.ts +++ b/@types/node/environment.d.ts @@ -4,11 +4,11 @@ declare global { PORT: number; LOG_LEVEL: string; FASTIFY_CLOSE_GRACE_DELAY: number; - MYSQL_HOST: string - MYSQL_PORT: number - MYSQL_DATABASE: string - MYSQL_USER: string - MYSQL_PASSWORD: string + POSTGRES_HOST: string + POSTGRES_PORT: number + POSTGRES_DATABASE: string + POSTGRES_USER: string + POSTGRES_PASSWORD: string } } } From d5f0bab88b4ccdb6585ba68dad5840aea230d463 Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 17:43:05 +0100 Subject: [PATCH 07/14] fix(tasks): stream csv export with pipeline to avoid pg hangs --- src/routes/api/tasks/index.ts | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/routes/api/tasks/index.ts b/src/routes/api/tasks/index.ts index edf623d6..ea0a81bd 100644 --- a/src/routes/api/tasks/index.ts +++ b/src/routes/api/tasks/index.ts @@ -13,6 +13,8 @@ import { import path from 'node:path' import { stringify } from 'csv-stringify' import { createGzip } from 'node:zlib' +import { PassThrough } from 'node:stream' +import { pipeline } from 'node:stream/promises' const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const { tasksRepository, tasksFileManager } = fastify @@ -316,20 +318,21 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { } }, async function (request, reply) { - const queryStream = tasksRepository.createStream() - - const csvTransform = stringify({ - header: true, - columns: undefined - }) + const queryStream = await tasksRepository.createStream() + const csvTransform = stringify({ header: true }) + const gzip = createGzip() + + reply + .header('Content-Type', 'application/gzip') + .header( + 'Content-Disposition', + `attachment; filename="${encodeURIComponent('tasks.csv.gz')}"` + ) - reply.header('Content-Type', 'application/gzip') - reply.header( - 'Content-Disposition', - `attachment; filename="${encodeURIComponent('tasks.csv.gz')}"` - ) + const out = new PassThrough() + reply.send(out) - return queryStream.pipe(csvTransform).pipe(createGzip()) + await pipeline(queryStream, csvTransform, gzip, out) } ) } From 178b8f123036e73e4cf3377bb8715528ebf0e7dd Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 18:15:56 +0100 Subject: [PATCH 08/14] fix(tasks): revert csv stream to pipe chain --- src/routes/api/tasks/index.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/routes/api/tasks/index.ts b/src/routes/api/tasks/index.ts index ea0a81bd..533b8b51 100644 --- a/src/routes/api/tasks/index.ts +++ b/src/routes/api/tasks/index.ts @@ -13,8 +13,6 @@ import { import path from 'node:path' import { stringify } from 'csv-stringify' import { createGzip } from 'node:zlib' -import { PassThrough } from 'node:stream' -import { pipeline } from 'node:stream/promises' const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const { tasksRepository, tasksFileManager } = fastify @@ -319,8 +317,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { }, async function (request, reply) { const queryStream = await tasksRepository.createStream() - const csvTransform = stringify({ header: true }) - const gzip = createGzip() + const csvTransform = stringify({ header: true, columns: undefined }) reply .header('Content-Type', 'application/gzip') @@ -329,10 +326,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { `attachment; filename="${encodeURIComponent('tasks.csv.gz')}"` ) - const out = new PassThrough() - reply.send(out) - - await pipeline(queryStream, csvTransform, gzip, out) + return queryStream.pipe(csvTransform).pipe(createGzip()) } ) } From d31ce8f8f3330681ce937b694824c11faf004c84 Mon Sep 17 00:00:00 2001 From: lraveri Date: Thu, 25 Dec 2025 18:24:18 +0100 Subject: [PATCH 09/14] chore(ci): add temporary 10m job timeout --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12ae875f..679d9245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: build-and-test: permissions: contents: read + timeout-minutes: 10 runs-on: ubuntu-latest strategy: matrix: From 86a673204358ab39bed57b41e63b505475072869 Mon Sep 17 00:00:00 2001 From: lraveri Date: Wed, 31 Dec 2025 15:49:49 +0100 Subject: [PATCH 10/14] chore: add debug logs --- src/routes/api/tasks/index.ts | 17 ++++++++++++++++- test/routes/api/tasks/tasks.test.ts | 8 ++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/routes/api/tasks/index.ts b/src/routes/api/tasks/index.ts index 533b8b51..e8e30359 100644 --- a/src/routes/api/tasks/index.ts +++ b/src/routes/api/tasks/index.ts @@ -318,6 +318,21 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { async function (request, reply) { const queryStream = await tasksRepository.createStream() const csvTransform = stringify({ header: true, columns: undefined }) + const gzipStream = createGzip() + + console.log('[tasks csv] start', { requestId: request.id }) + queryStream.on('error', (err) => { + console.log('[tasks csv] queryStream error', err) + }) + csvTransform.on('error', (err) => { + console.log('[tasks csv] csvTransform error', err) + }) + gzipStream.on('error', (err) => { + console.log('[tasks csv] gzipStream error', err) + }) + gzipStream.on('finish', () => { + console.log('[tasks csv] done', { requestId: request.id }) + }) reply .header('Content-Type', 'application/gzip') @@ -326,7 +341,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { `attachment; filename="${encodeURIComponent('tasks.csv.gz')}"` ) - return queryStream.pipe(csvTransform).pipe(createGzip()) + return queryStream.pipe(csvTransform).pipe(gzipStream) } ) } diff --git a/test/routes/api/tasks/tasks.test.ts b/test/routes/api/tasks/tasks.test.ts index 7dfd2349..ad7b1767 100644 --- a/test/routes/api/tasks/tasks.test.ts +++ b/test/routes/api/tasks/tasks.test.ts @@ -866,6 +866,14 @@ describe('Tasks api (logged user only)', () => { url: '/api/tasks/download/csv' }) + console.log('[test gzipped csv] status', res.statusCode) + console.log('[test gzipped csv] content-type', res.headers['content-type']) + console.log( + '[test gzipped csv] content-disposition', + res.headers['content-disposition'] + ) + console.log('[test gzipped csv] payload-bytes', res.rawPayload.length) + assert.strictEqual(res.statusCode, 200) assert.strictEqual(res.headers['content-type'], 'application/gzip') assert.strictEqual( From 315c21b027c6d1af9a937d822748754c396d56d5 Mon Sep 17 00:00:00 2001 From: lraveri Date: Wed, 31 Dec 2025 15:56:13 +0100 Subject: [PATCH 11/14] chore: trigger ci From bcbca490bd542cd367e018fbf61dfd467a7324e5 Mon Sep 17 00:00:00 2001 From: lraveri Date: Wed, 31 Dec 2025 16:02:06 +0100 Subject: [PATCH 12/14] chore: add pg-query-stream for postgres streaming --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 1aa95e05..f1e3b5c7 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "fastify-plugin": "^5.0.1", "knex": "^3.1.0", "pg": "^8.16.3", + "pg-query-stream": "^4.10.3", "postgrator": "^8.0.0", "sanitize-filename": "^1.6.3" }, From 8793afd811b7a6b09d1dedb25323a2bcea6df8b7 Mon Sep 17 00:00:00 2001 From: lraveri Date: Wed, 31 Dec 2025 16:02:46 +0100 Subject: [PATCH 13/14] chore: removed debug logs --- src/routes/api/tasks/index.ts | 14 -------------- test/routes/api/tasks/tasks.test.ts | 8 -------- 2 files changed, 22 deletions(-) diff --git a/src/routes/api/tasks/index.ts b/src/routes/api/tasks/index.ts index e8e30359..9f615136 100644 --- a/src/routes/api/tasks/index.ts +++ b/src/routes/api/tasks/index.ts @@ -320,20 +320,6 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const csvTransform = stringify({ header: true, columns: undefined }) const gzipStream = createGzip() - console.log('[tasks csv] start', { requestId: request.id }) - queryStream.on('error', (err) => { - console.log('[tasks csv] queryStream error', err) - }) - csvTransform.on('error', (err) => { - console.log('[tasks csv] csvTransform error', err) - }) - gzipStream.on('error', (err) => { - console.log('[tasks csv] gzipStream error', err) - }) - gzipStream.on('finish', () => { - console.log('[tasks csv] done', { requestId: request.id }) - }) - reply .header('Content-Type', 'application/gzip') .header( diff --git a/test/routes/api/tasks/tasks.test.ts b/test/routes/api/tasks/tasks.test.ts index ad7b1767..7dfd2349 100644 --- a/test/routes/api/tasks/tasks.test.ts +++ b/test/routes/api/tasks/tasks.test.ts @@ -866,14 +866,6 @@ describe('Tasks api (logged user only)', () => { url: '/api/tasks/download/csv' }) - console.log('[test gzipped csv] status', res.statusCode) - console.log('[test gzipped csv] content-type', res.headers['content-type']) - console.log( - '[test gzipped csv] content-disposition', - res.headers['content-disposition'] - ) - console.log('[test gzipped csv] payload-bytes', res.rawPayload.length) - assert.strictEqual(res.statusCode, 200) assert.strictEqual(res.headers['content-type'], 'application/gzip') assert.strictEqual( From b93cca230fb2ad45d782ce186f9c106776f6aeaa Mon Sep 17 00:00:00 2001 From: lraveri Date: Wed, 31 Dec 2025 16:07:14 +0100 Subject: [PATCH 14/14] chore: remove ci job timeout --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 679d9245..12ae875f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: build-and-test: permissions: contents: read - timeout-minutes: 10 runs-on: ubuntu-latest strategy: matrix: