From db2a016900dadc4ccf3f944ffe70074e4f9a7a59 Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:12:48 +0200 Subject: [PATCH 01/12] Small improvements to the server setup docs (#66) * Small improvements to the server setup docs * PR feedback --- server/.env.example | 12 ++++++++---- server/README.md | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/.env.example b/server/.env.example index 147bcb6..3c179d8 100644 --- a/server/.env.example +++ b/server/.env.example @@ -1,8 +1,12 @@ -DATABASE_URL=postgresql://... # Point to your local PG installation +DATABASE_URL=postgresql://postgres:pass@localhost:5432/postgres # Point to your local PG installation +# format: postgresql://:@:/ + EXTRACTION_FILES_PATH=/path/to/local/folder # Create a local folder to store these -ENCRYPTION_KEY= # Generate a random long string (UUID) -COOKIE_KEY= # Generate a random long string (UUID) -CLIENT_PATH=/path/to/local/client/dist/folder # In the client project there will be a dist folder when you compile, add that here +ENCRYPTION_KEY= # Generate a random 32 character string (UUID) +COOKIE_KEY= # Generate a hex key with `pnpm exec secure-session | xxd -p -c 0` +CLIENT_PATH= # In the client project there will be a dist folder when you compile, add that here +# you can run `realpath ../client/dist` after client build to get the exact value needed + FRONTEND_URL=http://localhost:5173 #SMTP_USER= You can leave blank for development #SMTP_PASSWORD= You can leave blank for development diff --git a/server/README.md b/server/README.md index c372954..5822fb9 100644 --- a/server/README.md +++ b/server/README.md @@ -12,16 +12,21 @@ for the application. - **Web scraping**: Puppeteer - **Email**: React Email for templating and Nodemailer for sending - **Database**: PostgreSQL with Drizzle ORM +- **Cache**: Redis - **Error monitoring**: Airbrake ## Setup -1. Install [node.js](https://nodejs.org/en) (20+) and [pnpm](https://pnpm.io/) +1. Install [node.js](https://nodejs.org/en) (20+) (using [nvm](https://github.com/nvm-sh/nvm) is a plus) and [pnpm](https://pnpm.io/) 2. Install dependencies: ```bash pnpm install +pnpm dlx puppeteer browsers install + +# or specific version +pnpm dlx puppeteer browsers install chrome@130.0.6723.58 ``` 3. Set up environment variables: @@ -30,6 +35,12 @@ pnpm install cp .env.example .env # edit your env vars in .env ``` +Generate keys required for [secure-session](https://github.com/fastify/fastify-secure-session) encryption (32 bytes - 64 hex encoded characters) +```bash +pnpm exec secure-session | xxd -p -c 0 +``` +Use generated key in the `COOKIE_KEY` env variable. + ## Development Run the development server: @@ -50,6 +61,12 @@ Run the email preview server: pnpm run dev:email ``` +Docker: +```bash +docker run --env=POSTGRES_PASSWORD=pass -p 5432:5432 -d postgres:13-alpine +docker run -p 6379:6379 -d redis +``` + ## Database The application uses PostgreSQL with Drizzle ORM for database management. @@ -66,6 +83,8 @@ Apply migrations: pnpm run db:migrate ``` +To seed a development user, use `src/createUser.ts`: + ## Testing Run tests: From bd8b929a9683509541a73742746c97551111d646 Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:12:48 +0200 Subject: [PATCH 02/12] Add docker-compose file to help start the service faster --- docker-compose.yml | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d3df73d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + + ctdl-xtra: + build: . + container_name: ctdl-xtra + depends_on: + - postgres + - redis + environment: + DATABASE_URL: postgres://xtra:xtra@postgres:5432/cdtlxtra + REDIS_URL: redis://redis:6379 + ports: + - "3000:3000" + volumes: + - .:/app + ctdl-xtra-worker: + build: + context: . + dockerfile: worker.Dockerfile + container_name: ctdl-xtra-worker + depends_on: + - postgres + - redis + environment: + DATABASE_URL: postgres://xtra:xtra@postgres:5432/xtra + REDIS_URL: redis://redis:6379 + volumes: + - .:/app + + postgres: + image: postgres:latest + container_name: my_postgres + restart: always + environment: + POSTGRES_USER: xtra + POSTGRES_PASSWORD: xtra + POSTGRES_DB: xtra + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + + redis: + image: redis:latest + container_name: my_redis + restart: always + ports: + - "6379:6379" + volumes: + - redis_data:/data + +volumes: + pg_data: + redis_data: From 348e375c520280802a37590f12c09c3af9142ead Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:55:41 +0200 Subject: [PATCH 03/12] Add support for user roles and organizations to the DB --- server/migrations/0002_sudden_sprite.sql | 36 + server/migrations/meta/0002_snapshot.json | 1181 +++++++++++++++++++++ server/migrations/meta/_journal.json | 7 + server/src/data/schema.ts | 32 + 4 files changed, 1256 insertions(+) create mode 100644 server/migrations/0002_sudden_sprite.sql create mode 100644 server/migrations/meta/0002_snapshot.json diff --git a/server/migrations/0002_sudden_sprite.sql b/server/migrations/0002_sudden_sprite.sql new file mode 100644 index 0000000..f7831c6 --- /dev/null +++ b/server/migrations/0002_sudden_sprite.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS "orgs" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text DEFAULT 'null', + "description" text DEFAULT 'null', + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "orgs_users_roles" ( + "org_id" integer NOT NULL, + "user_id" integer NOT NULL +); +--> statement-breakpoint +ALTER TABLE "settings" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_user_id_user_roles_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user_roles"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "settings" ADD CONSTRAINT "settings_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/server/migrations/meta/0002_snapshot.json b/server/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000..ccaeb52 --- /dev/null +++ b/server/migrations/meta/0002_snapshot.json @@ -0,0 +1,1181 @@ +{ + "id": "7ca450cc-9206-4830-99b7-d38471fbb9bc", + "prevId": "a30ef025-2510-4069-95cd-7e87c7728d5d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.catalogues": { + "name": "catalogues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "catalogues_url_unique": { + "name": "catalogues_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + } + }, + "public.crawl_pages": { + "name": "crawl_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_step_id": { + "name": "crawl_step_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "page_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "screenshot": { + "name": "screenshot", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetch_failure_reason": { + "name": "fetch_failure_reason", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "data_type": { + "name": "data_type", + "type": "page_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "data_extraction_started_at": { + "name": "data_extraction_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_pages_extraction_idx": { + "name": "crawl_pages_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_status_idx": { + "name": "crawl_pages_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_data_type_idx": { + "name": "crawl_pages_data_type_idx", + "columns": [ + { + "expression": "data_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_step_idx": { + "name": "crawl_pages_step_idx", + "columns": [ + { + "expression": "crawl_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_pages_extraction_id_extractions_id_fk": { + "name": "crawl_pages_extraction_id_extractions_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_pages_crawl_step_id_crawl_steps_id_fk": { + "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "crawl_steps", + "columnsFrom": [ + "crawl_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "crawl_pages_extraction_id_url_unique": { + "name": "crawl_pages_extraction_id_url_unique", + "nullsNotDistinct": false, + "columns": [ + "extraction_id", + "url" + ] + } + } + }, + "public.crawl_steps": { + "name": "crawl_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "step": { + "name": "step", + "type": "step", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_step_id": { + "name": "parent_step_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_steps_extraction_idx": { + "name": "crawl_steps_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_steps_parent_step_idx": { + "name": "crawl_steps_parent_step_idx", + "columns": [ + { + "expression": "parent_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_steps_extraction_id_extractions_id_fk": { + "name": "crawl_steps_extraction_id_extractions_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_steps_parent_step_id_crawl_steps_id_fk": { + "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "crawl_steps", + "columnsFrom": [ + "parent_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.data_items": { + "name": "data_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_page_id": { + "name": "crawl_page_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "structured_data": { + "name": "structured_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "text_inclusion": { + "name": "text_inclusion", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_items_dataset_idx": { + "name": "data_items_dataset_idx", + "columns": [ + { + "expression": "dataset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_items_crawl_page_idx": { + "name": "data_items_crawl_page_idx", + "columns": [ + { + "expression": "crawl_page_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_items_dataset_id_datasets_id_fk": { + "name": "data_items_dataset_id_datasets_id_fk", + "tableFrom": "data_items", + "tableTo": "datasets", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_items_crawl_page_id_crawl_pages_id_fk": { + "name": "data_items_crawl_page_id_crawl_pages_id_fk", + "tableFrom": "data_items", + "tableTo": "crawl_pages", + "columnsFrom": [ + "crawl_page_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.datasets": { + "name": "datasets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_catalogue_idx": { + "name": "datasets_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_extraction_idx": { + "name": "datasets_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_catalogue_id_catalogues_id_fk": { + "name": "datasets_catalogue_id_catalogues_id_fk", + "tableFrom": "datasets", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_extraction_id_extractions_id_fk": { + "name": "datasets_extraction_id_extractions_id_fk", + "tableFrom": "datasets", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "datasets_catalogue_id_extraction_id_unique": { + "name": "datasets_catalogue_id_extraction_id_unique", + "nullsNotDistinct": false, + "columns": [ + "catalogue_id", + "extraction_id" + ] + } + } + }, + "public.extraction_logs": { + "name": "extraction_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "log": { + "name": "log", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "log_level": { + "name": "log_level", + "type": "log_level", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'INFO'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extraction_logs_extraction_idx": { + "name": "extraction_logs_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extraction_logs_extraction_id_extractions_id_fk": { + "name": "extraction_logs_extraction_id_extractions_id_fk", + "tableFrom": "extraction_logs", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.extractions": { + "name": "extractions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "completion_stats": { + "name": "completion_stats", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "extraction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extractions_recipe_idx": { + "name": "extractions_recipe_idx", + "columns": [ + { + "expression": "recipe_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extractions_recipe_id_recipes_id_fk": { + "name": "extractions_recipe_id_recipes_id_fk", + "tableFrom": "extractions", + "tableTo": "recipes", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.model_api_calls": { + "name": "model_api_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "provider_model", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "call_site": { + "name": "call_site", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_count": { + "name": "input_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "output_token_count": { + "name": "output_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "model_api_calls_extraction_idx": { + "name": "model_api_calls_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "model_api_calls_extraction_id_extractions_id_fk": { + "name": "model_api_calls_extraction_id_extractions_id_fk", + "tableFrom": "model_api_calls", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.orgs_users_roles": { + "name": "orgs_users_roles", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "orgs_users_roles_org_id_orgs_id_fk": { + "name": "orgs_users_roles_org_id_orgs_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_users_id_fk": { + "name": "orgs_users_roles_user_id_users_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_user_roles_id_fk": { + "name": "orgs_users_roles_user_id_user_roles_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "user_roles", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.recipes": { + "name": "recipes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "detection_failure_reason": { + "name": "detection_failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "recipe_detection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + } + }, + "indexes": { + "recipes_catalogue_idx": { + "name": "recipes_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "recipes_catalogue_id_catalogues_id_fk": { + "name": "recipes_catalogue_id_catalogues_id_fk", + "tableFrom": "recipes", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_encrypted": { + "name": "is_encrypted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "encrypted_preview": { + "name": "encrypted_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "settings_org_id_orgs_id_fk": { + "name": "settings_org_id_orgs_id_fk", + "tableFrom": "settings", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_key_unique": { + "name": "settings_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + } + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + } + }, + "enums": { + "public.extraction_status": { + "name": "extraction_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "COMPLETE", + "STALE", + "CANCELLED" + ] + }, + "public.log_level": { + "name": "log_level", + "schema": "public", + "values": [ + "INFO", + "ERROR" + ] + }, + "public.page_status": { + "name": "page_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.page_type": { + "name": "page_type", + "schema": "public", + "values": [ + "COURSE_DETAIL_PAGE", + "CATEGORY_LINKS_PAGE", + "COURSE_LINKS_PAGE" + ] + }, + "public.provider": { + "name": "provider", + "schema": "public", + "values": [ + "openai" + ] + }, + "public.provider_model": { + "name": "provider_model", + "schema": "public", + "values": [ + "gpt-4o" + ] + }, + "public.recipe_detection_status": { + "name": "recipe_detection_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.step": { + "name": "step", + "schema": "public", + "values": [ + "FETCH_ROOT", + "FETCH_PAGINATED", + "FETCH_LINKS" + ] + }, + "public.url_pattern_type": { + "name": "url_pattern_type", + "schema": "public", + "values": [ + "page_num", + "offset" + ] + } + }, + "schemas": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/migrations/meta/_journal.json b/server/migrations/meta/_journal.json index a879964..8a62c06 100644 --- a/server/migrations/meta/_journal.json +++ b/server/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1727814665852, "tag": "0001_normal_beast", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1738950572242, + "tag": "0002_sudden_sprite", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/src/data/schema.ts b/server/src/data/schema.ts index cf78be6..3ef4962 100644 --- a/server/src/data/schema.ts +++ b/server/src/data/schema.ts @@ -468,6 +468,9 @@ const settings = pgTable("settings", { isEncrypted: boolean("is_encrypted").default(false).notNull(), encryptedPreview: text("encrypted_preview"), createdAt: timestamp("created_at").notNull().defaultNow(), + orgId: integer("org_id") + .notNull() + .references(() => orgs.id), }); const users = pgTable("users", { @@ -478,6 +481,33 @@ const users = pgTable("users", { createdAt: timestamp("created_at").notNull().defaultNow(), }); +const orgs = pgTable("orgs", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + logo: text("name").default("null"), + description: text("description").default("null"), + createdAt: timestamp("created_at").notNull().defaultNow(), +}); + +const userRoles = pgTable("user_roles", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + description: text("description").default("null"), + createdAt: timestamp("created_at").notNull().defaultNow(), +}); + +const orgsUsers = pgTable("orgs_users_roles", { + orgId: integer("org_id") + .notNull() + .references(() => orgs.id), + userId: integer("user_id") + .notNull() + .references(() => users.id), + roleId: integer("user_id") + .notNull() + .references(() => userRoles.id), +}) + export function encryptForDb(text: string) { const IV = randomBytes(16); let cipher = createCipheriv("aes-256-cbc", Buffer.from(ENCRYPTION_KEY), IV); @@ -520,4 +550,6 @@ export { recipesRelations, settings, users, + orgs, + orgsUsers, }; From e47945fad2bd2233a5a10f4d1f74ddf60f90e1bf Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Fri, 7 Feb 2025 14:22:11 +0200 Subject: [PATCH 04/12] Update drizzle to latest to be able to use seed features --- server/package.json | 4 +-- server/pnpm-lock.yaml | 77 +++++++++++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/server/package.json b/server/package.json index 2e3dbcd..4336b8e 100644 --- a/server/package.json +++ b/server/package.json @@ -21,7 +21,7 @@ "bullmq": "^5.10.0", "cheerio": "1.0.0-rc.12", "dotenv": "^16.4.5", - "drizzle-orm": "^0.31.2", + "drizzle-orm": "^0.39.2", "fast-csv": "^5.0.1", "fast-equals": "^5.0.1", "fastify": "^4.26.2", @@ -47,7 +47,7 @@ "@types/react": "^18.3.3", "@types/turndown": "^5.0.4", "@vitest/ui": "^2.1.4", - "drizzle-kit": "^0.22.7", + "drizzle-kit": "^0.30.4", "nodemon": "^3.1.0", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 0526c67..59a1c7d 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -42,8 +42,8 @@ importers: specifier: ^16.4.5 version: 16.4.5 drizzle-orm: - specifier: ^0.31.2 - version: 0.31.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + specifier: ^0.39.2 + version: 0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) fast-csv: specifier: ^5.0.1 version: 5.0.1 @@ -115,8 +115,8 @@ importers: specifier: ^2.1.4 version: 2.1.4(vitest@2.1.4) drizzle-kit: - specifier: ^0.22.7 - version: 0.22.7 + specifier: ^0.30.4 + version: 0.30.4 nodemon: specifier: ^3.1.0 version: 3.1.0 @@ -128,7 +128,7 @@ importers: version: 18.3.1(react@18.3.1) react-email: specifier: ^2.1.6 - version: 2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + version: 2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) tsx: specifier: ^4.16.2 version: 4.16.2 @@ -268,6 +268,9 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} + '@drizzle-team/brocli@0.10.2': + resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@emotion/is-prop-valid@0.8.8': resolution: {integrity: sha512-u5WtneEAr5IDG2Wv65yhunPSMLIpuKsbuOktRojfrEiEvRyC85LgPMZI63cr7NUqT8ZIGdSVg8ZKGxIug4lXcA==} @@ -276,9 +279,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild/aix-ppc64@0.19.11': resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} @@ -2428,21 +2433,23 @@ packages: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} - drizzle-kit@0.22.7: - resolution: {integrity: sha512-9THPCb2l1GPt7wxhws9LvTR0YG565ZlVgTuqGMwjs590Kch1pXu4GyjEArVijSF5m0OBj3qgdeKmuJXhKXgWFw==} + drizzle-kit@0.30.4: + resolution: {integrity: sha512-B2oJN5UkvwwNHscPWXDG5KqAixu7AUzZ3qbe++KU9SsQ+cZWR4DXEPYcvWplyFAno0dhRJECNEhNxiDmFaPGyQ==} hasBin: true - drizzle-orm@0.31.2: - resolution: {integrity: sha512-QnenevbnnAzmbNzQwbhklvIYrDE8YER8K7kSrAWQSV1YvFCdSQPzj+jzqRdTSsV2cDqSpQ0NXGyL1G9I43LDLg==} + drizzle-orm@0.39.2: + resolution: {integrity: sha512-cuopo+udkKEGGpSxCML9ZRQ43R01zYCTsbqCrb9kJkabx1QEwFlPIFoQfx6E6tuawtiX930Gwyrkwj4inlpzDg==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' - '@cloudflare/workers-types': '>=3' - '@electric-sql/pglite': '>=0.1.1' - '@libsql/client': '*' - '@neondatabase/serverless': '>=0.1' + '@cloudflare/workers-types': '>=4' + '@electric-sql/pglite': '>=0.2.0' + '@libsql/client': '>=0.10.0' + '@libsql/client-wasm': '>=0.10.0' + '@neondatabase/serverless': '>=0.10.0' '@op-engineering/op-sqlite': '>=2' '@opentelemetry/api': ^1.4.1 '@planetscale/database': '>=1' + '@prisma/client': '*' '@tidbcloud/serverless': '*' '@types/better-sqlite3': '*' '@types/pg': '*' @@ -2452,12 +2459,13 @@ packages: '@xata.io/client': '*' better-sqlite3: '>=7' bun-types: '*' - expo-sqlite: '>=13.2.0' + expo-sqlite: '>=14.0.0' knex: '*' kysely: '*' mysql2: '>=2' pg: '>=8' postgres: '>=3' + prisma: '*' react: '>=18' sql.js: '>=1' sqlite3: '>=5' @@ -2470,6 +2478,8 @@ packages: optional: true '@libsql/client': optional: true + '@libsql/client-wasm': + optional: true '@neondatabase/serverless': optional: true '@op-engineering/op-sqlite': @@ -2478,6 +2488,8 @@ packages: optional: true '@planetscale/database': optional: true + '@prisma/client': + optional: true '@tidbcloud/serverless': optional: true '@types/better-sqlite3': @@ -2508,6 +2520,8 @@ packages: optional: true postgres: optional: true + prisma: + optional: true react: optional: true sql.js: @@ -4879,6 +4893,8 @@ snapshots: '@jridgewell/trace-mapping': 0.3.9 optional: true + '@drizzle-team/brocli@0.10.2': {} + '@emotion/is-prop-valid@0.8.8': dependencies: '@emotion/memoize': 0.7.4 @@ -6074,7 +6090,7 @@ snapshots: dependencies: '@types/node': 20.12.7 tapable: 2.2.1 - webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11) + webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12) transitivePeerDependencies: - '@swc/core' - esbuild @@ -6752,15 +6768,16 @@ snapshots: dotenv@16.4.5: {} - drizzle-kit@0.22.7: + drizzle-kit@0.30.4: dependencies: + '@drizzle-team/brocli': 0.10.2 '@esbuild-kit/esm-loader': 2.6.5 esbuild: 0.19.12 esbuild-register: 3.5.0(esbuild@0.19.12) transitivePeerDependencies: - supports-color - drizzle-orm@0.31.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1): + drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1): optionalDependencies: '@types/better-sqlite3': 7.6.9 '@types/pg': 8.11.9 @@ -6840,7 +6857,7 @@ snapshots: esbuild-register@3.5.0(esbuild@0.19.12): dependencies: - debug: 4.3.5 + debug: 4.3.7 esbuild: 0.19.12 transitivePeerDependencies: - supports-color @@ -8044,13 +8061,13 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.4.38 - postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: lilconfig: 3.1.2 yaml: 2.5.0 optionalDependencies: postcss: 8.4.38 - ts-node: 10.9.2(@types/node@20.12.7)(typescript@5.4.5) + ts-node: 10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5) postcss-nested@6.2.0(postcss@8.4.38): dependencies: @@ -8275,7 +8292,7 @@ snapshots: react: 18.3.1 scheduler: 0.23.2 - react-email@2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + react-email@2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: '@babel/core': 7.24.5 '@babel/parser': 7.24.5 @@ -8315,7 +8332,7 @@ snapshots: source-map-js: 1.0.2 stacktrace-parser: 0.1.10 tailwind-merge: 2.2.0 - tailwindcss: 3.4.0(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + tailwindcss: 3.4.0(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) typescript: 5.1.6 transitivePeerDependencies: - '@opentelemetry/api' @@ -8731,7 +8748,7 @@ snapshots: dependencies: '@babel/runtime': 7.24.8 - tailwindcss@3.4.0(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + tailwindcss@3.4.0(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -8750,7 +8767,7 @@ snapshots: postcss: 8.4.38 postcss-import: 15.1.0(postcss@8.4.38) postcss-js: 4.0.1(postcss@8.4.38) - postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) postcss-nested: 6.2.0(postcss@8.4.38) postcss-selector-parser: 6.1.1 resolve: 1.22.8 @@ -8801,14 +8818,14 @@ snapshots: dependencies: bintrees: 1.0.2 - terser-webpack-plugin@5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(esbuild@0.19.12)): + terser-webpack-plugin@5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.31.3 - webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11) + webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12) optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.2) esbuild: 0.19.11 @@ -8871,7 +8888,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5): + ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 @@ -8888,6 +8905,8 @@ snapshots: typescript: 5.4.5 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.2) optional: true tslib@2.6.2: {} @@ -9046,7 +9065,7 @@ snapshots: webpack-sources@3.2.3: {} - webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11): + webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12): dependencies: '@types/eslint-scope': 3.7.7 '@types/estree': 1.0.5 @@ -9069,7 +9088,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(esbuild@0.19.12)) + terser-webpack-plugin: 5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)) watchpack: 2.4.1 webpack-sources: 3.2.3 transitivePeerDependencies: From 0e57141dd3d9c8b4b33afb095d2098bfcc8310e7 Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Fri, 7 Feb 2025 20:36:26 +0200 Subject: [PATCH 05/12] WIP --- client/src/components/app/dashboard.tsx | 5 + client/src/components/app/routes.tsx | 7 + client/src/components/app/users/create.tsx | 4 + server/migrations/meta/0003_snapshot.json | 1257 ++++++++++++++++++++ server/migrations/meta/0004_snapshot.json | 1244 +++++++++++++++++++ server/migrations/meta/_journal.json | 14 + server/package.json | 1 + server/pnpm-lock.yaml | 22 + server/src/data/catalogues.ts | 17 +- server/src/data/orgs.ts | 9 + server/src/data/schema.ts | 4 + server/src/data/settings.ts | 12 +- server/src/fastifySessionAuth.ts | 2 + server/src/routers/catalogues.ts | 6 +- server/src/routers/settings.ts | 6 +- 15 files changed, 2597 insertions(+), 13 deletions(-) create mode 100644 server/migrations/meta/0003_snapshot.json create mode 100644 server/migrations/meta/0004_snapshot.json create mode 100644 server/src/data/orgs.ts diff --git a/client/src/components/app/dashboard.tsx b/client/src/components/app/dashboard.tsx index 4fd65b8..5948e59 100644 --- a/client/src/components/app/dashboard.tsx +++ b/client/src/components/app/dashboard.tsx @@ -25,6 +25,11 @@ import Routes from "./routes"; export function Dashboard() { const [location] = useLocation(); + // IF the user is part of more than one organization or the user isStaff + // We will adjust this and any other component that has the header + // besides the user icon we will add the organization selector + + return (
diff --git a/client/src/components/app/routes.tsx b/client/src/components/app/routes.tsx index 68422f5..1bd6154 100644 --- a/client/src/components/app/routes.tsx +++ b/client/src/components/app/routes.tsx @@ -22,6 +22,13 @@ import ResetUserPassword from "./users/reset-password"; import Welcome from "./welcome"; export default function Routes() { + // We will add a new route called /select-org that will be the thing a user sees + // if their session does not contain an org ID. So immediately after login. + + // We will create a new organization list selector component in this folder. + // We will ensure backend sets a correct orgId if the user is part of a single + // org as to not bother the user selecting their only organization. + return ( diff --git a/client/src/components/app/users/create.tsx b/client/src/components/app/users/create.tsx index 5eb3fff..9fb7471 100644 --- a/client/src/components/app/users/create.tsx +++ b/client/src/components/app/users/create.tsx @@ -72,6 +72,10 @@ export default function CreateUser() { } } + // Update component with organization list selection + // for the new user inferred from the current user + // org+roles + return ( <>

Create user

diff --git a/server/migrations/meta/0003_snapshot.json b/server/migrations/meta/0003_snapshot.json new file mode 100644 index 0000000..21ef40a --- /dev/null +++ b/server/migrations/meta/0003_snapshot.json @@ -0,0 +1,1257 @@ +{ + "id": "66d696cc-e665-44f2-b7bb-903a35f2a4f2", + "prevId": "7ca450cc-9206-4830-99b7-d38471fbb9bc", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.catalogues": { + "name": "catalogues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "catalogues_recipe_id_orgs_id_fk": { + "name": "catalogues_recipe_id_orgs_id_fk", + "tableFrom": "catalogues", + "tableTo": "orgs", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "catalogues_url_unique": { + "name": "catalogues_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_pages": { + "name": "crawl_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_step_id": { + "name": "crawl_step_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "page_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "screenshot": { + "name": "screenshot", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetch_failure_reason": { + "name": "fetch_failure_reason", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "data_type": { + "name": "data_type", + "type": "page_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "data_extraction_started_at": { + "name": "data_extraction_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_pages_extraction_idx": { + "name": "crawl_pages_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_status_idx": { + "name": "crawl_pages_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_data_type_idx": { + "name": "crawl_pages_data_type_idx", + "columns": [ + { + "expression": "data_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_step_idx": { + "name": "crawl_pages_step_idx", + "columns": [ + { + "expression": "crawl_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_pages_extraction_id_extractions_id_fk": { + "name": "crawl_pages_extraction_id_extractions_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_pages_crawl_step_id_crawl_steps_id_fk": { + "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "crawl_steps", + "columnsFrom": [ + "crawl_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "crawl_pages_extraction_id_url_unique": { + "name": "crawl_pages_extraction_id_url_unique", + "nullsNotDistinct": false, + "columns": [ + "extraction_id", + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_steps": { + "name": "crawl_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "step": { + "name": "step", + "type": "step", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_step_id": { + "name": "parent_step_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_steps_extraction_idx": { + "name": "crawl_steps_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_steps_parent_step_idx": { + "name": "crawl_steps_parent_step_idx", + "columns": [ + { + "expression": "parent_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_steps_extraction_id_extractions_id_fk": { + "name": "crawl_steps_extraction_id_extractions_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_steps_parent_step_id_crawl_steps_id_fk": { + "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "crawl_steps", + "columnsFrom": [ + "parent_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_items": { + "name": "data_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_page_id": { + "name": "crawl_page_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "structured_data": { + "name": "structured_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "text_inclusion": { + "name": "text_inclusion", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_items_dataset_idx": { + "name": "data_items_dataset_idx", + "columns": [ + { + "expression": "dataset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_items_crawl_page_idx": { + "name": "data_items_crawl_page_idx", + "columns": [ + { + "expression": "crawl_page_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_items_dataset_id_datasets_id_fk": { + "name": "data_items_dataset_id_datasets_id_fk", + "tableFrom": "data_items", + "tableTo": "datasets", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_items_crawl_page_id_crawl_pages_id_fk": { + "name": "data_items_crawl_page_id_crawl_pages_id_fk", + "tableFrom": "data_items", + "tableTo": "crawl_pages", + "columnsFrom": [ + "crawl_page_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.datasets": { + "name": "datasets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_catalogue_idx": { + "name": "datasets_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_extraction_idx": { + "name": "datasets_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_catalogue_id_catalogues_id_fk": { + "name": "datasets_catalogue_id_catalogues_id_fk", + "tableFrom": "datasets", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_extraction_id_extractions_id_fk": { + "name": "datasets_extraction_id_extractions_id_fk", + "tableFrom": "datasets", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "datasets_catalogue_id_extraction_id_unique": { + "name": "datasets_catalogue_id_extraction_id_unique", + "nullsNotDistinct": false, + "columns": [ + "catalogue_id", + "extraction_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extraction_logs": { + "name": "extraction_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "log": { + "name": "log", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "log_level": { + "name": "log_level", + "type": "log_level", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'INFO'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extraction_logs_extraction_idx": { + "name": "extraction_logs_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extraction_logs_extraction_id_extractions_id_fk": { + "name": "extraction_logs_extraction_id_extractions_id_fk", + "tableFrom": "extraction_logs", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extractions": { + "name": "extractions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "completion_stats": { + "name": "completion_stats", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "extraction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extractions_recipe_idx": { + "name": "extractions_recipe_idx", + "columns": [ + { + "expression": "recipe_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extractions_recipe_id_recipes_id_fk": { + "name": "extractions_recipe_id_recipes_id_fk", + "tableFrom": "extractions", + "tableTo": "recipes", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "extractions_recipe_id_orgs_id_fk": { + "name": "extractions_recipe_id_orgs_id_fk", + "tableFrom": "extractions", + "tableTo": "orgs", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_api_calls": { + "name": "model_api_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "provider_model", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "call_site": { + "name": "call_site", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_count": { + "name": "input_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "output_token_count": { + "name": "output_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "model_api_calls_extraction_idx": { + "name": "model_api_calls_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "model_api_calls_extraction_id_extractions_id_fk": { + "name": "model_api_calls_extraction_id_extractions_id_fk", + "tableFrom": "model_api_calls", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs_users_roles": { + "name": "orgs_users_roles", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "orgs_users_roles_org_id_orgs_id_fk": { + "name": "orgs_users_roles_org_id_orgs_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_users_id_fk": { + "name": "orgs_users_roles_user_id_users_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_user_roles_id_fk": { + "name": "orgs_users_roles_user_id_user_roles_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "user_roles", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.recipes": { + "name": "recipes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "detection_failure_reason": { + "name": "detection_failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "recipe_detection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + } + }, + "indexes": { + "recipes_catalogue_idx": { + "name": "recipes_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "recipes_catalogue_id_catalogues_id_fk": { + "name": "recipes_catalogue_id_catalogues_id_fk", + "tableFrom": "recipes", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_encrypted": { + "name": "is_encrypted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "encrypted_preview": { + "name": "encrypted_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "settings_org_id_orgs_id_fk": { + "name": "settings_org_id_orgs_id_fk", + "tableFrom": "settings", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_key_unique": { + "name": "settings_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.extraction_status": { + "name": "extraction_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "COMPLETE", + "STALE", + "CANCELLED" + ] + }, + "public.log_level": { + "name": "log_level", + "schema": "public", + "values": [ + "INFO", + "ERROR" + ] + }, + "public.page_status": { + "name": "page_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.page_type": { + "name": "page_type", + "schema": "public", + "values": [ + "COURSE_DETAIL_PAGE", + "CATEGORY_LINKS_PAGE", + "COURSE_LINKS_PAGE" + ] + }, + "public.provider": { + "name": "provider", + "schema": "public", + "values": [ + "openai" + ] + }, + "public.provider_model": { + "name": "provider_model", + "schema": "public", + "values": [ + "gpt-4o" + ] + }, + "public.recipe_detection_status": { + "name": "recipe_detection_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.step": { + "name": "step", + "schema": "public", + "values": [ + "FETCH_ROOT", + "FETCH_PAGINATED", + "FETCH_LINKS" + ] + }, + "public.url_pattern_type": { + "name": "url_pattern_type", + "schema": "public", + "values": [ + "page_num", + "offset" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/migrations/meta/0004_snapshot.json b/server/migrations/meta/0004_snapshot.json new file mode 100644 index 0000000..fc8cbce --- /dev/null +++ b/server/migrations/meta/0004_snapshot.json @@ -0,0 +1,1244 @@ +{ + "id": "d980c159-9ee4-409d-825d-a52c3b67eb45", + "prevId": "66d696cc-e665-44f2-b7bb-903a35f2a4f2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.catalogues": { + "name": "catalogues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "catalogues_recipe_id_orgs_id_fk": { + "name": "catalogues_recipe_id_orgs_id_fk", + "tableFrom": "catalogues", + "tableTo": "orgs", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "catalogues_url_unique": { + "name": "catalogues_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_pages": { + "name": "crawl_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_step_id": { + "name": "crawl_step_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "page_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "screenshot": { + "name": "screenshot", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetch_failure_reason": { + "name": "fetch_failure_reason", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "data_type": { + "name": "data_type", + "type": "page_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "data_extraction_started_at": { + "name": "data_extraction_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_pages_extraction_idx": { + "name": "crawl_pages_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_status_idx": { + "name": "crawl_pages_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_data_type_idx": { + "name": "crawl_pages_data_type_idx", + "columns": [ + { + "expression": "data_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_step_idx": { + "name": "crawl_pages_step_idx", + "columns": [ + { + "expression": "crawl_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_pages_extraction_id_extractions_id_fk": { + "name": "crawl_pages_extraction_id_extractions_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_pages_crawl_step_id_crawl_steps_id_fk": { + "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "crawl_steps", + "columnsFrom": [ + "crawl_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "crawl_pages_extraction_id_url_unique": { + "name": "crawl_pages_extraction_id_url_unique", + "nullsNotDistinct": false, + "columns": [ + "extraction_id", + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_steps": { + "name": "crawl_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "step": { + "name": "step", + "type": "step", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_step_id": { + "name": "parent_step_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_steps_extraction_idx": { + "name": "crawl_steps_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_steps_parent_step_idx": { + "name": "crawl_steps_parent_step_idx", + "columns": [ + { + "expression": "parent_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_steps_extraction_id_extractions_id_fk": { + "name": "crawl_steps_extraction_id_extractions_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_steps_parent_step_id_crawl_steps_id_fk": { + "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "crawl_steps", + "columnsFrom": [ + "parent_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_items": { + "name": "data_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_page_id": { + "name": "crawl_page_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "structured_data": { + "name": "structured_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "text_inclusion": { + "name": "text_inclusion", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_items_dataset_idx": { + "name": "data_items_dataset_idx", + "columns": [ + { + "expression": "dataset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_items_crawl_page_idx": { + "name": "data_items_crawl_page_idx", + "columns": [ + { + "expression": "crawl_page_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_items_dataset_id_datasets_id_fk": { + "name": "data_items_dataset_id_datasets_id_fk", + "tableFrom": "data_items", + "tableTo": "datasets", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_items_crawl_page_id_crawl_pages_id_fk": { + "name": "data_items_crawl_page_id_crawl_pages_id_fk", + "tableFrom": "data_items", + "tableTo": "crawl_pages", + "columnsFrom": [ + "crawl_page_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.datasets": { + "name": "datasets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_catalogue_idx": { + "name": "datasets_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_extraction_idx": { + "name": "datasets_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_catalogue_id_catalogues_id_fk": { + "name": "datasets_catalogue_id_catalogues_id_fk", + "tableFrom": "datasets", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_extraction_id_extractions_id_fk": { + "name": "datasets_extraction_id_extractions_id_fk", + "tableFrom": "datasets", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "datasets_catalogue_id_extraction_id_unique": { + "name": "datasets_catalogue_id_extraction_id_unique", + "nullsNotDistinct": false, + "columns": [ + "catalogue_id", + "extraction_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extraction_logs": { + "name": "extraction_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "log": { + "name": "log", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "log_level": { + "name": "log_level", + "type": "log_level", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'INFO'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extraction_logs_extraction_idx": { + "name": "extraction_logs_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extraction_logs_extraction_id_extractions_id_fk": { + "name": "extraction_logs_extraction_id_extractions_id_fk", + "tableFrom": "extraction_logs", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extractions": { + "name": "extractions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "completion_stats": { + "name": "completion_stats", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "extraction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extractions_recipe_idx": { + "name": "extractions_recipe_idx", + "columns": [ + { + "expression": "recipe_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extractions_recipe_id_recipes_id_fk": { + "name": "extractions_recipe_id_recipes_id_fk", + "tableFrom": "extractions", + "tableTo": "recipes", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_api_calls": { + "name": "model_api_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "provider_model", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "call_site": { + "name": "call_site", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_count": { + "name": "input_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "output_token_count": { + "name": "output_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "model_api_calls_extraction_idx": { + "name": "model_api_calls_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "model_api_calls_extraction_id_extractions_id_fk": { + "name": "model_api_calls_extraction_id_extractions_id_fk", + "tableFrom": "model_api_calls", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs_users_roles": { + "name": "orgs_users_roles", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "orgs_users_roles_org_id_orgs_id_fk": { + "name": "orgs_users_roles_org_id_orgs_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_users_id_fk": { + "name": "orgs_users_roles_user_id_users_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "orgs_users_roles_user_id_user_roles_id_fk": { + "name": "orgs_users_roles_user_id_user_roles_id_fk", + "tableFrom": "orgs_users_roles", + "tableTo": "user_roles", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.recipes": { + "name": "recipes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "detection_failure_reason": { + "name": "detection_failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "recipe_detection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + } + }, + "indexes": { + "recipes_catalogue_idx": { + "name": "recipes_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "recipes_catalogue_id_catalogues_id_fk": { + "name": "recipes_catalogue_id_catalogues_id_fk", + "tableFrom": "recipes", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_encrypted": { + "name": "is_encrypted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "encrypted_preview": { + "name": "encrypted_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "settings_org_id_orgs_id_fk": { + "name": "settings_org_id_orgs_id_fk", + "tableFrom": "settings", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_key_unique": { + "name": "settings_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.extraction_status": { + "name": "extraction_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "COMPLETE", + "STALE", + "CANCELLED" + ] + }, + "public.log_level": { + "name": "log_level", + "schema": "public", + "values": [ + "INFO", + "ERROR" + ] + }, + "public.page_status": { + "name": "page_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.page_type": { + "name": "page_type", + "schema": "public", + "values": [ + "COURSE_DETAIL_PAGE", + "CATEGORY_LINKS_PAGE", + "COURSE_LINKS_PAGE" + ] + }, + "public.provider": { + "name": "provider", + "schema": "public", + "values": [ + "openai" + ] + }, + "public.provider_model": { + "name": "provider_model", + "schema": "public", + "values": [ + "gpt-4o" + ] + }, + "public.recipe_detection_status": { + "name": "recipe_detection_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.step": { + "name": "step", + "schema": "public", + "values": [ + "FETCH_ROOT", + "FETCH_PAGINATED", + "FETCH_LINKS" + ] + }, + "public.url_pattern_type": { + "name": "url_pattern_type", + "schema": "public", + "values": [ + "page_num", + "offset" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/migrations/meta/_journal.json b/server/migrations/meta/_journal.json index 8a62c06..5f1397c 100644 --- a/server/migrations/meta/_journal.json +++ b/server/migrations/meta/_journal.json @@ -22,6 +22,20 @@ "when": 1738950572242, "tag": "0002_sudden_sprite", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1738953325622, + "tag": "0003_fixed_hydra", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1738953356264, + "tag": "0004_sticky_onslaught", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/package.json b/server/package.json index 4336b8e..ee60eac 100644 --- a/server/package.json +++ b/server/package.json @@ -22,6 +22,7 @@ "cheerio": "1.0.0-rc.12", "dotenv": "^16.4.5", "drizzle-orm": "^0.39.2", + "drizzle-seed": "^0.3.1", "fast-csv": "^5.0.1", "fast-equals": "^5.0.1", "fastify": "^4.26.2", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 59a1c7d..79b86f5 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: drizzle-orm: specifier: ^0.39.2 version: 0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + drizzle-seed: + specifier: ^0.3.1 + version: 0.3.1(drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1)) fast-csv: specifier: ^5.0.1 version: 5.0.1 @@ -2529,6 +2532,14 @@ packages: sqlite3: optional: true + drizzle-seed@0.3.1: + resolution: {integrity: sha512-F/0lgvfOAsqlYoHM/QAGut4xXIOXoE5VoAdv2FIl7DpGYVXlAzKuJO+IphkKUFK3Dz+rFlOsQLnMNrvoQ0cx7g==} + peerDependencies: + drizzle-orm: '>=0.36.4' + peerDependenciesMeta: + drizzle-orm: + optional: true + eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -3884,6 +3895,9 @@ packages: deprecated: < 22.8.2 is no longer supported hasBin: true + pure-rand@6.1.0: + resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} + queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6786,6 +6800,12 @@ snapshots: pg: 8.12.0 react: 18.3.1 + drizzle-seed@0.3.1(drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1)): + dependencies: + pure-rand: 6.1.0 + optionalDependencies: + drizzle-orm: 0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + eastasianwidth@0.2.0: {} editorconfig@1.0.4: @@ -8269,6 +8289,8 @@ snapshots: - typescript - utf-8-validate + pure-rand@6.1.0: {} + queue-microtask@1.2.3: {} queue-tick@1.0.1: {} diff --git a/server/src/data/catalogues.ts b/server/src/data/catalogues.ts index 33e3878..c8247cd 100644 --- a/server/src/data/catalogues.ts +++ b/server/src/data/catalogues.ts @@ -1,4 +1,4 @@ -import { desc, eq, sql } from "drizzle-orm"; +import { desc, eq, sql, and } from "drizzle-orm"; import db from "../data"; import { catalogues, extractions, recipes } from "../data/schema"; @@ -34,9 +34,12 @@ export async function findCatalogueById(id: number) { return result; } -export async function findCatalogueByUrl(url: string) { +export async function findCatalogueByUrl(url: string, orgIds: number[]) { return db.query.catalogues.findFirst({ - where: (catalogues, { eq }) => eq(catalogues.url, url), + where: and( + (catalogues, { eq }) => eq(catalogues.url, url), + (catalogues, { in }) => in(catalogues.orgId, orgIds), + ), }); } @@ -51,9 +54,10 @@ export async function findLatestExtractionsForCatalogue(catalogueId: number) { return catExtractions.map((e) => e.extractions); } -export async function findCatalogues(limit: number = 20, offset?: number) { +export async function findCatalogues(orgId: number[], limit: number = 20, offset?: number) { offset = offset || 0; return db.query.catalogues.findMany({ + where: (catalogues, { eq }) => eq(catalogues.orgId, orgId), limit, offset, with: { @@ -65,11 +69,12 @@ export async function findCatalogues(limit: number = 20, offset?: number) { export async function createCatalogue( name: string, url: string, - thumbnailUrl?: string + orgId: number, + thumbnailUrl?: string, ) { const result = await db .insert(catalogues) - .values({ name, url, thumbnailUrl }) + .values({ name, url, thumbnailUrl, orgId }) .returning(); return result[0]; } diff --git a/server/src/data/orgs.ts b/server/src/data/orgs.ts new file mode 100644 index 0000000..48362db --- /dev/null +++ b/server/src/data/orgs.ts @@ -0,0 +1,9 @@ +import db from "../data"; + +export async function findOrgsByUser(userId: number) { + return db.query.orgsUsers.findMany({ + where(fields, { eq }) { + return eq(fields?.userId, userId) + }, + }); +} \ No newline at end of file diff --git a/server/src/data/schema.ts b/server/src/data/schema.ts index 3ef4962..19d5913 100644 --- a/server/src/data/schema.ts +++ b/server/src/data/schema.ts @@ -206,6 +206,9 @@ const catalogues = pgTable("catalogues", { url: text("url").notNull().unique(), thumbnailUrl: text("thumbnail_url"), createdAt: timestamp("created_at").notNull().defaultNow(), + orgId: integer("recipe_id") + .notNull() + .references(() => orgs.id), }); const cataloguesRelations = relations(catalogues, ({ many }) => ({ @@ -478,6 +481,7 @@ const users = pgTable("users", { name: text("name").notNull(), email: text("email").notNull().unique(), password: text("password").notNull(), + isStaff: boolean("is_staff").notNull().default(false), createdAt: timestamp("created_at").notNull().defaultNow(), }); diff --git a/server/src/data/settings.ts b/server/src/data/settings.ts index 3550d47..1920cc1 100644 --- a/server/src/data/settings.ts +++ b/server/src/data/settings.ts @@ -1,15 +1,20 @@ import db from "../data"; import { encryptForDb, settings } from "../data/schema"; -export async function findSettings() { - return db.query.settings.findMany(); +export async function findSettings(orgId: number) { + return db.query.settings.findMany({ + where(fields, { eq }) { + return eq(fields?.orgId, orgId) + }, + }); } export async function createOrUpdate( key: string, value: string, isEncrypted: boolean = false, - encryptedPreview: string | null + encryptedPreview: string | null, + orgId: number, ) { value = isEncrypted ? encryptForDb(value) : value; return db @@ -19,6 +24,7 @@ export async function createOrUpdate( value, isEncrypted, encryptedPreview, + orgId, }) .onConflictDoUpdate({ target: settings.key, diff --git a/server/src/fastifySessionAuth.ts b/server/src/fastifySessionAuth.ts index 337ea77..64400d0 100644 --- a/server/src/fastifySessionAuth.ts +++ b/server/src/fastifySessionAuth.ts @@ -22,11 +22,13 @@ const fastifySessionAuth: FastifyPluginCallback = fastifyPlugin( return; } + // adjust frontend to send the desired orgId const userId = req.session.get("userId"); if (userId) { const user = await findUserById(parseInt(userId)); if (user) { req.user = user; + // add selected orgId to the user context } else { req.session.set("userId", undefined); } diff --git a/server/src/routers/catalogues.ts b/server/src/routers/catalogues.ts index 78c1cfb..c226257 100644 --- a/server/src/routers/catalogues.ts +++ b/server/src/routers/catalogues.ts @@ -11,6 +11,7 @@ import { } from "../data/catalogues"; import { findDatasets } from "../data/datasets"; import { fetchPreview } from "../extraction/browser"; +import { findOrgsByUser } from "@/data/orgs"; export const cataloguesRouter = router({ preview: publicProcedure @@ -35,7 +36,7 @@ export const cataloguesRouter = router({ return { totalItems, totalPages, - results: await findCatalogues(20, opts.input.page * 20 - 20), + results: await findCatalogues(opts.ctx.user.orgId, 20, opts.input.page * 20 - 20), }; }), create: publicProcedure @@ -48,7 +49,8 @@ export const cataloguesRouter = router({ ) .mutation(async (opts) => { const { name, url, thumbnailUrl } = opts.input; - const existingCatalogue = await findCatalogueByUrl(url); + const userOrgs = await findOrgsByUser(opts.ctx.user.id); + const existingCatalogue = await findCatalogueByUrl(url, userOrgs.map(org => org.orgId)); if (existingCatalogue) { return { id: existingCatalogue.id, diff --git a/server/src/routers/settings.ts b/server/src/routers/settings.ts index 27f6b57..5050483 100644 --- a/server/src/routers/settings.ts +++ b/server/src/routers/settings.ts @@ -1,10 +1,11 @@ import { z } from "zod"; import { publicProcedure, router } from "."; import { createOrUpdate, findSettings } from "../data/settings"; +import { findUserById } from '../data/users'; export const settingsRouter = router({ list: publicProcedure.query(async (_opts) => { - const allSettings = await findSettings(); + const allSettings = await findSettings(_opts.ctx.user?.orgId); return allSettings.map((setting) => ({ ...setting, value: setting.isEncrypted ? "*****" : setting.value, @@ -21,7 +22,8 @@ export const settingsRouter = router({ "OPENAI_API_KEY", opts.input.apiKey, true, - `sk-...${opts.input.apiKey.slice(-4)}` + `sk-...${opts.input.apiKey.slice(-4)}`, + opts.ctx.user.orgId ); }), }); From ffca44923d2570ce8fc4a05dc5be4dd97ff23df9 Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Mon, 10 Feb 2025 22:11:18 +0200 Subject: [PATCH 06/12] WIP --- client/src/components/app/routes.tsx | 132 +- .../src/components/app/selectOrganization.tsx | 5 + client/src/main.tsx | 10 +- client/src/userContext.tsx | 65 +- common/types.ts | 4 + server/migrations/0002_cooing_gauntlet.sql | 21 + server/migrations/0002_sudden_sprite.sql | 36 - server/migrations/meta/0002_snapshot.json | 226 ++- server/migrations/meta/0003_snapshot.json | 1257 ----------------- server/migrations/meta/0004_snapshot.json | 1244 ---------------- server/migrations/meta/_journal.json | 18 +- server/src/data/catalogues.ts | 10 +- server/src/data/orgs.ts | 11 +- server/src/data/schema.ts | 25 +- server/src/data/types.ts | 4 + server/src/fastifySessionAuth.ts | 1 - server/src/trpcContext.ts | 5 +- 17 files changed, 353 insertions(+), 2721 deletions(-) create mode 100644 client/src/components/app/selectOrganization.tsx create mode 100644 server/migrations/0002_cooing_gauntlet.sql delete mode 100644 server/migrations/0002_sudden_sprite.sql delete mode 100644 server/migrations/meta/0003_snapshot.json delete mode 100644 server/migrations/meta/0004_snapshot.json create mode 100644 server/src/data/types.ts diff --git a/client/src/components/app/routes.tsx b/client/src/components/app/routes.tsx index 1bd6154..f1fc8c5 100644 --- a/client/src/components/app/routes.tsx +++ b/client/src/components/app/routes.tsx @@ -1,4 +1,4 @@ -import { Route, Switch } from "wouter"; +import { Router, Route, Switch, BaseLocationHook, useParams } from "wouter"; import Catalogues from "./catalogues"; import CreateCatalogue from "./catalogues/create"; import CatalogueDetail from "./catalogues/detail"; @@ -20,66 +20,84 @@ import CreateUser from "./users/create"; import DeleteUser from "./users/delete"; import ResetUserPassword from "./users/reset-password"; import Welcome from "./welcome"; +import { useCallback, useContext, useEffect } from "react"; +import { UserContext } from "@/userContext"; +import { SelectOrganization } from "./selectOrganization"; export default function Routes() { - // We will add a new route called /select-org that will be the thing a user sees - // if their session does not contain an org ID. So immediately after login. + const { uriOrg } = useParams(); + const { orgId, setOrgId } = useContext(UserContext); - // We will create a new organization list selector component in this folder. - // We will ensure backend sets a correct orgId if the user is part of a single - // org as to not bother the user selecting their only organization. + useEffect(() => { + setOrgId(orgId) + }, [uriOrg]) + + const onNavigate = useCallback( + (location, setLocation) => { + if (!uriOrg) { + setLocation('/select-org') + } + + return [location, setLocation] + }, [uriOrg] + ) return ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); } diff --git a/client/src/components/app/selectOrganization.tsx b/client/src/components/app/selectOrganization.tsx new file mode 100644 index 0000000..0469f71 --- /dev/null +++ b/client/src/components/app/selectOrganization.tsx @@ -0,0 +1,5 @@ + + +export const SelectOrganization = () => { + return
Hello
+} \ No newline at end of file diff --git a/client/src/main.tsx b/client/src/main.tsx index 6a5180c..a259586 100644 --- a/client/src/main.tsx +++ b/client/src/main.tsx @@ -1,12 +1,12 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { httpBatchLink } from "@trpc/client"; -import React, { useEffect, useState } from "react"; +import React, { useContext, useEffect, useState } from "react"; import ReactDOM from "react-dom/client"; import { Dashboard } from "./components/app/dashboard"; import Unauthenticated from "./components/app/unauthenticated"; import { API_URL } from "./constants"; import "./main.css"; -import UserContext from "./userContext"; +import { UserContext, UserProvider } from "./userContext"; import { trpc } from "./utils"; const queryClient = new QueryClient(); @@ -25,8 +25,8 @@ const trpcClient = trpc.createClient({ }); export function App() { - const [user, setUser] = useState(); const [isLoading, setIsLoading] = useState(true); + const {user, setUser} = useContext(UserContext); useEffect(() => { setIsLoading(true); @@ -50,13 +50,13 @@ export function App() { return ( - + {user ? : } - + ); } diff --git a/client/src/userContext.tsx b/client/src/userContext.tsx index 0f6eea8..36f2dd3 100644 --- a/client/src/userContext.tsx +++ b/client/src/userContext.tsx @@ -1,8 +1,65 @@ -import { createContext } from "react"; +import { + createContext, + FC, + PropsWithChildren, + useEffect, + useMemo, + useState +} from "react"; -const UserContext = createContext<{ +import { useQuery, UseQueryResult } from "@tanstack/react-query"; + +import { Organizations } from "@common/types"; + +export interface UserContextProps { user: any; + userOrganizationsQuery?: UseQueryResult, setUser: React.Dispatch; -}>({ user: null, setUser: () => {} }); + orgId?: number; + setOrgId: React.Dispatch; +} + +export const UserContext = createContext({ + user: null, + setUser: () => { }, + setOrgId: () => { } +}); + +export const UserProvider: FC = ({ children }) => { + const [user, setUser] = useState(); + const [orgId, setOrgId] = useState(); + // const [userOrganizations, setUserOrganizations] = useState([]); + const userOrganizationsQuery = useUserOrganizationsQuery(user?.id); + + useEffect(() => { + if (user?.id) { + // TODO: Call get user orgs + } + }, [user?.id]) + + const value = useMemo(() => ({ + user, + setUser, + orgId, + setOrgId, + userOrganizationsQuery, + }), + [ + user?.id, + orgId + ] + ); + return ( + + {children} + + ); +} -export default UserContext; +const useUserOrganizationsQuery = (userId: string | null) => { + return useQuery({ + queryKey: ["user", userId], // The query depends on `userId` + queryFn: () => { console.log('hello!') }, // Fetch function + enabled: !!userId, // Only run query if `userId` is not null/undefined + }); +}; \ No newline at end of file diff --git a/common/types.ts b/common/types.ts index 9189acd..7fe3914 100644 --- a/common/types.ts +++ b/common/types.ts @@ -1,3 +1,5 @@ +import { orgs } from '../server/src/data/schema'; + export enum CatalogueType { COURSES = "COURSES", LEARNING_PROGRAMS = "LEARNING_PROGRAMS", @@ -117,3 +119,5 @@ export interface CompletionStats { steps: StepCompletionStats[]; generatedAt: string; } + +export { Organizations } from '../server/src/data/types'; \ No newline at end of file diff --git a/server/migrations/0002_cooing_gauntlet.sql b/server/migrations/0002_cooing_gauntlet.sql new file mode 100644 index 0000000..61c66e0 --- /dev/null +++ b/server/migrations/0002_cooing_gauntlet.sql @@ -0,0 +1,21 @@ +CREATE TYPE "public"."role_type" AS ENUM('viewer', 'member', 'admin');--> statement-breakpoint +CREATE TABLE "memberships" ( + "org_id" integer NOT NULL, + "user_id" integer NOT NULL, + "role" "role_type" DEFAULT 'member' +); +--> statement-breakpoint +CREATE TABLE "orgs" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text DEFAULT 'null', + "description" text DEFAULT 'null', + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "catalogues" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint +ALTER TABLE "settings" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint +ALTER TABLE "users" ADD COLUMN "is_staff" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "memberships" ADD CONSTRAINT "memberships_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "memberships" ADD CONSTRAINT "memberships_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "catalogues" ADD CONSTRAINT "catalogues_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "settings" ADD CONSTRAINT "settings_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/server/migrations/0002_sudden_sprite.sql b/server/migrations/0002_sudden_sprite.sql deleted file mode 100644 index f7831c6..0000000 --- a/server/migrations/0002_sudden_sprite.sql +++ /dev/null @@ -1,36 +0,0 @@ -CREATE TABLE IF NOT EXISTS "orgs" ( - "id" serial PRIMARY KEY NOT NULL, - "name" text DEFAULT 'null', - "description" text DEFAULT 'null', - "created_at" timestamp DEFAULT now() NOT NULL -); ---> statement-breakpoint -CREATE TABLE IF NOT EXISTS "orgs_users_roles" ( - "org_id" integer NOT NULL, - "user_id" integer NOT NULL -); ---> statement-breakpoint -ALTER TABLE "settings" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint -DO $$ BEGIN - ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; -EXCEPTION - WHEN duplicate_object THEN null; -END $$; ---> statement-breakpoint -DO $$ BEGIN - ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action; -EXCEPTION - WHEN duplicate_object THEN null; -END $$; ---> statement-breakpoint -DO $$ BEGIN - ALTER TABLE "orgs_users_roles" ADD CONSTRAINT "orgs_users_roles_user_id_user_roles_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user_roles"("id") ON DELETE no action ON UPDATE no action; -EXCEPTION - WHEN duplicate_object THEN null; -END $$; ---> statement-breakpoint -DO $$ BEGIN - ALTER TABLE "settings" ADD CONSTRAINT "settings_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; -EXCEPTION - WHEN duplicate_object THEN null; -END $$; diff --git a/server/migrations/meta/0002_snapshot.json b/server/migrations/meta/0002_snapshot.json index ccaeb52..4dc0acd 100644 --- a/server/migrations/meta/0002_snapshot.json +++ b/server/migrations/meta/0002_snapshot.json @@ -1,5 +1,5 @@ { - "id": "7ca450cc-9206-4830-99b7-d38471fbb9bc", + "id": "7d940431-8c0e-451e-b69d-ac5a1826cb84", "prevId": "a30ef025-2510-4069-95cd-7e87c7728d5d", "version": "7", "dialect": "postgresql", @@ -38,10 +38,30 @@ "primaryKey": false, "notNull": true, "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true } }, "indexes": {}, - "foreignKeys": {}, + "foreignKeys": { + "catalogues_org_id_orgs_id_fk": { + "name": "catalogues_org_id_orgs_id_fk", + "tableFrom": "catalogues", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, "compositePrimaryKeys": {}, "uniqueConstraints": { "catalogues_url_unique": { @@ -51,7 +71,10 @@ "url" ] } - } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.crawl_pages": { "name": "crawl_pages", @@ -228,7 +251,10 @@ "url" ] } - } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.crawl_steps": { "name": "crawl_steps", @@ -334,7 +360,10 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.data_items": { "name": "data_items", @@ -439,7 +468,10 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.datasets": { "name": "datasets", @@ -541,7 +573,10 @@ "extraction_id" ] } - } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.extraction_logs": { "name": "extraction_logs", @@ -614,7 +649,10 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.extractions": { "name": "extractions", @@ -687,7 +725,70 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'member'" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_org_id_orgs_id_fk": { + "name": "memberships_org_id_orgs_id_fk", + "tableFrom": "memberships", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.model_api_calls": { "name": "model_api_calls", @@ -778,7 +879,10 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.orgs": { "name": "orgs", @@ -815,69 +919,10 @@ "indexes": {}, "foreignKeys": {}, "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "public.orgs_users_roles": { - "name": "orgs_users_roles", - "schema": "", - "columns": { - "org_id": { - "name": "org_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "orgs_users_roles_org_id_orgs_id_fk": { - "name": "orgs_users_roles_org_id_orgs_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "orgs", - "columnsFrom": [ - "org_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_users_id_fk": { - "name": "orgs_users_roles_user_id_users_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_user_roles_id_fk": { - "name": "orgs_users_roles_user_id_user_roles_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "user_roles", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.recipes": { "name": "recipes", @@ -969,7 +1014,10 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.settings": { "name": "settings", @@ -1039,7 +1087,10 @@ "key" ] } - } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, "public.users": { "name": "users", @@ -1069,6 +1120,13 @@ "primaryKey": false, "notNull": true }, + "is_staff": { + "name": "is_staff", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, "created_at": { "name": "created_at", "type": "timestamp", @@ -1088,7 +1146,10 @@ "email" ] } - } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false } }, "enums": { @@ -1154,6 +1215,15 @@ "ERROR" ] }, + "public.role_type": { + "name": "role_type", + "schema": "public", + "values": [ + "viewer", + "member", + "admin" + ] + }, "public.step": { "name": "step", "schema": "public", @@ -1173,6 +1243,10 @@ } }, "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, "_meta": { "columns": {}, "schemas": {}, diff --git a/server/migrations/meta/0003_snapshot.json b/server/migrations/meta/0003_snapshot.json deleted file mode 100644 index 21ef40a..0000000 --- a/server/migrations/meta/0003_snapshot.json +++ /dev/null @@ -1,1257 +0,0 @@ -{ - "id": "66d696cc-e665-44f2-b7bb-903a35f2a4f2", - "prevId": "7ca450cc-9206-4830-99b7-d38471fbb9bc", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.catalogues": { - "name": "catalogues", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "thumbnail_url": { - "name": "thumbnail_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "recipe_id": { - "name": "recipe_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "catalogues_recipe_id_orgs_id_fk": { - "name": "catalogues_recipe_id_orgs_id_fk", - "tableFrom": "catalogues", - "tableTo": "orgs", - "columnsFrom": [ - "recipe_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "catalogues_url_unique": { - "name": "catalogues_url_unique", - "nullsNotDistinct": false, - "columns": [ - "url" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.crawl_pages": { - "name": "crawl_pages", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "crawl_step_id": { - "name": "crawl_step_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "status": { - "name": "status", - "type": "page_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "screenshot": { - "name": "screenshot", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "fetch_failure_reason": { - "name": "fetch_failure_reason", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "data_type": { - "name": "data_type", - "type": "page_type", - "typeSchema": "public", - "primaryKey": false, - "notNull": false - }, - "data_extraction_started_at": { - "name": "data_extraction_started_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "crawl_pages_extraction_idx": { - "name": "crawl_pages_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_status_idx": { - "name": "crawl_pages_status_idx", - "columns": [ - { - "expression": "status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_data_type_idx": { - "name": "crawl_pages_data_type_idx", - "columns": [ - { - "expression": "data_type", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_step_idx": { - "name": "crawl_pages_step_idx", - "columns": [ - { - "expression": "crawl_step_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "crawl_pages_extraction_id_extractions_id_fk": { - "name": "crawl_pages_extraction_id_extractions_id_fk", - "tableFrom": "crawl_pages", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "crawl_pages_crawl_step_id_crawl_steps_id_fk": { - "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", - "tableFrom": "crawl_pages", - "tableTo": "crawl_steps", - "columnsFrom": [ - "crawl_step_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "crawl_pages_extraction_id_url_unique": { - "name": "crawl_pages_extraction_id_url_unique", - "nullsNotDistinct": false, - "columns": [ - "extraction_id", - "url" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.crawl_steps": { - "name": "crawl_steps", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "step": { - "name": "step", - "type": "step", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "parent_step_id": { - "name": "parent_step_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "configuration": { - "name": "configuration", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "crawl_steps_extraction_idx": { - "name": "crawl_steps_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_steps_parent_step_idx": { - "name": "crawl_steps_parent_step_idx", - "columns": [ - { - "expression": "parent_step_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "crawl_steps_extraction_id_extractions_id_fk": { - "name": "crawl_steps_extraction_id_extractions_id_fk", - "tableFrom": "crawl_steps", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "crawl_steps_parent_step_id_crawl_steps_id_fk": { - "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", - "tableFrom": "crawl_steps", - "tableTo": "crawl_steps", - "columnsFrom": [ - "parent_step_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.data_items": { - "name": "data_items", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "dataset_id": { - "name": "dataset_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "crawl_page_id": { - "name": "crawl_page_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "structured_data": { - "name": "structured_data", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "text_inclusion": { - "name": "text_inclusion", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "data_items_dataset_idx": { - "name": "data_items_dataset_idx", - "columns": [ - { - "expression": "dataset_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "data_items_crawl_page_idx": { - "name": "data_items_crawl_page_idx", - "columns": [ - { - "expression": "crawl_page_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "data_items_dataset_id_datasets_id_fk": { - "name": "data_items_dataset_id_datasets_id_fk", - "tableFrom": "data_items", - "tableTo": "datasets", - "columnsFrom": [ - "dataset_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "data_items_crawl_page_id_crawl_pages_id_fk": { - "name": "data_items_crawl_page_id_crawl_pages_id_fk", - "tableFrom": "data_items", - "tableTo": "crawl_pages", - "columnsFrom": [ - "crawl_page_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.datasets": { - "name": "datasets", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "catalogue_id": { - "name": "catalogue_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "datasets_catalogue_idx": { - "name": "datasets_catalogue_idx", - "columns": [ - { - "expression": "catalogue_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "datasets_extraction_idx": { - "name": "datasets_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "datasets_catalogue_id_catalogues_id_fk": { - "name": "datasets_catalogue_id_catalogues_id_fk", - "tableFrom": "datasets", - "tableTo": "catalogues", - "columnsFrom": [ - "catalogue_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "datasets_extraction_id_extractions_id_fk": { - "name": "datasets_extraction_id_extractions_id_fk", - "tableFrom": "datasets", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "datasets_catalogue_id_extraction_id_unique": { - "name": "datasets_catalogue_id_extraction_id_unique", - "nullsNotDistinct": false, - "columns": [ - "catalogue_id", - "extraction_id" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.extraction_logs": { - "name": "extraction_logs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "log": { - "name": "log", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "log_level": { - "name": "log_level", - "type": "log_level", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'INFO'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "extraction_logs_extraction_idx": { - "name": "extraction_logs_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "extraction_logs_extraction_id_extractions_id_fk": { - "name": "extraction_logs_extraction_id_extractions_id_fk", - "tableFrom": "extraction_logs", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.extractions": { - "name": "extractions", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "recipe_id": { - "name": "recipe_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "completion_stats": { - "name": "completion_stats", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "extraction_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "extractions_recipe_idx": { - "name": "extractions_recipe_idx", - "columns": [ - { - "expression": "recipe_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "extractions_recipe_id_recipes_id_fk": { - "name": "extractions_recipe_id_recipes_id_fk", - "tableFrom": "extractions", - "tableTo": "recipes", - "columnsFrom": [ - "recipe_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "extractions_recipe_id_orgs_id_fk": { - "name": "extractions_recipe_id_orgs_id_fk", - "tableFrom": "extractions", - "tableTo": "orgs", - "columnsFrom": [ - "recipe_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.model_api_calls": { - "name": "model_api_calls", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "provider": { - "name": "provider", - "type": "provider", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "model": { - "name": "model", - "type": "provider_model", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "call_site": { - "name": "call_site", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "input_token_count": { - "name": "input_token_count", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "output_token_count": { - "name": "output_token_count", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "model_api_calls_extraction_idx": { - "name": "model_api_calls_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "model_api_calls_extraction_id_extractions_id_fk": { - "name": "model_api_calls_extraction_id_extractions_id_fk", - "tableFrom": "model_api_calls", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.orgs": { - "name": "orgs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": false, - "default": "'null'" - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": false, - "default": "'null'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.orgs_users_roles": { - "name": "orgs_users_roles", - "schema": "", - "columns": { - "org_id": { - "name": "org_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "orgs_users_roles_org_id_orgs_id_fk": { - "name": "orgs_users_roles_org_id_orgs_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "orgs", - "columnsFrom": [ - "org_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_users_id_fk": { - "name": "orgs_users_roles_user_id_users_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_user_roles_id_fk": { - "name": "orgs_users_roles_user_id_user_roles_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "user_roles", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.recipes": { - "name": "recipes", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "is_default": { - "name": "is_default", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - }, - "catalogue_id": { - "name": "catalogue_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "configuration": { - "name": "configuration", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "detection_failure_reason": { - "name": "detection_failure_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "recipe_detection_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - } - }, - "indexes": { - "recipes_catalogue_idx": { - "name": "recipes_catalogue_idx", - "columns": [ - { - "expression": "catalogue_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "recipes_catalogue_id_catalogues_id_fk": { - "name": "recipes_catalogue_id_catalogues_id_fk", - "tableFrom": "recipes", - "tableTo": "catalogues", - "columnsFrom": [ - "catalogue_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.settings": { - "name": "settings", - "schema": "", - "columns": { - "key": { - "name": "key", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "value": { - "name": "value", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "is_encrypted": { - "name": "is_encrypted", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - }, - "encrypted_preview": { - "name": "encrypted_preview", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "org_id": { - "name": "org_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "settings_org_id_orgs_id_fk": { - "name": "settings_org_id_orgs_id_fk", - "tableFrom": "settings", - "tableTo": "orgs", - "columnsFrom": [ - "org_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "settings_key_unique": { - "name": "settings_key_unique", - "nullsNotDistinct": false, - "columns": [ - "key" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.users": { - "name": "users", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "users_email_unique": { - "name": "users_email_unique", - "nullsNotDistinct": false, - "columns": [ - "email" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": { - "public.extraction_status": { - "name": "extraction_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "COMPLETE", - "STALE", - "CANCELLED" - ] - }, - "public.log_level": { - "name": "log_level", - "schema": "public", - "values": [ - "INFO", - "ERROR" - ] - }, - "public.page_status": { - "name": "page_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "SUCCESS", - "ERROR" - ] - }, - "public.page_type": { - "name": "page_type", - "schema": "public", - "values": [ - "COURSE_DETAIL_PAGE", - "CATEGORY_LINKS_PAGE", - "COURSE_LINKS_PAGE" - ] - }, - "public.provider": { - "name": "provider", - "schema": "public", - "values": [ - "openai" - ] - }, - "public.provider_model": { - "name": "provider_model", - "schema": "public", - "values": [ - "gpt-4o" - ] - }, - "public.recipe_detection_status": { - "name": "recipe_detection_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "SUCCESS", - "ERROR" - ] - }, - "public.step": { - "name": "step", - "schema": "public", - "values": [ - "FETCH_ROOT", - "FETCH_PAGINATED", - "FETCH_LINKS" - ] - }, - "public.url_pattern_type": { - "name": "url_pattern_type", - "schema": "public", - "values": [ - "page_num", - "offset" - ] - } - }, - "schemas": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "views": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} \ No newline at end of file diff --git a/server/migrations/meta/0004_snapshot.json b/server/migrations/meta/0004_snapshot.json deleted file mode 100644 index fc8cbce..0000000 --- a/server/migrations/meta/0004_snapshot.json +++ /dev/null @@ -1,1244 +0,0 @@ -{ - "id": "d980c159-9ee4-409d-825d-a52c3b67eb45", - "prevId": "66d696cc-e665-44f2-b7bb-903a35f2a4f2", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.catalogues": { - "name": "catalogues", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "thumbnail_url": { - "name": "thumbnail_url", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "recipe_id": { - "name": "recipe_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "catalogues_recipe_id_orgs_id_fk": { - "name": "catalogues_recipe_id_orgs_id_fk", - "tableFrom": "catalogues", - "tableTo": "orgs", - "columnsFrom": [ - "recipe_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "catalogues_url_unique": { - "name": "catalogues_url_unique", - "nullsNotDistinct": false, - "columns": [ - "url" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.crawl_pages": { - "name": "crawl_pages", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "crawl_step_id": { - "name": "crawl_step_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "status": { - "name": "status", - "type": "page_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "screenshot": { - "name": "screenshot", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "fetch_failure_reason": { - "name": "fetch_failure_reason", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "data_type": { - "name": "data_type", - "type": "page_type", - "typeSchema": "public", - "primaryKey": false, - "notNull": false - }, - "data_extraction_started_at": { - "name": "data_extraction_started_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "crawl_pages_extraction_idx": { - "name": "crawl_pages_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_status_idx": { - "name": "crawl_pages_status_idx", - "columns": [ - { - "expression": "status", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_data_type_idx": { - "name": "crawl_pages_data_type_idx", - "columns": [ - { - "expression": "data_type", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_pages_step_idx": { - "name": "crawl_pages_step_idx", - "columns": [ - { - "expression": "crawl_step_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "crawl_pages_extraction_id_extractions_id_fk": { - "name": "crawl_pages_extraction_id_extractions_id_fk", - "tableFrom": "crawl_pages", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "crawl_pages_crawl_step_id_crawl_steps_id_fk": { - "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", - "tableFrom": "crawl_pages", - "tableTo": "crawl_steps", - "columnsFrom": [ - "crawl_step_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "crawl_pages_extraction_id_url_unique": { - "name": "crawl_pages_extraction_id_url_unique", - "nullsNotDistinct": false, - "columns": [ - "extraction_id", - "url" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.crawl_steps": { - "name": "crawl_steps", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "step": { - "name": "step", - "type": "step", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "parent_step_id": { - "name": "parent_step_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "configuration": { - "name": "configuration", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "crawl_steps_extraction_idx": { - "name": "crawl_steps_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "crawl_steps_parent_step_idx": { - "name": "crawl_steps_parent_step_idx", - "columns": [ - { - "expression": "parent_step_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "crawl_steps_extraction_id_extractions_id_fk": { - "name": "crawl_steps_extraction_id_extractions_id_fk", - "tableFrom": "crawl_steps", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "crawl_steps_parent_step_id_crawl_steps_id_fk": { - "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", - "tableFrom": "crawl_steps", - "tableTo": "crawl_steps", - "columnsFrom": [ - "parent_step_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.data_items": { - "name": "data_items", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "dataset_id": { - "name": "dataset_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "crawl_page_id": { - "name": "crawl_page_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "structured_data": { - "name": "structured_data", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "text_inclusion": { - "name": "text_inclusion", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "data_items_dataset_idx": { - "name": "data_items_dataset_idx", - "columns": [ - { - "expression": "dataset_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "data_items_crawl_page_idx": { - "name": "data_items_crawl_page_idx", - "columns": [ - { - "expression": "crawl_page_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "data_items_dataset_id_datasets_id_fk": { - "name": "data_items_dataset_id_datasets_id_fk", - "tableFrom": "data_items", - "tableTo": "datasets", - "columnsFrom": [ - "dataset_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "data_items_crawl_page_id_crawl_pages_id_fk": { - "name": "data_items_crawl_page_id_crawl_pages_id_fk", - "tableFrom": "data_items", - "tableTo": "crawl_pages", - "columnsFrom": [ - "crawl_page_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.datasets": { - "name": "datasets", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "catalogue_id": { - "name": "catalogue_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "datasets_catalogue_idx": { - "name": "datasets_catalogue_idx", - "columns": [ - { - "expression": "catalogue_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - }, - "datasets_extraction_idx": { - "name": "datasets_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "datasets_catalogue_id_catalogues_id_fk": { - "name": "datasets_catalogue_id_catalogues_id_fk", - "tableFrom": "datasets", - "tableTo": "catalogues", - "columnsFrom": [ - "catalogue_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "datasets_extraction_id_extractions_id_fk": { - "name": "datasets_extraction_id_extractions_id_fk", - "tableFrom": "datasets", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "datasets_catalogue_id_extraction_id_unique": { - "name": "datasets_catalogue_id_extraction_id_unique", - "nullsNotDistinct": false, - "columns": [ - "catalogue_id", - "extraction_id" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.extraction_logs": { - "name": "extraction_logs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "log": { - "name": "log", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "log_level": { - "name": "log_level", - "type": "log_level", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'INFO'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "extraction_logs_extraction_idx": { - "name": "extraction_logs_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "extraction_logs_extraction_id_extractions_id_fk": { - "name": "extraction_logs_extraction_id_extractions_id_fk", - "tableFrom": "extraction_logs", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.extractions": { - "name": "extractions", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "recipe_id": { - "name": "recipe_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "completion_stats": { - "name": "completion_stats", - "type": "jsonb", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "extraction_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "extractions_recipe_idx": { - "name": "extractions_recipe_idx", - "columns": [ - { - "expression": "recipe_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "extractions_recipe_id_recipes_id_fk": { - "name": "extractions_recipe_id_recipes_id_fk", - "tableFrom": "extractions", - "tableTo": "recipes", - "columnsFrom": [ - "recipe_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.model_api_calls": { - "name": "model_api_calls", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "extraction_id": { - "name": "extraction_id", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "provider": { - "name": "provider", - "type": "provider", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "model": { - "name": "model", - "type": "provider_model", - "typeSchema": "public", - "primaryKey": false, - "notNull": true - }, - "call_site": { - "name": "call_site", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "input_token_count": { - "name": "input_token_count", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "output_token_count": { - "name": "output_token_count", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "model_api_calls_extraction_idx": { - "name": "model_api_calls_extraction_idx", - "columns": [ - { - "expression": "extraction_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "model_api_calls_extraction_id_extractions_id_fk": { - "name": "model_api_calls_extraction_id_extractions_id_fk", - "tableFrom": "model_api_calls", - "tableTo": "extractions", - "columnsFrom": [ - "extraction_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.orgs": { - "name": "orgs", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": false, - "default": "'null'" - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": false, - "default": "'null'" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.orgs_users_roles": { - "name": "orgs_users_roles", - "schema": "", - "columns": { - "org_id": { - "name": "org_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "user_id": { - "name": "user_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "orgs_users_roles_org_id_orgs_id_fk": { - "name": "orgs_users_roles_org_id_orgs_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "orgs", - "columnsFrom": [ - "org_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_users_id_fk": { - "name": "orgs_users_roles_user_id_users_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "orgs_users_roles_user_id_user_roles_id_fk": { - "name": "orgs_users_roles_user_id_user_roles_id_fk", - "tableFrom": "orgs_users_roles", - "tableTo": "user_roles", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.recipes": { - "name": "recipes", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "is_default": { - "name": "is_default", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - }, - "catalogue_id": { - "name": "catalogue_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "configuration": { - "name": "configuration", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "detection_failure_reason": { - "name": "detection_failure_reason", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "status": { - "name": "status", - "type": "recipe_detection_status", - "typeSchema": "public", - "primaryKey": false, - "notNull": true, - "default": "'WAITING'" - } - }, - "indexes": { - "recipes_catalogue_idx": { - "name": "recipes_catalogue_idx", - "columns": [ - { - "expression": "catalogue_id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": { - "recipes_catalogue_id_catalogues_id_fk": { - "name": "recipes_catalogue_id_catalogues_id_fk", - "tableFrom": "recipes", - "tableTo": "catalogues", - "columnsFrom": [ - "catalogue_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.settings": { - "name": "settings", - "schema": "", - "columns": { - "key": { - "name": "key", - "type": "text", - "primaryKey": true, - "notNull": true - }, - "value": { - "name": "value", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "is_encrypted": { - "name": "is_encrypted", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "default": false - }, - "encrypted_preview": { - "name": "encrypted_preview", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "org_id": { - "name": "org_id", - "type": "integer", - "primaryKey": false, - "notNull": true - } - }, - "indexes": {}, - "foreignKeys": { - "settings_org_id_orgs_id_fk": { - "name": "settings_org_id_orgs_id_fk", - "tableFrom": "settings", - "tableTo": "orgs", - "columnsFrom": [ - "org_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "settings_key_unique": { - "name": "settings_key_unique", - "nullsNotDistinct": false, - "columns": [ - "key" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.users": { - "name": "users", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "serial", - "primaryKey": true, - "notNull": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "users_email_unique": { - "name": "users_email_unique", - "nullsNotDistinct": false, - "columns": [ - "email" - ] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": { - "public.extraction_status": { - "name": "extraction_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "COMPLETE", - "STALE", - "CANCELLED" - ] - }, - "public.log_level": { - "name": "log_level", - "schema": "public", - "values": [ - "INFO", - "ERROR" - ] - }, - "public.page_status": { - "name": "page_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "SUCCESS", - "ERROR" - ] - }, - "public.page_type": { - "name": "page_type", - "schema": "public", - "values": [ - "COURSE_DETAIL_PAGE", - "CATEGORY_LINKS_PAGE", - "COURSE_LINKS_PAGE" - ] - }, - "public.provider": { - "name": "provider", - "schema": "public", - "values": [ - "openai" - ] - }, - "public.provider_model": { - "name": "provider_model", - "schema": "public", - "values": [ - "gpt-4o" - ] - }, - "public.recipe_detection_status": { - "name": "recipe_detection_status", - "schema": "public", - "values": [ - "WAITING", - "IN_PROGRESS", - "SUCCESS", - "ERROR" - ] - }, - "public.step": { - "name": "step", - "schema": "public", - "values": [ - "FETCH_ROOT", - "FETCH_PAGINATED", - "FETCH_LINKS" - ] - }, - "public.url_pattern_type": { - "name": "url_pattern_type", - "schema": "public", - "values": [ - "page_num", - "offset" - ] - } - }, - "schemas": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "views": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} \ No newline at end of file diff --git a/server/migrations/meta/_journal.json b/server/migrations/meta/_journal.json index 5f1397c..5b3480c 100644 --- a/server/migrations/meta/_journal.json +++ b/server/migrations/meta/_journal.json @@ -19,22 +19,8 @@ { "idx": 2, "version": "7", - "when": 1738950572242, - "tag": "0002_sudden_sprite", - "breakpoints": true - }, - { - "idx": 3, - "version": "7", - "when": 1738953325622, - "tag": "0003_fixed_hydra", - "breakpoints": true - }, - { - "idx": 4, - "version": "7", - "when": 1738953356264, - "tag": "0004_sticky_onslaught", + "when": 1739203036407, + "tag": "0002_cooing_gauntlet", "breakpoints": true } ] diff --git a/server/src/data/catalogues.ts b/server/src/data/catalogues.ts index c8247cd..f125353 100644 --- a/server/src/data/catalogues.ts +++ b/server/src/data/catalogues.ts @@ -1,4 +1,4 @@ -import { desc, eq, sql, and } from "drizzle-orm"; +import { desc, eq, sql, and, inArray, } from "drizzle-orm"; import db from "../data"; import { catalogues, extractions, recipes } from "../data/schema"; @@ -37,8 +37,8 @@ export async function findCatalogueById(id: number) { export async function findCatalogueByUrl(url: string, orgIds: number[]) { return db.query.catalogues.findFirst({ where: and( - (catalogues, { eq }) => eq(catalogues.url, url), - (catalogues, { in }) => in(catalogues.orgId, orgIds), + eq(catalogues.url, url), + inArray(catalogues.orgId, orgIds), ), }); } @@ -54,10 +54,10 @@ export async function findLatestExtractionsForCatalogue(catalogueId: number) { return catExtractions.map((e) => e.extractions); } -export async function findCatalogues(orgId: number[], limit: number = 20, offset?: number) { +export async function findCatalogues(orgId: number, limit: number = 20, offset?: number) { offset = offset || 0; return db.query.catalogues.findMany({ - where: (catalogues, { eq }) => eq(catalogues.orgId, orgId), + where: eq(catalogues.orgId, orgId), limit, offset, with: { diff --git a/server/src/data/orgs.ts b/server/src/data/orgs.ts index 48362db..760c2c9 100644 --- a/server/src/data/orgs.ts +++ b/server/src/data/orgs.ts @@ -1,9 +1,10 @@ +import { eq } from 'drizzle-orm' + import db from "../data"; +import { memberships, orgs } from "./schema"; export async function findOrgsByUser(userId: number) { - return db.query.orgsUsers.findMany({ - where(fields, { eq }) { - return eq(fields?.userId, userId) - }, - }); + return db.selectDistinct() + .from(orgs) + .innerJoin(memberships, eq(memberships.userId, userId)) } \ No newline at end of file diff --git a/server/src/data/schema.ts b/server/src/data/schema.ts index 19d5913..0c6fd8f 100644 --- a/server/src/data/schema.ts +++ b/server/src/data/schema.ts @@ -206,7 +206,7 @@ const catalogues = pgTable("catalogues", { url: text("url").notNull().unique(), thumbnailUrl: text("thumbnail_url"), createdAt: timestamp("created_at").notNull().defaultNow(), - orgId: integer("recipe_id") + orgId: integer("org_id") .notNull() .references(() => orgs.id), }); @@ -493,24 +493,22 @@ const orgs = pgTable("orgs", { createdAt: timestamp("created_at").notNull().defaultNow(), }); -const userRoles = pgTable("user_roles", { - id: serial("id").primaryKey(), - name: text("name").notNull(), - description: text("description").default("null"), - createdAt: timestamp("created_at").notNull().defaultNow(), -}); +const roleTypes = pgEnum('role_type', [ + 'viewer', // Unused, reserved for future read-only use + 'member', // Default, can see organization data and run extractions + 'admin', // Same as member but can invite new or remove existing members +]); -const orgsUsers = pgTable("orgs_users_roles", { +const memberships = pgTable("memberships", { orgId: integer("org_id") .notNull() .references(() => orgs.id), userId: integer("user_id") .notNull() .references(() => users.id), - roleId: integer("user_id") - .notNull() - .references(() => userRoles.id), -}) + role: roleTypes().default('member'), +}); + export function encryptForDb(text: string) { const IV = randomBytes(16); @@ -555,5 +553,6 @@ export { settings, users, orgs, - orgsUsers, + memberships, + roleTypes, }; diff --git a/server/src/data/types.ts b/server/src/data/types.ts new file mode 100644 index 0000000..81301dd --- /dev/null +++ b/server/src/data/types.ts @@ -0,0 +1,4 @@ +import { InferSelectModel } from "drizzle-orm"; +import { orgs } from "./schema"; + +export type Organizations = InferSelectModel; \ No newline at end of file diff --git a/server/src/fastifySessionAuth.ts b/server/src/fastifySessionAuth.ts index 64400d0..5c58d07 100644 --- a/server/src/fastifySessionAuth.ts +++ b/server/src/fastifySessionAuth.ts @@ -22,7 +22,6 @@ const fastifySessionAuth: FastifyPluginCallback = fastifyPlugin( return; } - // adjust frontend to send the desired orgId const userId = req.session.get("userId"); if (userId) { const user = await findUserById(parseInt(userId)); diff --git a/server/src/trpcContext.ts b/server/src/trpcContext.ts index 8bc64ab..628a9d4 100644 --- a/server/src/trpcContext.ts +++ b/server/src/trpcContext.ts @@ -1,6 +1,7 @@ import { CreateFastifyContextOptions } from "@trpc/server/adapters/fastify"; export function createContext({ req, res }: CreateFastifyContextOptions) { - const user = (req as any).user; - return { req, res, user }; + const user = req.user; + const orgId = req.headers['x-ce-org-id']; + return { req, res, user, orgId }; } export type Context = Awaited>; From 0d8f18ba35e5f78992b474a985366d358086152e Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Wed, 12 Feb 2025 21:52:58 +0200 Subject: [PATCH 07/12] Fix missing shadcn declarations --- client/components.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/components.json b/client/components.json index ee4191d..47080d5 100644 --- a/client/components.json +++ b/client/components.json @@ -12,6 +12,9 @@ }, "aliases": { "components": "@/components", - "utils": "@/utils" + "utils": "@/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" } -} +} \ No newline at end of file From 1f6afb4be33d137de7cd4134440b8ecf83f7fc65 Mon Sep 17 00:00:00 2001 From: Alex Culea <195758113+alexculealt@users.noreply.github.com> Date: Wed, 12 Feb 2025 22:07:36 +0200 Subject: [PATCH 08/12] WIP --- client/src/components/app/dashboard.tsx | 21 +-- client/src/components/app/login.tsx | 2 +- client/src/components/app/logout.tsx | 2 +- client/src/components/app/routes.tsx | 146 +++++++++--------- .../src/components/app/selectOrganization.tsx | 75 ++++++++- client/src/components/ui/alert.tsx | 59 +++++++ client/src/main.tsx | 38 +++-- client/src/userContext.tsx | 37 ++--- client/tsconfig.json | 1 + common/types.ts | 4 +- server/src/appRouter.ts | 2 + server/src/routers/orgs.ts | 10 ++ server/tsconfig.json | 1 + 13 files changed, 272 insertions(+), 126 deletions(-) create mode 100644 client/src/components/ui/alert.tsx create mode 100644 server/src/routers/orgs.ts diff --git a/client/src/components/app/dashboard.tsx b/client/src/components/app/dashboard.tsx index 5948e59..0fd9eca 100644 --- a/client/src/components/app/dashboard.tsx +++ b/client/src/components/app/dashboard.tsx @@ -20,16 +20,11 @@ import { Link, useLocation } from "wouter"; import MenuLink from "../ui/menu-link"; import { Toaster } from "../ui/toaster"; import { TooltipProvider } from "../ui/tooltip"; -import Routes from "./routes"; +import { Routes } from "./routes"; export function Dashboard() { const [location] = useLocation(); - // IF the user is part of more than one organization or the user isStaff - // We will adjust this and any other component that has the header - // besides the user icon we will add the organization selector - - return (
@@ -37,7 +32,7 @@ export function Dashboard() {
CTDL xTRA @@ -45,23 +40,23 @@ export function Dashboard() {