fixing init from csv

Author: gubanovpm

database/init-scripts/02-store-init.sql

@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS purchases (
     id INTEGER PRIMARY KEY,
     owner_id INTEGER NOT NULL,
     buyer_id INTEGER NOT NULL,
-    ts INTEGER, -- make it zero if game not buyed yet but in cart already
+    ts INTEGER, -- make it NULL if game not buyed yet but in cart already
     game_id INTEGER NOT NULL,
 
     FOREIGN KEY (owner_id) REFERENCES users (id),

database/init-scripts/01-store-schema.sql database/init-scripts/10-store-schema.sqldatabase/init-scripts/01-store-schema.sql renamed to database/init-scripts/10-store-schema.sql

@@ -0,0 +1,78 @@
+COPY users (id, password_hash, name, balance) 
+FROM '/store-init-csvs/init-data-users.csv' 
+WITH (FORMAT CSV, HEADER, DELIMITER ',');
+
+COPY tags(id, name) 
+FROM '/store-init-csvs/init-data-tags.csv' 
+WITH (FORMAT CSV, HEADER, DELIMITER ',');
+
+COPY studios(id, name) 
+FROM '/store-init-csvs/init-data-studios.csv'
+WITH (FORMAT CSV, HEADER, DELIMITER ',');
+
+COPY games(id, name, price, description, brief, studio_id)
+FROM '/store-init-csvs/init-data-games.csv'
+WITH (FORMAT CSV, HEADER, DELIMITER ',');
+
+-- generator can generate duplicates for this categories
+-- COPY game_tags(game_id, tag_id)
+-- FROM '/store-init-csvs/init-data-game-tags.csv'
+-- WITH (FORMAT CSV, HEADER, DELIMITER ',');
+
+-- COPY purchases(id, owner_id, buyer_id, ts, game_id)
+-- FROM '/store-init-csvs/init-data-purchases.csv'
+-- WITH (FORMAT CSV, HEADER, DELIMITER ',', NULL 'NULL');
+
+-- create temporary table for games pictures to transform csv
+CREATE TEMP TABLE temp4gpictures (
+    id INTEGER PRIMARY KEY,
+    name TEXT,
+    game_id INTEGER,
+    img_type TEXT,
+    img_fmt TEXT,
+    img TEXT
+);
+
+COPY temp4gpictures 
+FROM '/store-init-csvs/init-games-pictures.csv' 
+WITH (FORMAT CSV, HEADER);
+
+INSERT INTO games_pictures (id, name, game_id, img_type, img_fmt)
+SELECT
+    s.id,
+    s.id::TEXT || 'g.' || s.img_fmt as name,
+    s.game_id,
+    s.img_type,
+    s.img_fmt
+FROM
+    temp4gpictures s
+ON CONFLICT (id) DO NOTHING;
+
+DROP TABLE temp4gpictures;
+-- end of copy games pictures
+
+-- create temporary table for profile pictures to transform csv
+CREATE TEMP TABLE temp4ppictures (
+  id INTEGER PRIMARY KEY,
+  name TEXT,
+  user_id INTEGER,
+  img_fmt TEXT,
+  img TEXT
+);
+
+COPY temp4ppictures
+FROM '/store-init-csvs/init-profile-pictures.csv'
+WITH (FORMAT CSV, HEADER);
+
+INSERT INTO profiles_pictures (id, name, user_id, img_fmt)
+SELECT
+  s.id,
+  s.id::TEXT || 'p.' || s.img_fmt as name,
+  s.user_id,
+  s.img_fmt
+FROM
+  temp4ppictures s
+ON CONFLICT (id) DO NOTHING;
+
+DROP TABLE temp4ppictures;
+-- end of copy profile pictures

database/init-scripts/15-store-init.sql

@@ -29,11 +29,10 @@ services:
       - "5000:5000"
     depends_on:
       - db
-      - bank-service
     environment:
       BANK_SERVICE_URL: http://bank-service:5001
     volumes:
-      - store-db-data:/var/store-db
+      - ./database/store-init-csvs:/tmp/init-csvs/
     networks:
       - my-network
 

docker-compose.yaml

@@ -2,3 +2,4 @@ flask
 uwsgi
 bleach
 pillow
+psycopg

store-service/requirements.txt

@@ -10,32 +10,9 @@
 from store.utility.User import User
 
 DB_PATH = os.environ.get('DB_PATH', '/var/store-db/store.sql3.db')
-SCHEMA_PATH = os.environ.get('SCHEME_PATH', '/store/store/database/store-schema.sql')
-INIT_CSV_PATH = os.environ.get('INIT_CSV_PATH', '/store/store/database/csv-init')
+INIT_CSV_PATH = os.environ.get('INIT_CSV_PATH', '/tmp/init-csvs')
 STATIC_PATH = os.environ.get('STATIC_PATH', '/store/store/static')
 
-def init_from_csv(db_file:str, csv_file:str, table_name:str):
-	conn = sqlite3.connect(db_file)
-	cursor = conn.cursor()
-	try:
-		with open(csv_file, 'r', encoding='UTF-8') as file:
-			reader = csv.reader(file)
-			headers = next(reader)
-			query = f"""
-				INSERT OR IGNORE INTO {table_name} ({','.join(headers)})
-				VALUES ({','.join(['?'] * len(headers))});
-			"""
-			for row in reader:
-				data = [None if x == 'NULL' else x for x in row]
-				cursor.execute(query, data)
-			conn.commit()
-			conn.close()
-	except Exception as e:
-		print(f'error:{e}')
-		conn.rollback()
-		conn.close()
-		raise ValueError(str(e)) from e
-
 def init_pics_from_csv(db_file:str, csv_file:str, table_name:str, save_to:str):
 	conn = sqlite3.connect(db_file)
 	cursor = conn.cursor()
@@ -79,20 +56,6 @@ def execute_sql_file(conn, sql_file):
 		conn.rollback()
 
 def init_database():
-	schema_file = SCHEMA_PATH
-	conn = sqlite3.connect(DB_PATH)
-	if conn:
-		execute_sql_file(conn, schema_file)
-		conn.commit()
-	conn.close()
-	init_csvs = [
-		{'file': f'{INIT_CSV_PATH}/init-data-users.csv',     'table':'users'},
-		{'file': f'{INIT_CSV_PATH}/init-data-studios.csv',   'table':'studios'},
-		{'file': f'{INIT_CSV_PATH}/init-data-games.csv',     'table':'games'},
-		{'file': f'{INIT_CSV_PATH}/init-data-tags.csv',      'table':'tags'},
-		{'file': f'{INIT_CSV_PATH}/init-data-game-tags.csv', 'table':'game_tags'},
-		{'file': f'{INIT_CSV_PATH}/init-data-purchases.csv', 'table':'purchases'}
-	]
 	init_pics_csvs = [{
 		'file': f'{INIT_CSV_PATH}/init-profile-pictures.csv', 
 		'table':'profiles_pictures',