new tests

api_gateway/requirements.txt

@@ -15,4 +15,5 @@ pytest-cov
 pytest-mock>=3.0.0
 pytest-dependency
 coverage
-confluent-kafka==2.2.0
+confluent-kafka==2.2.0
+clickhouse-sqlalchemy

api_gateway/tests/test_kafka.py

@@ -2,8 +2,18 @@
 import json
 from confluent_kafka import Consumer, KafkaException
 import time
-from datetime import datetime
 import requests
+from sqlalchemy import create_engine, select
+from clickhouse_sqlalchemy import make_session
+from statistic_service.db.clickhouse_models import Event, EventType
+
+
+@pytest.fixture(scope="module")
+def clickhouse_session():
+    engine = create_engine('clickhouse://default:password@clickhouse:8123/default')
+    session = make_session(engine)
+    yield session
+    session.close()
 
 
 @pytest.fixture(scope="module")
@@ -58,29 +68,24 @@ def test_user_registration_event(kafka_consumer, test_user):
     pytest.fail("Event not received in Kafka within timeout")
 
 
-def test_post_view_event(kafka_consumer, test_user):
+def test_post_view_event(kafka_consumer, test_user, clickhouse_session):
     login_response = requests.post(
         'http://api_gateway:8080/api/v1/login',
-        json={
-            "login": test_user["login"],
-            "password": test_user["password"]
-        }
+        json={"login": test_user["login"], "password": test_user["password"]}
     )
     assert login_response.status_code == 200
     token = login_response.json()["token"]
 
     post_response = requests.post(
         'http://api_gateway:8080/api/v1/posts',
         headers={"Authorization": token},
-        json={
-            "title": "Kafka Test Post",
-            "description": "Post for Kafka testing",
-            "is_private": False
-        }
+        json={"title": "Kafka Test Post", "description": "Post for Kafka testing", "is_private": False}
     )
     assert post_response.status_code == 201
     post_id = post_response.json()["post_id"]
+
     kafka_consumer.subscribe(['post_views'])
+
     view_response = requests.post(
         f'http://api_gateway:8080/api/v1/posts/{post_id}/view',
         headers={"Authorization": token}
@@ -98,38 +103,36 @@ def test_post_view_event(kafka_consumer, test_user):
         event = json.loads(msg.value())
         if event.get("post_id") == str(post_id):
             assert event['event_type'] == 'post_viewed'
-            assert 'user_id' in event
-            assert 'post_id' in event
-            assert datetime.fromisoformat(event['timestamp']).tzinfo is None
-            assert msg.key() == str(post_id).encode('utf-8')
+            time.sleep(1)
+            stmt = select(Event).where(
+                Event.post_id == str(post_id),
+                Event.event_type == EventType.VIEW
+            )
+            result = clickhouse_session.execute(stmt).fetchone()
+            assert result is not None, "Event not found in ClickHouse"
             return
 
-    pytest.fail("Event not received in Kafka within timeout")
+    pytest.fail("Kafka event not received within timeout")
 
 
-def test_post_like_event(kafka_consumer, test_user):
+def test_post_like_event(kafka_consumer, test_user, clickhouse_session):
     login_response = requests.post(
         'http://api_gateway:8080/api/v1/login',
-        json={
-            "login": test_user["login"],
-            "password": test_user["password"]
-        }
+        json={"login": test_user["login"], "password": test_user["password"]}
     )
     assert login_response.status_code == 200
     token = login_response.json()["token"]
 
     post_response = requests.post(
         'http://api_gateway:8080/api/v1/posts',
         headers={"Authorization": token},
-        json={
-            "title": "Kafka Like Test",
-            "description": "Post for like testing",
-            "is_private": False
-        }
+        json={"title": "Kafka Like Test", "description": "Post for like testing", "is_private": False}
     )
     assert post_response.status_code == 201
     post_id = post_response.json()["post_id"]
+
     kafka_consumer.subscribe(['post_likes'])
+
     like_response = requests.post(
         f'http://api_gateway:8080/api/v1/posts/{post_id}/like',
         headers={"Authorization": token}
@@ -147,39 +150,36 @@ def test_post_like_event(kafka_consumer, test_user):
         event = json.loads(msg.value())
         if event.get("post_id") == str(post_id):
             assert event['event_type'] == 'post_liked'
-            assert 'user_id' in event
-            assert 'post_id' in event
-            assert datetime.fromisoformat(event['timestamp']).tzinfo is None
-            assert msg.key() == str(post_id).encode('utf-8')
+            time.sleep(1)
+            stmt = select(Event).where(
+                Event.post_id == str(post_id),
+                Event.event_type == EventType.LIKE
+            )
+            result = clickhouse_session.execute(stmt).fetchone()
+            assert result is not None, "Like event not found in ClickHouse"
             return
 
-    pytest.fail("Event not received in Kafka within timeout")
+    pytest.fail("Kafka event not received within timeout")
 
 
-def test_post_comment_event(kafka_consumer, test_user):
+def test_post_comment_event(kafka_consumer, test_user, clickhouse_session):
     login_response = requests.post(
         'http://api_gateway:8080/api/v1/login',
-        json={
-            "login": test_user["login"],
-            "password": test_user["password"]
-        }
+        json={"login": test_user["login"], "password": test_user["password"]}
     )
     assert login_response.status_code == 200
     token = login_response.json()["token"]
 
     post_response = requests.post(
         'http://api_gateway:8080/api/v1/posts',
         headers={"Authorization": token},
-        json={
-            "title": "Kafka Comment Test",
-            "description": "Post for comment testing",
-            "is_private": False
-        }
+        json={"title": "Kafka Comment Test", "description": "Post for comment testing", "is_private": False}
     )
     assert post_response.status_code == 201
     post_id = post_response.json()["post_id"]
 
     kafka_consumer.subscribe(['post_comments'])
+
     comment_text = "Test comment for Kafka"
     comment_response = requests.post(
         f'http://api_gateway:8080/api/v1/posts/{post_id}/comment',
@@ -200,12 +200,13 @@ def test_post_comment_event(kafka_consumer, test_user):
         event = json.loads(msg.value())
         if event.get("post_id") == str(post_id) and event.get("comment_id") == str(comment_id):
             assert event['event_type'] == 'post_commented'
-            assert 'user_id' in event
-            assert 'post_id' in event
-            assert 'comment_id' in event
-            assert datetime.fromisoformat(event['timestamp']).tzinfo is None
-            assert msg.key() == str(post_id).encode('utf-8')
-            assert event.get("text_preview") == comment_text[:100]
+            time.sleep(1)
+            stmt = select(Event).where(
+                Event.post_id == str(post_id),
+                Event.event_type == EventType.COMMENT
+            )
+            result = clickhouse_session.execute(stmt).fetchone()
+            assert result is not None, "Comment event not found in ClickHouse"
             return
 
-    pytest.fail("Event not received in Kafka within timeout")
+    pytest.fail("Kafka event not received within timeout")

docker-compose.yml

@@ -15,6 +15,7 @@ services:
     environment:
       - POST_SERVICE_HOST=post_service
       - POST_SERVICE_PORT=50051
+      - USER_SERVICE_URL=http://user_service:5000
       - PYTHONPATH=/app:/app/proto
 
   user_service:
@@ -65,54 +66,66 @@ services:
       timeout: 5s
       retries: 10
 
-  #  tests:
-  #    build:
-  #      context: .
-  #      dockerfile: api_gateway/Dockerfile
-  #    volumes:
-  #      - ./api_gateway:/app/api_gateway
-  #      - ./proto:/app/proto
-  #      - ./user_service:/app/user_service
-  #      - ./post_service:/app/post_service
-  #      - ./broker:/app/broker
-  #    command: >
-  #      sh -c "
-  #        echo 'Ожидание готовности сервисов...' &&
-  #        while ! nc -z user_service 5000; do sleep 2; done &&
-  #        while ! nc -z post_service 50051; do sleep 2; done &&
-  #        while ! curl -f http://user_service:5000/health; do sleep 2; done &&
-  #        while ! nc -z kafka 9092; do sleep 2; done &&
-  #
-  #        export PYTHONPATH=/app:/app/user_service:/app/post_service:/app/api_gateway:/app/broker:/app/proto &&
-  #
-  #        echo 'Запуск тестов api_gateway...' &&
-  #        cd /app &&
-  #        pytest api_gateway/tests/test_user.py api_gateway/tests/test_post.py api_gateway/tests/test_kafka.py -v || exit 1 &&
-  #
-  #        echo 'Запуск тестов user_service...' &&
-  #        cd /app/user_service &&
-  #        pytest tests/test_unit_models.py tests/test_unit_user_service.py tests/test_unit_validators.py -v || exit 1 &&
-  #
-  #        echo 'Запуск тестов post_service...' &&
-  #        cd /app/post_service &&
-  #        pytest tests/test_unit_models.py tests/test_unit_post_service.py tests/test_unit_kafka_events.py -v || exit 1
-  #      "
-  #    depends_on:
-  #      - api_gateway
-  #      - user_service
-  #      - post_service
-  #      - db
-  #      - kafka
-  #    networks:
-  #      - social-network
-  #    environment:
-  #      - PYTHONPATH=/app:/app/user_service:/app/post_service:/app/api_gateway:/app/broker:/app/proto
-  #      - FLASK_ENV=testing
-  #      - TZ=Europe/Moscow
-  #      - POST_SERVICE_HOST=post_service
-  #      - POST_SERVICE_PORT=50051
-  #      - USER_SERVICE_URL=http://user_service:5000
-  #      - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
+  tests:
+    build:
+      context: .
+      dockerfile: api_gateway/Dockerfile
+    volumes:
+      - ./api_gateway:/app/api_gateway
+      - ./proto:/app/proto
+      - ./user_service:/app/user_service
+      - ./post_service:/app/post_service
+      - ./statistic_service:/app/statistic_service
+      - ./broker:/app/broker
+      - ./e2e_tests:/app/e2e_tests
+    command: >
+      sh -c "
+        echo 'Ожидание готовности сервисов...' &&
+        while ! nc -z user_service 5000; do sleep 2; done &&
+        while ! nc -z post_service 50051; do sleep 2; done &&
+        while ! curl -f http://user_service:5000/health; do sleep 2; done &&
+        while ! nc -z kafka 9092; do sleep 2; done &&
+        while ! nc -z kafka 9092; do sleep 2; done &&
+
+        export PYTHONPATH=/app:/app/user_service:/app/post_service:/app/api_gateway:/app/broker:/app/proto:/app/statistic_service:/app/e2e_tests &&
+
+        echo 'Запуск тестов user_service...' &&
+        cd /app/user_service &&
+        pytest tests/test_unit_models.py tests/test_unit_user_service.py tests/test_unit_validators.py -v || exit 1 &&
+
+        echo 'Запуск тестов post_service...' &&
+        cd /app/post_service &&
+        pytest tests/test_unit_models.py tests/test_unit_post_service.py tests/test_unit_kafka_events.py -v || exit 1 &&
+
+        echo 'Запуск тестов statistic_service...' &&
+        cd /app/statistic_service &&
+        pytest tests/test_statistic_db.py tests/test_statistic_service.py tests/test_unit_models.py || exit 1 &&
+
+        echo 'Запуск тестов Kafka...' &&
+        cd /app/api_gateway &&
+        pytest tests/test_kafka.py || exit 1 &&
+
+        echo 'Запуск e2e тестов...' &&
+        cd /app/e2e_tests &&
+        pytest test_full_flow.py || exit 1
+      "
+    depends_on:
+      - api_gateway
+      - user_service
+      - post_service
+      - db
+      - kafka
+      - kafka-init
+    networks:
+      - social-network
+    environment:
+      - PYTHONPATH=/app:/app/user_service:/app/post_service:/app/api_gateway:/app/broker:/app/proto
+      - FLASK_ENV=testing
+      - TZ=Europe/Moscow
+      - POST_SERVICE_HOST=post_service
+      - POST_SERVICE_PORT=50051
+      - USER_SERVICE_URL=http://user_service:5000
+      - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
 
   zookeeper:
     image: confluentinc/cp-zookeeper:7.3.0
@@ -168,6 +181,7 @@ services:
         kafka-topics --bootstrap-server kafka:9092 --create --topic post_views --partitions 1 --replication-factor 1;
         kafka-topics --bootstrap-server kafka:9092 --create --topic post_likes --partitions 1 --replication-factor 1;
         kafka-topics --bootstrap-server kafka:9092 --create --topic post_comments --partitions 1 --replication-factor 1;
+        kafka-topics --bootstrap-server kafka:9092 --create --topic user_registrations --partitions 1 --replication-factor 1;
         echo 'Topics created successfully';
       "
     networks: