From ce96a662fa564a8b39f33f4050002f617c7383ff Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev Date: Wed, 7 May 2025 12:03:47 +0300 Subject: [PATCH 01/61] adding pip mirror config --- Dockerfile | 5 +++-- pip.conf | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 pip.conf diff --git a/Dockerfile b/Dockerfile index 27d3806..3520f4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ ENV PYTHONUNBUFFERED=1 # Enables env file ENV APP_ENV=development - +#add pyppi mirror to config +COPY pip.conf /etc/xdg/pip/pip.conf # Install pip requirements COPY requirements.txt . RUN python -m pip install -r requirements.txt @@ -22,4 +23,4 @@ WORKDIR /app COPY . /app # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug -CMD ["gunicorn", "--bind", "0.0.0.0:80", "-k", "uvicorn.workers.UvicornWorker", "--workers", "4", "app.main:app"] \ No newline at end of file +CMD ["gunicorn", "--bind", "0.0.0.0:80", "-k", "uvicorn.workers.UvicornWorker", "--workers", "4", "app.main:app"] diff --git a/pip.conf b/pip.conf new file mode 100644 index 0000000..774069a --- /dev/null +++ b/pip.conf @@ -0,0 +1,4 @@ +[global] +index-url=http://10.32.1.108:3141/root/pypi/+simple/ +trusted-host=10.32.1.108 +timeout=120 From 58186b9f373a334ba5835b53ebe99bc427b5478f Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev Date: Tue, 13 May 2025 13:33:35 +0300 Subject: [PATCH 02/61] adding build and deploy workflow --- .github/workflows/build_and_deploy.yml | 46 ++++++++++++++++++++++++++ docker-compose.actions.yml | 10 ++++++ 2 files changed, 56 insertions(+) create mode 100644 .github/workflows/build_and_deploy.yml create mode 100644 docker-compose.actions.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml new file mode 100644 index 0000000..7a6e1e9 --- /dev/null +++ b/.github/workflows/build_and_deploy.yml @@ -0,0 +1,46 @@ +name: build_and_deploy +on: workflow_dispatch +env: + IMAGE_NAME: ${{secrets.REGISTRY}}/object_effects + CONTAINER_NAME: object_effects + +jobs: + build: + runs-on: 65_runner + outputs: + now: ${{steps.date.outputs.NOW}} + steps: + - name: Set current date as env variable + id: date + run: echo "NOW=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT + - name: checkout + uses: actions/checkout@v4 + - name: copy_env + env: + ENV_PATH: ${{secrets.ENV_PATH}} + run: cp "$ENV_PATH"/.env.development ./ + - name: build + env: + NOW: ${{steps.date.outputs.now}} + run: docker build -t "$IMAGE_NAME":"$NOW" . + - name: push_to_registry + env: + NOW: ${{steps.date.outputs.now}} + run: docker push "$IMAGE_NAME":"$NOW" + stop_container: + runs-on: self-hosted + needs: build + steps: + - name: stop_container + run: docker rm -f "$CONTAINER_NAME" + run_container: + runs-on: self-hosted + needs: [build, stop_container] + env: + NOW: ${{needs.build.outputs.now}} + steps: + - name: set env + run: echo "IMAGE=$IMAGE_NAME:$NOW" >> $GITHUB_ENV + - name: run +# run: docker run -d --name "$CONTAINER_NAME" --env-file ./.env.development -p 8210:8000 "$IMAGE_NAME":"$NOW" + run: docker compose -f docker-compose.actions.yml up -d diff --git a/docker-compose.actions.yml b/docker-compose.actions.yml new file mode 100644 index 0000000..ffeb3c4 --- /dev/null +++ b/docker-compose.actions.yml @@ -0,0 +1,10 @@ +services: + object_effects: + image: ${IMAGE} + container_name: ${CONTAINER_NAME} + ports: + - 5080:80 + env_file: + - .env.development + restart: always + From f980cd8c968c8478b8805dbf6b8d1fadd2d8d8a3 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 26 May 2025 15:31:19 +0300 Subject: [PATCH 03/61] fix(effects_api_gateway): - added no normative for year error --- app/common/api_handler/api_handler.py | 8 ++++++++ app/effects/effects_service.py | 1 + app/effects/modules/effects_api_gateway.py | 22 ++++++++++++++-------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index 6f430d1..4f8bc08 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -89,7 +89,15 @@ async def get( params=params ) as response: result = await self._check_response_status(response) + if isinstance(result, list): + return result + elif isinstance(result, dict): + return result if not result: + if isinstance(result, list): + return result + elif isinstance(result, dict): + return result return await self.get( endpoint_url=endpoint_url, headers=headers, diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index de44c66..d9ee444 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -61,6 +61,7 @@ async def _get_pivot( result["median_index_scenario_project"] = int(effects[effects["is_project"]]["index_scenario_project"].median()) return result + # ToDo Add population retrievement by year # ToDo Split function # ToDo Rewrite to context ids normal handling async def calculate_effects( diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 34568ab..a16e36a 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -51,20 +51,26 @@ async def get_service_normative( return service_type else: raise http_exception( - status_code=400, - msg="Service type normative not found", - _input={"service_type_id": service_type_id}, + status_code=404, + msg="Service type normative not found in urban_db. ", + _input={ + "year": year, + "service_type_id": service_type_id, + }, _detail={ "Available service ids": [service_type["id"] for service_type in response] }, ) raise http_exception( - status_code=400, - msg="Service type normative not found", - _input={"service_type_id": service_type_id}, + status_code=404, + msg="Service type normative not found in urban_db. Try another year or service type.", + _input={ + "year": year, + "service_type_id": service_type_id, + }, _detail={ - "Available service ids": [service_type["service_type"]["id"] for service_type in response] - } + "Available service ids": [service_type["id"] for service_type in response] + }, ) @staticmethod From 410df8b873f4c5595a9a95f73c98d6d7f803968c Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 26 May 2025 16:29:50 +0300 Subject: [PATCH 04/61] fix(effects_api_gateway): - added normative extraction from context ter if only one --- app/effects/effects_service.py | 1 + app/effects/modules/effects_api_gateway.py | 30 +++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index d9ee444..d45d0bc 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -84,6 +84,7 @@ async def calculate_effects( ) normative_data = await effects_api_gateway.get_service_normative( territory_id=project_data["territory"]["id"], + context_ids=project_data["properties"]["context"], service_type_id=effects_params.service_type_id, year=effects_params.year, ) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index a16e36a..08ff8a5 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -10,6 +10,7 @@ class EffectsAPIGateway: @staticmethod async def get_service_normative( territory_id: int, + context_ids: list[int], service_type_id: int, year: int = 2024, ) -> dict[str, int | str]: @@ -17,6 +18,7 @@ async def get_service_normative( Function retrieves normative data from urban_api Args: territory_id: territory id to get normative from + context_ids: context id to get normative from service_type_id: service to get normative from year: year to get normative from Returns: @@ -25,12 +27,22 @@ async def get_service_normative( 400, http exception id not found """ - response = await urban_api_handler.get( - f"/api/v1/territory/{territory_id}/normatives", - params={ - "year": year, - } - ) + if len(context_ids) == 1: + response = await urban_api_handler.get( + f"/api/v1/territory/{context_ids[0]}/normatives", + params={ + "year": year, + } + ) + request_ter_id = context_ids[0] + else: + response = await urban_api_handler.get( + f"/api/v1/territory/{territory_id}/normatives", + params={ + "year": year, + } + ) + request_ter_id = territory_id for service_type in response: if service_type["service_type"]["id"] == service_type_id: if normative_value:=service_type["radius_availability_meters"]: @@ -54,6 +66,9 @@ async def get_service_normative( status_code=404, msg="Service type normative not found in urban_db. ", _input={ + "territory_id": territory_id, + "context_ids": context_ids, + "request_ter_id": request_ter_id, "year": year, "service_type_id": service_type_id, }, @@ -65,6 +80,9 @@ async def get_service_normative( status_code=404, msg="Service type normative not found in urban_db. Try another year or service type.", _input={ + "territory_id": territory_id, + "context_ids": context_ids, + "request_ter_id": request_ter_id, "year": year, "service_type_id": service_type_id, }, From d70e0514152cfd2c1a471f62fe1ae66c9999a8c0 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 26 May 2025 16:36:47 +0300 Subject: [PATCH 05/61] fix(docker-compose.actions): - returned actions-compose --- docker-compose.actions.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docker-compose.actions.yml diff --git a/docker-compose.actions.yml b/docker-compose.actions.yml new file mode 100644 index 0000000..ffeb3c4 --- /dev/null +++ b/docker-compose.actions.yml @@ -0,0 +1,10 @@ +services: + object_effects: + image: ${IMAGE} + container_name: ${CONTAINER_NAME} + ports: + - 5080:80 + env_file: + - .env.development + restart: always + From fcd37d62371f7a0b61c66e608bae1b45c0240b73 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 27 May 2025 14:13:23 +0300 Subject: [PATCH 06/61] chore(ToDo): - added todo for code response --- app/effects/effects_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index d45d0bc..cd91b78 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -110,6 +110,7 @@ async def calculate_effects( service_type_id=effects_params.service_type_id, ) if context_services.empty: + #ToDo Revise to another code raise http_exception( status_code=404, msg="No services of {service_type_id} type found in context", From b5af7e871c35c4e1b5088743e0b782fadac5d35a Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 28 May 2025 13:50:10 +0300 Subject: [PATCH 07/61] feat(effects_api_gateway): - changed normative to last year - removed year from requests params - services por people normative added as simple provision (demand=population) --- app/effects/dto/effects_dto.py | 4 -- app/effects/effects_service.py | 1 - app/effects/modules/data_restorator.py | 7 +- app/effects/modules/effects_api_gateway.py | 82 ++++++++++------------ 4 files changed, 44 insertions(+), 50 deletions(-) diff --git a/app/effects/dto/effects_dto.py b/app/effects/dto/effects_dto.py index 57052f5..087a604 100644 --- a/app/effects/dto/effects_dto.py +++ b/app/effects/dto/effects_dto.py @@ -8,10 +8,6 @@ class EffectsDTO(BaseModel): project_id: int = Field(..., examples=[72], description="Project ID") scenario_id: int = Field(..., examples=[192], description="Scenario ID") service_type_id: int = Field(..., examples=[7], description="Service type ID") - year: Optional[int] = Field( - default=2024, - examples=[2024], - description="Year for data retrieval") target_population: Optional[int] = Field( default=None, examples=[200], diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index cd91b78..16a6195 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -86,7 +86,6 @@ async def calculate_effects( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=effects_params.service_type_id, - year=effects_params.year, ) context_population = await effects_api_gateway.get_context_population( territory_ids_list=project_data["properties"]["context"] diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index 1e7a95f..2a4f7a2 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -132,16 +132,19 @@ def restore_demands( target_demand=target_total_demand ) return buildings + elif service_normative_type == "unit": + buildings["demand"] = buildings["population"].astype(int).copy() + return buildings else: raise http_exception( - status_code=400, + status_code=500, msg="Service demand normative not found", _input={ "service_normative_type": service_normative_type, }, _detail={ "available_demand_type": [ - "num", "capacity" + "unit", "capacity" ] } ) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 08ff8a5..674a08b 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -1,6 +1,8 @@ import asyncio import geopandas as gpd +import numpy as np +import pandas as pd from app.dependencies import urban_api_handler, http_exception @@ -12,7 +14,6 @@ async def get_service_normative( territory_id: int, context_ids: list[int], service_type_id: int, - year: int = 2024, ) -> dict[str, int | str]: """ Function retrieves normative data from urban_api @@ -30,52 +31,49 @@ async def get_service_normative( if len(context_ids) == 1: response = await urban_api_handler.get( f"/api/v1/territory/{context_ids[0]}/normatives", - params={ - "year": year, - } ) request_ter_id = context_ids[0] else: response = await urban_api_handler.get( f"/api/v1/territory/{territory_id}/normatives", - params={ - "year": year, - } ) request_ter_id = territory_id - for service_type in response: - if service_type["service_type"]["id"] == service_type_id: - if normative_value:=service_type["radius_availability_meters"]: - service_type["normative_value"] = normative_value - service_type["normative_type"] = "dist" - if service_type.get("services_per_1000_normative"): - service_type["capacity_type"] = "unit" - else: - service_type["capacity_type"] = "capacity" - return service_type - elif normative_value:=service_type["time_availability_minutes"]: - service_type["normative_value"] = normative_value - service_type["normative_type"] = "time" - if service_type.get("services_per_1000_normative"): - service_type["capacity_type"] = "unit" - else: - service_type["capacity_type"] = "capacity" - return service_type + response_df = pd.DataFrame.from_records(response) + response_df["service_type_id"] = response_df["service_type"].apply(lambda x: x["id"]) + service_type = response_df[(response_df["year"] == response_df["year"].max()) & ( + response_df["service_type_id"] == service_type_id)].iloc[0].to_dict() + + if service_type["service_type"]["id"] == service_type_id: + if not pd.isna(service_type["radius_availability_meters"]): + service_type["normative_value"] = service_type["radius_availability_meters"] + service_type["normative_type"] = "dist" + if service_type.get("services_per_1000_normative"): + service_type["capacity_type"] = "unit" else: - raise http_exception( - status_code=404, - msg="Service type normative not found in urban_db. ", - _input={ - "territory_id": territory_id, - "context_ids": context_ids, - "request_ter_id": request_ter_id, - "year": year, - "service_type_id": service_type_id, - }, - _detail={ - "Available service ids": [service_type["id"] for service_type in response] - }, - ) + service_type["capacity_type"] = "capacity" + return service_type + elif not pd.isna(service_type["time_availability_minutes"]): + service_type["normative_value"] = service_type["time_availability_minutes"] + service_type["normative_type"] = "time" + if service_type.get("services_per_1000_normative"): + service_type["capacity_type"] = "unit" + else: + service_type["capacity_type"] = "capacity" + return service_type + else: + raise http_exception( + status_code=404, + msg="Service type normative not found in urban_db. ", + _input={ + "territory_id": territory_id, + "context_ids": context_ids, + "request_ter_id": request_ter_id, + "service_type_id": service_type_id, + }, + _detail={ + "Available service ids": [service_type["id"] for service_type in response] + }, + ) raise http_exception( status_code=404, msg="Service type normative not found in urban_db. Try another year or service type.", @@ -83,11 +81,10 @@ async def get_service_normative( "territory_id": territory_id, "context_ids": context_ids, "request_ter_id": request_ter_id, - "year": year, "service_type_id": service_type_id, }, _detail={ - "Available service ids": [service_type["id"] for service_type in response] + "Available service ids": response_df["service_type_id"].to_list() }, ) @@ -228,7 +225,7 @@ async def get_scenario_population_data( } ) - if (value:=population[0]["value"]) < 1: + if (value := population[0]["value"]) < 1: return None return value @@ -255,5 +252,4 @@ async def get_context_population( return sum([item[0]["value"] for item in result]) - effects_api_gateway = EffectsAPIGateway() From c051fb55df56bce5bb3c838f272f6652c5575853 Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev Date: Wed, 28 May 2025 14:42:51 +0300 Subject: [PATCH 08/61] fixing workflow --- .github/workflows/build_and_deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 7a6e1e9..0c31548 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -41,6 +41,8 @@ jobs: steps: - name: set env run: echo "IMAGE=$IMAGE_NAME:$NOW" >> $GITHUB_ENV + - name: checkout + uses: actions/checkout@v4 - name: run # run: docker run -d --name "$CONTAINER_NAME" --env-file ./.env.development -p 8210:8000 "$IMAGE_NAME":"$NOW" run: docker compose -f docker-compose.actions.yml up -d From 535eaf02881745e57351949ca1afede23ef9f628 Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev Date: Wed, 28 May 2025 14:46:10 +0300 Subject: [PATCH 09/61] fixing workflow --- .github/workflows/build_and_deploy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 0c31548..3cc2b01 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -43,6 +43,10 @@ jobs: run: echo "IMAGE=$IMAGE_NAME:$NOW" >> $GITHUB_ENV - name: checkout uses: actions/checkout@v4 + - name: copy_env + env: + ENV_PATH: ${{secrets.ENV_PATH}} + run: cp "$ENV_PATH"/.env.development ./ - name: run # run: docker run -d --name "$CONTAINER_NAME" --env-file ./.env.development -p 8210:8000 "$IMAGE_NAME":"$NOW" run: docker compose -f docker-compose.actions.yml up -d From 2cda2cd323057a789a66c32bdd153e2a2a1d793b Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Thu, 29 May 2025 01:58:34 +0300 Subject: [PATCH 10/61] fix(effects_api_gateway): - fixed nan check for normative request --- app/effects/modules/effects_api_gateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 44a745a..be929dc 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -48,7 +48,7 @@ async def get_service_normative( if not pd.isna(service_type["radius_availability_meters"]): service_type["normative_value"] = service_type["radius_availability_meters"] service_type["normative_type"] = "dist" - if service_type.get("services_per_1000_normative"): + if not pd.isna(service_type.get("services_per_1000_normative")): service_type["capacity_type"] = "unit" else: service_type["capacity_type"] = "capacity" @@ -56,7 +56,7 @@ async def get_service_normative( elif not pd.isna(service_type["time_availability_minutes"]): service_type["normative_value"] = service_type["time_availability_minutes"] service_type["normative_type"] = "time" - if service_type.get("services_per_1000_normative"): + if not pd.isna(service_type.get("services_per_1000_normative")): service_type["capacity_type"] = "unit" else: service_type["capacity_type"] = "capacity" From 0ec32d640a3e6444a3b7ff7930416c367ca9481d Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 16:50:03 +0300 Subject: [PATCH 11/61] chore(effects_api_gateway): - renamed parameter in indicators request --- app/effects/modules/effects_api_gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index be929dc..4124056 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -222,7 +222,7 @@ async def get_scenario_population_data( population = await urban_api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/indicators_values", params={ - "indicators_ids": 1, + "indicator_ids": 1, } ) From 71780b7114256aaa0990813d4a9e5b46903a9edd Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 17:51:01 +0300 Subject: [PATCH 12/61] fix(effects_api_gateway): - normative retrievement changed to last available year --- app/effects/modules/effects_api_gateway.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 4124056..48d2a6d 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -22,7 +22,6 @@ async def get_service_normative( territory_id: territory id to get normative from context_ids: context id to get normative from service_type_id: service to get normative from - year: year to get normative from Returns: dict[str, int | str]: normative data with normative value and normative type (Literal["time", "dist"]) Raises: @@ -41,8 +40,8 @@ async def get_service_normative( request_ter_id = territory_id response_df = pd.DataFrame.from_records(response) response_df["service_type_id"] = response_df["service_type"].apply(lambda x: x["id"]) - service_type = response_df[(response_df["year"] == response_df["year"].max()) & ( - response_df["service_type_id"] == service_type_id)].iloc[0].to_dict() + service_type = response_df[response_df["service_type_id"] == service_type_id].copy() + service_type = service_type[service_type["year"] == service_type["year"].max()].iloc[0].to_dict() if service_type["service_type"]["id"] == service_type_id: if not pd.isna(service_type["radius_availability_meters"]): From ca5ac0dd8e4fe1094b91b98cb4e887e4ae406950 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 17:59:30 +0300 Subject: [PATCH 13/61] fix(data_restorator): - storeys count ensured as int --- app/effects/modules/data_restorator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index 2a4f7a2..11a5a3a 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -70,6 +70,7 @@ def _restore_population( target_population = self._restore_target_population(buildings) local_crs = buildings.estimate_utm_crs() buildings = buildings.to_crs(local_crs) + buildings["storeys_count"] = buildings["storeys_count"].apply(lambda x: int(round(x))) buildings["living_area"] = buildings.area * buildings["storeys_count"] * 0.8 buildings["living_area"] = buildings["living_area"].astype(int) balanced_buildings = get_balanced_buildings( From 51137d2c04fda32f7623deaa8ed9791240143b91 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 18:33:42 +0300 Subject: [PATCH 14/61] fix(effects_service): - added exception handling - fixed log retrievement - requirements updated --- app/dependencies.py | 3 +-- app/effects/effects_service.py | 27 ++++++++++++++++++++++++++ app/main.py | 34 ++++++++++++++++++++++++++------- requirements.txt | Bin 510 -> 474 bytes 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 39a28a2..aa25afa 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -1,5 +1,4 @@ import sys -from datetime import datetime from loguru import logger from iduconfig import Config @@ -22,7 +21,7 @@ config = Config() logger.add( - f"{config.get('LOGS_FILE')}.log", + ".log", format=log_format, level="INFO", ) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index b907161..6b8092d 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -4,6 +4,7 @@ import geopandas as gpd import pandas as pd from loguru import logger +from fastapi import HTTPException from app.dependencies import http_exception from .dto.effects_dto import EffectsDTO @@ -13,6 +14,7 @@ attribute_parser, matrix_builder, objectnat_calculator ) +from .shemas.effects_base_schema import EffectsSchema class EffectsService: @@ -260,5 +262,30 @@ async def calculate_effects( } return result + async def handle_effects_calculation(self, effects_params: EffectsDTO) -> EffectsSchema: + """ + Function handles errors from effects calculations. + Args: + effects_params (EffectsDTO): + Returns: + EffectsSchema: with inf + Raises: + Any from Urban API + 500, if internal server error + """ + try: + result = await self.calculate_effects(effects_params) + return EffectsSchema(**result) + except HTTPException as http_e: + raise http_e + except Exception as e: + logger.exception(e) + raise http_exception( + 500, + msg="Error during effects calculation", + _input=effects_params.__dict__, + _detail={"error": e.__str__()} + ) + effects_service = EffectsService() diff --git a/app/main.py b/app/main.py index 4194241..5c5c1b4 100644 --- a/app/main.py +++ b/app/main.py @@ -1,9 +1,8 @@ -import aiofiles from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, FileResponse -from .dependencies import config +from .dependencies import config, http_exception from .effects.effects_controller import effects_router @@ -31,10 +30,31 @@ async def read_root(): return {"status": "OK"} @app.get("/logs") -async def read_logs(): - async with aiofiles.open(config.get("LOGS_FILE")) as logs_file: - logs = await logs_file.read() - return logs[-1:-10000] +async def get_logs(): + """ + Get logs file from app + """ + + try: + return FileResponse( + f"{config.get('LOG_FILE')}.log", + media_type='application/octet-stream', + filename=f"ObjectEffects.log", + ) + except FileNotFoundError as e: + raise http_exception( + status_code=404, + msg="Log file not found", + _input={"lof_file_name": f".log"}, + _detail={"error": e.__str__()} + ) + except Exception as e: + raise http_exception( + status_code=500, + msg="Internal server error during reading logs", + _input={"lof_file_name": f"{config.get('LOG_FILE')}.log"}, + _detail={"error": e.__str__()} + ) app.include_router(effects_router) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e478cf4509280b07a00df67268fbe130a0e96abb..428b92aa313e74bada2017eea6778b4575716bab 100644 GIT binary patch delta 11 Scmeyze2bas|G$lDHy8mTx&@8^ delta 47 zcmcb`{Eu1f|Gz|rOon`hG$5J7kjhZZP{&})V8md;pvPbc#0Ct!3|tHwnQkxw0797x A&j0`b From b74b0bd7ada5d853763d088129e625d254c66b5b Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 18:35:58 +0300 Subject: [PATCH 15/61] fix(main): - log names updated --- app/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index 5c5c1b4..baebce6 100644 --- a/app/main.py +++ b/app/main.py @@ -37,7 +37,7 @@ async def get_logs(): try: return FileResponse( - f"{config.get('LOG_FILE')}.log", + ".log", media_type='application/octet-stream', filename=f"ObjectEffects.log", ) @@ -45,14 +45,14 @@ async def get_logs(): raise http_exception( status_code=404, msg="Log file not found", - _input={"lof_file_name": f".log"}, + _input={"lof_file_name": ".log"}, _detail={"error": e.__str__()} ) except Exception as e: raise http_exception( status_code=500, msg="Internal server error during reading logs", - _input={"lof_file_name": f"{config.get('LOG_FILE')}.log"}, + _input={"log_file_name": ".log"}, _detail={"error": e.__str__()} ) From 8fc80469bb08844b9c43d96ffb8d5b28dcdd5ca7 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 18:53:16 +0300 Subject: [PATCH 16/61] fix(main): - logs to file fixed --- app/main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/main.py b/app/main.py index baebce6..e13d528 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,21 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse, FileResponse +from loguru import logger from .dependencies import config, http_exception from .effects.effects_controller import effects_router +log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" + +logger.add( + ".log", + format=log_format, + level="INFO", +) + + app = FastAPI( title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", From b5c0973db3ec889a3226f43400da7fcd1a1715a8 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 25 Jun 2025 18:55:38 +0300 Subject: [PATCH 17/61] fix(effects_controller): - controller method changed to exception handling --- app/effects/effects_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index 065786b..e3d7bcc 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -22,5 +22,4 @@ async def calculate_effects( scenario ID: Scenario ID """ - result = await effects_service.calculate_effects(params) - return EffectsSchema(**result) + return await effects_service.handle_effects_calculation(params) From a6c8ed796184d30657da58cc1726d1e74c77a979 Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev <62374972+Dmitry-Grachev@users.noreply.github.com> Date: Wed, 25 Jun 2025 19:23:23 +0300 Subject: [PATCH 18/61] Update build_and_deploy.yml runner tag changed --- .github/workflows/build_and_deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3cc2b01..b2815e3 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -28,13 +28,13 @@ jobs: NOW: ${{steps.date.outputs.now}} run: docker push "$IMAGE_NAME":"$NOW" stop_container: - runs-on: self-hosted + runs-on: 65_runner needs: build steps: - name: stop_container run: docker rm -f "$CONTAINER_NAME" run_container: - runs-on: self-hosted + runs-on: 65_runner needs: [build, stop_container] env: NOW: ${{needs.build.outputs.now}} From 1774c3419a980c593faa9bb23527450e15393f84 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 30 Jun 2025 15:56:20 +0300 Subject: [PATCH 19/61] fix(effects_service): - added mean service capacity restoration - no population indicator handling added --- app/effects/effects_service.py | 1 + app/effects/modules/effects_api_gateway.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 6b8092d..c08510c 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -216,6 +216,7 @@ async def calculate_effects( normative_value=normative_data["normative_value"], normative_type=normative_data["normative_type"], ) + before_services["capacity"] = before_services["capacity"].fillna(before_services["capacity"].mean()) before_prove_data = await asyncio.to_thread( objectnat_calculator.evaluate_provision, buildings=before_buildings, diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 48d2a6d..5287df4 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -225,7 +225,7 @@ async def get_scenario_population_data( } ) - if (value := population[0]["value"]) < 1: + if len(population) < 1 or (value := population[0]["value"]) < 1: return None return value From da9d0c28c7edfd5084c5157ec913e2610013d026 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 2 Jul 2025 15:37:08 +0300 Subject: [PATCH 20/61] fix(attribute_parser): - added int casting to capacity and demands --- app/effects/modules/attribute_parser.py | 3 ++- app/effects/modules/data_restorator.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/effects/modules/attribute_parser.py b/app/effects/modules/attribute_parser.py index 1aa4e87..c8f7e22 100644 --- a/app/effects/modules/attribute_parser.py +++ b/app/effects/modules/attribute_parser.py @@ -58,7 +58,8 @@ def _parse_service_capacity( gpd.GeoDataFrame: service capacity with parsed storeys data. Can be empty """ - services["capacity"] = services["services"].apply(lambda x: x[0].get("capacity")) + services["capacity"] = services["services"].apply(lambda x: x[0].get("capacity")).astype(int) + services["capacity"] = services["capacity"].fillna(0) return services @staticmethod diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index 11a5a3a..efd1429 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -77,6 +77,7 @@ def _restore_population( living_buildings=buildings, population=int(target_population), ) + balanced_buildings["population"] = balanced_buildings["population"].astype(int) return balanced_buildings.to_crs(4326) @staticmethod From b0257a7fd4cf6a23613be7a88e072a6cdab2be5b Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 6 Aug 2025 14:02:31 +0300 Subject: [PATCH 21/61] fix(effects_service): - context retrieval updated --- app/effects/effects_service.py | 4 ++-- app/effects/modules/effects_api_gateway.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index c08510c..58ca6b3 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -94,7 +94,7 @@ async def calculate_effects( territory_ids_list=project_data["properties"]["context"] ) context_buildings = await effects_api_gateway.get_project_context_buildings( - project_id=effects_params.project_id, + scenario_id=project_data["base_scenario"]["id"], ) context_buildings.drop(index=context_buildings.sjoin(project_territory).index, inplace=True) context_buildings = await attribute_parser.parse_all_from_buildings( @@ -109,7 +109,7 @@ async def calculate_effects( ) context_buildings["is_project"] = False context_services = await effects_api_gateway.get_project_context_services( - project_id=effects_params.project_id, + scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, ) if context_services.empty: diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 5287df4..3c1cd75 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -130,12 +130,12 @@ async def get_scenario_buildings( @staticmethod async def get_project_context_buildings( - project_id: int, + scenario_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario context buildings data from urban_api Args: - project_id: scenario id to get buildings from + scenario_id: scenario id to get buildings from Returns: gpd.GeoDataFrame: buildings layer Raises: @@ -143,7 +143,7 @@ async def get_project_context_buildings( """ context_buildings = await urban_api_handler.get( - endpoint_url=f"/api/v1/projects/{project_id}/context/geometries_with_all_objects", + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "physical_object_type_id": 4, } @@ -182,20 +182,20 @@ async def get_scenario_services( @staticmethod async def get_project_context_services( - project_id: int, + scenario_id: int, service_type_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario context services data from urban_api Args: - project_id: scenario id to get services from + scenario_id: scenario id to get services from service_type_id: service to get services from Returns: gpd.GeoDataFrame: context services layer. Can be empty """ context_services = await urban_api_handler.get( - endpoint_url=f"/api/v1/projects/{project_id}/context/geometries_with_all_objects", + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "service_type_id": service_type_id, } From 2c1dd2d0f05595eb8a63c6c8cdd548a255ca2f0d Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:01:39 +0300 Subject: [PATCH 22/61] fix(objectnat_calculator): - fixed absolute_total calculations - added dev requirements --- app/effects/modules/objectnat_calculator.py | 2 +- requirements-dev.txt | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 requirements-dev.txt diff --git a/app/effects/modules/objectnat_calculator.py b/app/effects/modules/objectnat_calculator.py index c97aca2..0e6d75b 100644 --- a/app/effects/modules/objectnat_calculator.py +++ b/app/effects/modules/objectnat_calculator.py @@ -199,7 +199,7 @@ def estimate_effects( provision_before[ "supplyed_demands_without_before" - ] = provision_before["supplyed_demands_without"] + ] = provision_before["supplyed_demands_without"] + provision_before["supplyed_demands_within_before"] provision_before[ "us_demands_without_before" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d5591ee --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,15 @@ +aiohttp~=3.11.12 +fastapi~=0.115.8 +geopandas~=0.14.4 +IDU-config~=1.0.2 +loguru~=0.7.3 +numba~=0.60.0 +numpy~=1.26.4 +ObjectNat~=0.2.6 +pandas~=2.2.3 +pydantic~=2.10.6 +scipy~=1.15.1 +requests~=2.32.3 +uvicorn~=0.34.0 +gunicorn~=23.0.0 +requests~=2.32.5 \ No newline at end of file From 18846d212b1b21f53898cc89995b03f62a70a209 Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Tue, 7 Oct 2025 12:14:13 +0300 Subject: [PATCH 23/61] fix(effects_api_gateway): - fixed no service type in response for request (400 http raise) - added dev pre-commit config - updated dev requirements - style changes (ran pre-commit for all files) --- .pre-commit-config.yaml | 13 ++ app/common/api_handler/api_handler.py | 82 ++++--- app/common/exceptions/exception_handler.py | 85 ++++++++ .../exceptions/http_exception_wrapper.py | 7 +- app/dependencies.py | 12 +- app/effects/dto/effects_dto.py | 2 +- app/effects/effects_controller.py | 7 +- app/effects/effects_service.py | 106 +++++---- app/effects/modules/__init__.py | 4 +- app/effects/modules/attribute_parser.py | 51 +++-- app/effects/modules/data_restorator.py | 51 ++--- app/effects/modules/effects_api_gateway.py | 104 +++++---- app/effects/modules/matrix_builder.py | 28 ++- app/effects/modules/objectnat_calculator.py | 203 ++++++++++-------- app/effects/shemas/effects_base_schema.py | 38 ++-- app/main.py | 18 +- requirements-dev.txt | 3 +- 17 files changed, 480 insertions(+), 334 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 app/common/exceptions/exception_handler.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9703de4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + language_version: python3.11 + + - repo: https://github.com/pycqa/isort + rev: 6.0.1 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black"] \ No newline at end of file diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index 4f8bc08..3f188e9 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -6,8 +6,8 @@ class APIHandler: def __init__( - self, - base_url: str, + self, + base_url: str, ) -> None: """Initialisation function @@ -21,7 +21,7 @@ def __init__( @staticmethod async def _check_response_status( - response: aiohttp.ClientResponse + response: aiohttp.ClientResponse, ) -> list | dict | None: """Function handles response @@ -57,11 +57,11 @@ async def _check_response_status( ) async def get( - self, - endpoint_url: str, - headers: dict | None = None, - params: dict | None = None, - session: aiohttp.ClientSession | None = None, + self, + endpoint_url: str, + headers: dict | None = None, + params: dict | None = None, + session: aiohttp.ClientSession | None = None, ) -> dict | list: """Function to get data from api @@ -83,11 +83,7 @@ async def get( session=session, ) url = self.base_url + endpoint_url - async with session.get( - url=url, - headers=headers, - params=params - ) as response: + async with session.get(url=url, headers=headers, params=params) as response: result = await self._check_response_status(response) if isinstance(result, list): return result @@ -107,13 +103,13 @@ async def get( return result async def post( - self, - endpoint_url: str, - headers: dict | None = None, - params: dict | None = None, - data: dict | None = None, - session: aiohttp.ClientSession | None = None, - ) -> dict | list: + self, + endpoint_url: str, + headers: dict | None = None, + params: dict | None = None, + data: dict | None = None, + session: aiohttp.ClientSession | None = None, + ) -> dict | list: """Function to post data from api Args: @@ -144,7 +140,7 @@ async def post( ) as response: result = await self._check_response_status(response) if not result: - return await self.post( + return await self.post( endpoint_url=endpoint_url, headers=headers, params=params, @@ -153,12 +149,12 @@ async def post( return result async def put( - self, - endpoint_url: str, - headers: dict | None = None, - params: dict | None = None, - data: dict | None = None, - session: aiohttp.ClientSession | None = None, + self, + endpoint_url: str, + headers: dict | None = None, + params: dict | None = None, + data: dict | None = None, + session: aiohttp.ClientSession | None = None, ) -> dict | list: """Function to post data from api @@ -183,14 +179,14 @@ async def put( ) url = self.base_url + endpoint_url async with session.put( - url=url, - headers=headers, - params=params, - data=data, + url=url, + headers=headers, + params=params, + data=data, ) as response: result = await self._check_response_status(response) if not result: - return await self.put( + return await self.put( endpoint_url=endpoint_url, headers=headers, params=params, @@ -199,12 +195,12 @@ async def put( return result async def delete( - self, - endpoint_url: str, - headers: dict | None = None, - params: dict | None = None, - data: dict | None = None, - session: aiohttp.ClientSession | None = None, + self, + endpoint_url: str, + headers: dict | None = None, + params: dict | None = None, + data: dict | None = None, + session: aiohttp.ClientSession | None = None, ) -> dict | list: """Function to post data from api @@ -229,14 +225,14 @@ async def delete( ) url = self.base_url + endpoint_url async with session.delete( - url=url, - headers=headers, - params=params, - data=data, + url=url, + headers=headers, + params=params, + data=data, ) as response: result = await self._check_response_status(response) if not result: - return await self.delete( + return await self.delete( endpoint_url=endpoint_url, headers=headers, params=params, diff --git a/app/common/exceptions/exception_handler.py b/app/common/exceptions/exception_handler.py new file mode 100644 index 0000000..b42acfc --- /dev/null +++ b/app/common/exceptions/exception_handler.py @@ -0,0 +1,85 @@ +"""Exception handling middleware is defined here.""" + +import itertools +import json +import traceback + +from fastapi import FastAPI, HTTPException, Request +from loguru import logger +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import JSONResponse + +from .http_exception_wrapper import http_exception + + +class ExceptionHandlerMiddleware( + BaseHTTPMiddleware +): # pylint: disable=too-few-public-methods + """Handle exceptions, so they become http response code 500 - Internal Server Error if not handled as HTTPException + previously. + Attributes: + app (FastAPI): The FastAPI application instance. + """ + + def __init__(self, app: FastAPI): + """ + Universal exception handler middleware init function. + Args: + app (FastAPI): The FastAPI application instance. + """ + + super().__init__(app) + + async def dispatch(self, request: Request, call_next): + """ + Dispatch function for sending errors to user from API + Args: + request (Request): The incoming request object. + call_next: function to extract. + """ + + try: + return await call_next(request) + except Exception as e: + request_info = { + "method": request.method, + "url": str(request.url), + "path_params": dict(request.path_params), + "query_params": dict(request.query_params), + "headers": dict(request.headers), + } + try: + request_info["body"] = await request.json() + except: + try: + request_info["body"] = str(await request.body()) + except: + request_info["body"] = "Could not read request body" + if isinstance(e, HTTPException): + return JSONResponse( + status_code=e.status_code, + content={ + "message": ( + e.detail.get("msg") + if isinstance(e.detail, dict) + else str(e.detail) + ), + "error_type": e.__class__.__name__, + "request": request_info, + "detail": ( + e.detail.get("detail") + if isinstance(e.detail, dict) + else None + ), + }, + ) + return JSONResponse( + status_code=500, + content={ + "message": "Internal server error", + "error_type": e.__class__.__name__, + "request": request_info, + "detail": str(e), + "traceback": traceback.format_exc().splitlines(), + }, + ) diff --git a/app/common/exceptions/http_exception_wrapper.py b/app/common/exceptions/http_exception_wrapper.py index c9957a8..57248b0 100644 --- a/app/common/exceptions/http_exception_wrapper.py +++ b/app/common/exceptions/http_exception_wrapper.py @@ -3,10 +3,5 @@ def http_exception(status_code: int, msg: str, _input, _detail) -> HTTPException: return HTTPException( - status_code=status_code, - detail={ - "msg": msg, - "input": _input, - "detail": _detail - } + status_code=status_code, detail={"msg": msg, "input": _input, "detail": _detail} ) diff --git a/app/dependencies.py b/app/dependencies.py index aa25afa..81a1606 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -1,22 +1,16 @@ import sys -from loguru import logger from iduconfig import Config +from loguru import logger -from app.common.exceptions.http_exception_wrapper import http_exception from app.common.api_handler.api_handler import APIHandler - +from app.common.exceptions.http_exception_wrapper import http_exception logger.remove() logger.add(sys.stderr, level="INFO") log_level = "INFO" log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" -logger.add( - sys.stderr, - format=log_format, - level=log_level, - colorize=True -) +logger.add(sys.stderr, format=log_format, level=log_level, colorize=True) config = Config() diff --git a/app/effects/dto/effects_dto.py b/app/effects/dto/effects_dto.py index 087a604..93dbffe 100644 --- a/app/effects/dto/effects_dto.py +++ b/app/effects/dto/effects_dto.py @@ -11,5 +11,5 @@ class EffectsDTO(BaseModel): target_population: Optional[int] = Field( default=None, examples=[200], - description="Target population for project territory" + description="Target population for project territory", ) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index e3d7bcc..3cf4eb4 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -2,17 +2,16 @@ from fastapi import APIRouter, Depends - from .dto.effects_dto import EffectsDTO -from .shemas.effects_base_schema import EffectsSchema from .effects_service import effects_service - +from .shemas.effects_base_schema import EffectsSchema effects_router = APIRouter(prefix="/effects") + @effects_router.get("/evaluate_provision", response_model=EffectsSchema) async def calculate_effects( - params: Annotated[EffectsDTO, Depends(EffectsDTO)], + params: Annotated[EffectsDTO, Depends(EffectsDTO)], ) -> EffectsSchema: """ Get method for retrieving effects with objectnat diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 58ca6b3..44fce9e 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -1,18 +1,19 @@ -import json import asyncio +import json import geopandas as gpd import pandas as pd from loguru import logger -from fastapi import HTTPException -from app.dependencies import http_exception +from app.common.exceptions.http_exception_wrapper import http_exception + from .dto.effects_dto import EffectsDTO from .modules import ( - effects_api_gateway, - data_restorator, attribute_parser, - matrix_builder, objectnat_calculator + data_restorator, + effects_api_gateway, + matrix_builder, + objectnat_calculator, ) from .shemas.effects_base_schema import EffectsSchema @@ -24,7 +25,7 @@ class EffectsService: @staticmethod async def _get_pivot( - effects: pd.DataFrame | gpd.GeoDataFrame, + effects: pd.DataFrame | gpd.GeoDataFrame, ) -> dict[str, int | float]: """ Function creates a pivot table for effects data @@ -47,29 +48,36 @@ async def _get_pivot( if effects[effects["is_project"]].empty: return result - result["median_index_scenario_project"] = int(effects[effects["is_project"]]["index_scenario_project"].median()) - result["average_index_scenario_project"] = effects[effects["is_project"]]["index_scenario_project"].mean() + result["median_index_scenario_project"] = int( + effects[effects["is_project"]]["index_scenario_project"].median() + ) + result["average_index_scenario_project"] = effects[effects["is_project"]][ + "index_scenario_project" + ].mean() result["sum_absolute_scenario_project"] = int( effects[effects["is_project"]]["absolute_scenario_project"].sum() ) result["median_absolute_scenario_project"] = int( effects[effects["is_project"]]["absolute_scenario_project"].median() ) - result["average_absolute_scenario_project"] = effects[effects["is_project"]]["absolute_scenario_project"].mean() + result["average_absolute_scenario_project"] = effects[effects["is_project"]][ + "absolute_scenario_project" + ].mean() result["median_absolute_scenario_project"] = int( effects[effects["is_project"]]["absolute_scenario_project"].median() ) - result["average_index_scenario_project"] = effects[effects["is_project"]]["index_scenario_project"].mean() - result["median_index_scenario_project"] = int(effects[effects["is_project"]]["index_scenario_project"].median()) + result["average_index_scenario_project"] = effects[effects["is_project"]][ + "index_scenario_project" + ].mean() + result["median_index_scenario_project"] = int( + effects[effects["is_project"]]["index_scenario_project"].median() + ) return result # ToDo Add population retrievement by year # ToDo Split function # ToDo Rewrite to context ids normal handling - async def calculate_effects( - self, - effects_params: EffectsDTO - ) -> dict[str, dict]: + async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: """ Calculate provision effects by project data and target scenario Args: @@ -84,7 +92,9 @@ async def calculate_effects( project_data = await effects_api_gateway.get_project_data( effects_params.project_id ) - project_territory = await effects_api_gateway.get_project_territory(effects_params.project_id) + project_territory = await effects_api_gateway.get_project_territory( + effects_params.project_id + ) normative_data = await effects_api_gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], @@ -96,7 +106,9 @@ async def calculate_effects( context_buildings = await effects_api_gateway.get_project_context_buildings( scenario_id=project_data["base_scenario"]["id"], ) - context_buildings.drop(index=context_buildings.sjoin(project_territory).index, inplace=True) + context_buildings.drop( + index=context_buildings.sjoin(project_territory).index, inplace=True + ) context_buildings = await attribute_parser.parse_all_from_buildings( living_buildings=context_buildings, ) @@ -113,18 +125,20 @@ async def calculate_effects( service_type_id=effects_params.service_type_id, ) if context_services.empty: - #ToDo Revise to another code + # ToDo Revise to another code raise http_exception( status_code=404, msg="No services of {service_type_id} type found in context", _input={"service_type_id": effects_params.service_type_id}, - _detail={} + _detail={}, ) context_services = await attribute_parser.parse_all_from_services( services=context_services, ) - target_scenario_population = await effects_api_gateway.get_scenario_population_data( - scenario_id=effects_params.scenario_id, + target_scenario_population = ( + await effects_api_gateway.get_scenario_population_data( + scenario_id=effects_params.scenario_id, + ) ) target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( scenario_id=effects_params.scenario_id @@ -168,20 +182,17 @@ async def calculate_effects( services=base_scenario_services, ) after_buildings = await asyncio.to_thread( - pd.concat, - objs=[context_buildings, target_scenario_buildings] + pd.concat, objs=[context_buildings, target_scenario_buildings] ) after_services = await asyncio.to_thread( - pd.concat, - objs=[context_services, target_scenario_services] + pd.concat, objs=[context_services, target_scenario_services] ) - before_buildings = await asyncio.to_thread( + before_buildings = await asyncio.to_thread( pd.concat, objs=[context_buildings, base_scenario_buildings], ) before_services = await asyncio.to_thread( - pd.concat, - objs=[context_services, base_scenario_services] + pd.concat, objs=[context_services, base_scenario_services] ) after_buildings.sort_values("is_project", ascending=False, inplace=True) after_buildings.drop_duplicates("building_id", keep="first", inplace=True) @@ -200,7 +211,7 @@ async def calculate_effects( before_services.to_crs(local_crs, inplace=True) after_buildings.to_crs(local_crs, inplace=True) after_services.to_crs(local_crs, inplace=True) - #ToDo context - project objects relation should be revised + # ToDo context - project objects relation should be revised after_services.drop_duplicates("geometry", inplace=True) before_matrix = await asyncio.to_thread( matrix_builder.calculate_availability_matrix, @@ -216,7 +227,9 @@ async def calculate_effects( normative_value=normative_data["normative_value"], normative_type=normative_data["normative_type"], ) - before_services["capacity"] = before_services["capacity"].fillna(before_services["capacity"].mean()) + before_services["capacity"] = before_services["capacity"].fillna( + before_services["capacity"].mean() + ) before_prove_data = await asyncio.to_thread( objectnat_calculator.evaluate_provision, buildings=before_buildings, @@ -255,38 +268,15 @@ async def calculate_effects( "buildings": json.loads( after_prove_data["buildings"].to_crs(4326).to_json() ), - "services": json.loads(after_prove_data["services"].to_crs(4326).to_json()), + "services": json.loads( + after_prove_data["services"].to_crs(4326).to_json() + ), "links": json.loads(after_prove_data["links"].to_crs(4326).to_json()), }, "effects": json.loads(effects.to_crs(4326).to_json()), "pivot": pivot, } - return result - - async def handle_effects_calculation(self, effects_params: EffectsDTO) -> EffectsSchema: - """ - Function handles errors from effects calculations. - Args: - effects_params (EffectsDTO): - Returns: - EffectsSchema: with inf - Raises: - Any from Urban API - 500, if internal server error - """ - try: - result = await self.calculate_effects(effects_params) - return EffectsSchema(**result) - except HTTPException as http_e: - raise http_e - except Exception as e: - logger.exception(e) - raise http_exception( - 500, - msg="Error during effects calculation", - _input=effects_params.__dict__, - _detail={"error": e.__str__()} - ) + return EffectsSchema(**result) effects_service = EffectsService() diff --git a/app/effects/modules/__init__.py b/app/effects/modules/__init__.py index cb0dff8..40d2f2d 100644 --- a/app/effects/modules/__init__.py +++ b/app/effects/modules/__init__.py @@ -1,5 +1,5 @@ from .attribute_parser import attribute_parser -from .effects_api_gateway import effects_api_gateway from .data_restorator import data_restorator +from .effects_api_gateway import effects_api_gateway from .matrix_builder import matrix_builder -from .objectnat_calculator import objectnat_calculator \ No newline at end of file +from .objectnat_calculator import objectnat_calculator diff --git a/app/effects/modules/attribute_parser.py b/app/effects/modules/attribute_parser.py index c8f7e22..94f745b 100644 --- a/app/effects/modules/attribute_parser.py +++ b/app/effects/modules/attribute_parser.py @@ -1,10 +1,7 @@ -import json import asyncio -import pandas as pd import geopandas as gpd - -from app.dependencies import http_exception +import pandas as pd class AttributeParser: @@ -14,7 +11,7 @@ class AttributeParser: @staticmethod async def parse_all_from_buildings( - living_buildings: pd.DataFrame | gpd.GeoDataFrame, + living_buildings: pd.DataFrame | gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """ Function purses living building area for buildings from nested response @@ -34,21 +31,28 @@ async def parse_all_from_buildings( if living_buildings["storeys_count"].isna().all(): living_buildings["storeys_count"] = await asyncio.to_thread( living_buildings["physical_objects"].apply, - lambda x: x[0].get("properties").get("Количество этажей") + lambda x: x[0].get("properties").get("Количество этажей"), ) living_buildings["building_id"] = await asyncio.to_thread( living_buildings["physical_objects"].apply, lambda x: x[0]["physical_object_id"], ) living_buildings = living_buildings.drop( - ['object_geometry_id', 'territory', 'address', 'osm_id', 'physical_objects', 'services'], + [ + "object_geometry_id", + "territory", + "address", + "osm_id", + "physical_objects", + "services", + ], axis=1, ) return living_buildings @staticmethod def _parse_service_capacity( - services:gpd.GeoDataFrame, + services: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """ Function parses capacity attributes from nested response @@ -58,14 +62,14 @@ def _parse_service_capacity( gpd.GeoDataFrame: service capacity with parsed storeys data. Can be empty """ - services["capacity"] = services["services"].apply(lambda x: x[0].get("capacity")).astype(int) + services["capacity"] = ( + services["services"].apply(lambda x: x[0].get("capacity")).astype(int) + ) services["capacity"] = services["capacity"].fillna(0) return services @staticmethod - def _parse_service_id( - services: gpd.GeoDataFrame - ) -> gpd.GeoDataFrame: + def _parse_service_id(services: gpd.GeoDataFrame) -> gpd.GeoDataFrame: """ Function parses service id from nested response Args: @@ -74,12 +78,14 @@ def _parse_service_id( gpd.GeoDataFrame: service id with parsed storeys data. Can be empty """ - services["service_id"] = services["services"].apply(lambda x: x[0].get("service_id")) + services["service_id"] = services["services"].apply( + lambda x: x[0].get("service_id") + ) return services async def parse_all_from_services( - self, - services: gpd.GeoDataFrame, + self, + services: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """ Function parses all required data from service request data @@ -96,13 +102,20 @@ async def parse_all_from_services( services=services, ) services = await asyncio.to_thread( - self._parse_service_capacity, - services=services + self._parse_service_capacity, services=services ) services = services.drop( - ['object_geometry_id', 'territory', 'address', 'osm_id', 'physical_objects', 'services'], - axis=1 + [ + "object_geometry_id", + "territory", + "address", + "osm_id", + "physical_objects", + "services", + ], + axis=1, ) return services + attribute_parser = AttributeParser() diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index efd1429..62a9196 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -1,11 +1,11 @@ from typing import Literal +import geopandas as gpd import numpy as np import pandas as pd -import geopandas as gpd from objectnat import get_balanced_buildings -from app.dependencies import http_exception +from app.common.exceptions.http_exception_wrapper import http_exception class DataRestorator: @@ -15,7 +15,7 @@ class DataRestorator: @staticmethod def _restore_stores( - buildings: gpd.GeoDataFrame, + buildings: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """ Function to restore stores from db, have to include columns stores_count @@ -36,7 +36,7 @@ def _restore_stores( @staticmethod def _restore_target_population( - buildings: gpd.GeoDataFrame, + buildings: gpd.GeoDataFrame, ) -> int: """ Function estimates target population for territory @@ -48,13 +48,13 @@ def _restore_target_population( local_crs = buildings.estimate_utm_crs() buildings = buildings.to_crs(local_crs) - return int(sum(buildings.area * buildings["storeys_count"]) * 0.8/33) + return int(sum(buildings.area * buildings["storeys_count"]) * 0.8 / 33) # ToDo delete crs transformation def _restore_population( - self, - buildings: gpd.GeoDataFrame, - target_population: int | None = None, + self, + buildings: gpd.GeoDataFrame, + target_population: int | None = None, ): """ Function fills population data with objectnat population restoration @@ -70,7 +70,9 @@ def _restore_population( target_population = self._restore_target_population(buildings) local_crs = buildings.estimate_utm_crs() buildings = buildings.to_crs(local_crs) - buildings["storeys_count"] = buildings["storeys_count"].apply(lambda x: int(round(x))) + buildings["storeys_count"] = buildings["storeys_count"].apply( + lambda x: int(round(x)) + ) buildings["living_area"] = buildings.area * buildings["storeys_count"] * 0.8 buildings["living_area"] = buildings["living_area"].astype(int) balanced_buildings = get_balanced_buildings( @@ -82,8 +84,8 @@ def _restore_population( @staticmethod def _generate_demand_per_building( - buildings: gpd.GeoDataFrame, - target_demand: int |float, + buildings: gpd.GeoDataFrame, + target_demand: int | float, ) -> pd.DataFrame | gpd.GeoDataFrame: """ Function generates random demands by probability with population data per building @@ -97,18 +99,20 @@ def _generate_demand_per_building( p = buildings["population"] / buildings["population"].sum() rng = np.random.default_rng(seed=0) r = pd.Series(0, p.index) - choice = np.unique(rng.choice(p.index, int(target_demand), p=p.values), return_counts=True) + choice = np.unique( + rng.choice(p.index, int(target_demand), p=p.values), return_counts=True + ) choice = r.add(pd.Series(choice[1], choice[0]), fill_value=0) buildings["demand"] = choice.astype(int) return buildings # Todo review provision model or at least create capacity solver def restore_demands( - self, - buildings: gpd.GeoDataFrame, - service_normative: int, - service_normative_type: Literal["unit", "capacity"], - target_population: int | None = None, + self, + buildings: gpd.GeoDataFrame, + service_normative: int, + service_normative_type: Literal["unit", "capacity"], + target_population: int | None = None, ) -> gpd.GeoDataFrame: """ Function restores demands in buildings by population for service @@ -128,10 +132,11 @@ def restore_demands( target_population=target_population, ) if service_normative_type == "capacity": - target_total_demand = buildings["population"].sum() / 1000 * service_normative + target_total_demand = ( + buildings["population"].sum() / 1000 * service_normative + ) buildings = self._generate_demand_per_building( - buildings=buildings, - target_demand=target_total_demand + buildings=buildings, target_demand=target_total_demand ) return buildings elif service_normative_type == "unit": @@ -144,11 +149,7 @@ def restore_demands( _input={ "service_normative_type": service_normative_type, }, - _detail={ - "available_demand_type": [ - "unit", "capacity" - ] - } + _detail={"available_demand_type": ["unit", "capacity"]}, ) diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 3c1cd75..e548cf7 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -1,20 +1,20 @@ import asyncio -from shapely.geometry import shape import geopandas as gpd -import numpy as np import pandas as pd +from shapely.geometry import shape -from app.dependencies import urban_api_handler, http_exception +from app.common.exceptions.http_exception_wrapper import http_exception +from app.dependencies import urban_api_handler class EffectsAPIGateway: @staticmethod async def get_service_normative( - territory_id: int, - context_ids: list[int], - service_type_id: int, + territory_id: int, + context_ids: list[int], + service_type_id: int, ) -> dict[str, int | str]: """ Function retrieves normative data from urban_api @@ -39,13 +39,37 @@ async def get_service_normative( ) request_ter_id = territory_id response_df = pd.DataFrame.from_records(response) - response_df["service_type_id"] = response_df["service_type"].apply(lambda x: x["id"]) - service_type = response_df[response_df["service_type_id"] == service_type_id].copy() - service_type = service_type[service_type["year"] == service_type["year"].max()].iloc[0].to_dict() + response_df["service_type_id"] = response_df["service_type"].apply( + lambda x: x["id"] + ) + service_type = response_df[ + response_df["service_type_id"] == service_type_id + ].copy() + if len(service_type) < 1: + raise http_exception( + 400, + msg="Service type id not found in urban_db for provided territory/context ids. ", + _input={ + "territory_id": territory_id, + "context_ids": context_ids, + "service_type_id": service_type_id, + }, + _detail={ + "Available service ids": response_df["service_type_id"].to_list() + }, + ) + + service_type = ( + service_type[service_type["year"] == service_type["year"].max()] + .iloc[0] + .to_dict() + ) if service_type["service_type"]["id"] == service_type_id: if not pd.isna(service_type["radius_availability_meters"]): - service_type["normative_value"] = service_type["radius_availability_meters"] + service_type["normative_value"] = service_type[ + "radius_availability_meters" + ] service_type["normative_type"] = "dist" if not pd.isna(service_type.get("services_per_1000_normative")): service_type["capacity_type"] = "unit" @@ -53,7 +77,9 @@ async def get_service_normative( service_type["capacity_type"] = "capacity" return service_type elif not pd.isna(service_type["time_availability_minutes"]): - service_type["normative_value"] = service_type["time_availability_minutes"] + service_type["normative_value"] = service_type[ + "time_availability_minutes" + ] service_type["normative_type"] = "time" if not pd.isna(service_type.get("services_per_1000_normative")): service_type["capacity_type"] = "unit" @@ -71,7 +97,9 @@ async def get_service_normative( "service_type_id": service_type_id, }, _detail={ - "Available service ids": [service_type["id"] for service_type in response] + "Available service ids": [ + service_type["id"] for service_type in response + ] }, ) raise http_exception( @@ -83,9 +111,7 @@ async def get_service_normative( "request_ter_id": request_ter_id, "service_type_id": service_type_id, }, - _detail={ - "Available service ids": response_df["service_type_id"].to_list() - }, + _detail={"Available service ids": response_df["service_type_id"].to_list()}, ) @staticmethod @@ -106,7 +132,7 @@ async def get_project_data(project_id: int) -> dict[str, int | dict]: @staticmethod async def get_scenario_buildings( - scenario_id: int, + scenario_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario buildings data from urban_api @@ -118,9 +144,7 @@ async def get_scenario_buildings( buildings = await urban_api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", - params={ - "physical_object_type_id": 4 - } + params={"physical_object_type_id": 4}, ) buildings_gdf = gpd.GeoDataFrame.from_features(buildings) if buildings_gdf.empty: @@ -130,7 +154,7 @@ async def get_scenario_buildings( @staticmethod async def get_project_context_buildings( - scenario_id: int, + scenario_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario context buildings data from urban_api @@ -146,7 +170,7 @@ async def get_project_context_buildings( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "physical_object_type_id": 4, - } + }, ) context_buildings_gdf = gpd.GeoDataFrame.from_features(context_buildings) if context_buildings_gdf.empty: @@ -156,8 +180,8 @@ async def get_project_context_buildings( @staticmethod async def get_scenario_services( - scenario_id: int, - service_type_id: int, + scenario_id: int, + service_type_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario services data from urban_api @@ -172,7 +196,7 @@ async def get_scenario_services( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={ "service_type_id": service_type_id, - } + }, ) services_gdf = gpd.GeoDataFrame.from_features(services) if services_gdf.empty: @@ -182,8 +206,8 @@ async def get_scenario_services( @staticmethod async def get_project_context_services( - scenario_id: int, - service_type_id: int, + scenario_id: int, + service_type_id: int, ) -> gpd.GeoDataFrame: """ Function retrieves scenario context services data from urban_api @@ -198,7 +222,7 @@ async def get_project_context_services( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "service_type_id": service_type_id, - } + }, ) context_services_gdf = gpd.GeoDataFrame.from_features(context_services) if context_services_gdf.empty: @@ -207,9 +231,7 @@ async def get_project_context_services( return context_services_gdf @staticmethod - async def get_scenario_population_data( - scenario_id: int | None - ) -> int | None: + async def get_scenario_population_data(scenario_id: int | None) -> int | None: """ Function retrieves population data from urban_api Args: @@ -222,7 +244,7 @@ async def get_scenario_population_data( endpoint_url=f"/api/v1/scenarios/{scenario_id}/indicators_values", params={ "indicator_ids": 1, - } + }, ) if len(population) < 1 or (value := population[0]["value"]) < 1: @@ -231,7 +253,7 @@ async def get_scenario_population_data( @staticmethod async def get_context_population( - territory_ids_list: list[int], + territory_ids_list: list[int], ) -> int: """ Function retrieves territory population data from urban_api by territory id @@ -241,12 +263,13 @@ async def get_context_population( gpd.GeoDataFrame: territory population data layer """ - task_list = [urban_api_handler.get( - endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", - params={ - "indicator_ids": 1 - } - ) for territory_id in territory_ids_list] + task_list = [ + urban_api_handler.get( + endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", + params={"indicator_ids": 1}, + ) + for territory_id in territory_ids_list + ] result = await asyncio.gather(*task_list) return sum([item[0]["value"] for item in result]) @@ -264,7 +287,10 @@ async def get_project_territory(project_id: int) -> gpd.GeoDataFrame: territory = await urban_api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}/territory", ) - territory_gdf = gpd.GeoDataFrame(geometry=[shape(territory["geometry"])], crs=4326) + territory_gdf = gpd.GeoDataFrame( + geometry=[shape(territory["geometry"])], crs=4326 + ) return territory_gdf + effects_api_gateway = EffectsAPIGateway() diff --git a/app/effects/modules/matrix_builder.py b/app/effects/modules/matrix_builder.py index 821b0c7..eb153a1 100644 --- a/app/effects/modules/matrix_builder.py +++ b/app/effects/modules/matrix_builder.py @@ -1,8 +1,8 @@ from typing import Literal +import geopandas as gpd import numpy as np import pandas as pd -import geopandas as gpd from scipy.spatial import KDTree @@ -10,10 +10,10 @@ class MatrixBuilder: @staticmethod def calculate_availability_matrix( - buildings: gpd.GeoDataFrame, - services: gpd.GeoDataFrame, - normative_value: int, - normative_type: Literal["time", "dist"] + buildings: gpd.GeoDataFrame, + services: gpd.GeoDataFrame, + normative_value: int, + normative_type: Literal["time", "dist"], ) -> pd.DataFrame: """ Calculated availability matrix with walk simulation @@ -27,20 +27,26 @@ def calculate_availability_matrix( """ if normative_type == "time": - normative_value = (normative_value * 1000/60 * 40 )/1.41 + normative_value = (normative_value * 1000 / 60 * 40) / 1.41 else: normative_value = (normative_value * 3) / 1.41 local_crs = buildings.estimate_utm_crs() buildings = buildings.to_crs(local_crs).set_index(buildings.index, drop=True) services = services.to_crs(local_crs).set_index(services.index, drop=True) - buildings_points = [geometry.coords[0] for geometry in buildings.geometry.centroid] - services_points = [geometry.coords[0] for geometry in services.geometry.centroid] + buildings_points = [ + geometry.coords[0] for geometry in buildings.geometry.centroid + ] + services_points = [ + geometry.coords[0] for geometry in services.geometry.centroid + ] buildings_kd_tree = KDTree(buildings_points) services_kd_tree = KDTree(services_points) distances = buildings_kd_tree.sparse_distance_matrix( - other=services_kd_tree, - max_distance=normative_value * 3) - matrix = pd.DataFrame.sparse.from_spmatrix(distances, index=buildings.index, columns=services.index) + other=services_kd_tree, max_distance=normative_value * 3 + ) + matrix = pd.DataFrame.sparse.from_spmatrix( + distances, index=buildings.index, columns=services.index + ) matrix = matrix.sparse.to_dense() matrix.replace(0.0, np.nan, inplace=True) return matrix diff --git a/app/effects/modules/objectnat_calculator.py b/app/effects/modules/objectnat_calculator.py index 0e6d75b..c9540f4 100644 --- a/app/effects/modules/objectnat_calculator.py +++ b/app/effects/modules/objectnat_calculator.py @@ -1,21 +1,16 @@ -import json -from typing import Literal - -import pandas as pd import geopandas as gpd +import pandas as pd from objectnat import get_service_provision -from app.dependencies import http_exception - class ObjectNatCalculator: @staticmethod def evaluate_provision( - buildings: gpd.GeoDataFrame, - services: gpd.GeoDataFrame, - matrix: pd.DataFrame, - service_normative: int + buildings: gpd.GeoDataFrame, + services: gpd.GeoDataFrame, + matrix: pd.DataFrame, + service_normative: int, ) -> dict[str, gpd.GeoDataFrame]: """ Function calculates provision and writes results as dict with fields "buildings", "services" and "links" @@ -28,7 +23,6 @@ def evaluate_provision( dict[str, gpd.GeoDataFrame]: dict with fields "buildings", "services" and "links" """ - build_prov, services_prov, links_prov = get_service_provision( buildings=buildings, services=services, @@ -44,11 +38,11 @@ def evaluate_provision( @staticmethod def _calculate_index( - supplied_demand_after: pd.Series, - supplied_demand_before: pd.Series, - unsupplied_demand_after: pd.Series, - unsupplied_demand_before: pd.Series, - total_demand: int + supplied_demand_after: pd.Series, + supplied_demand_before: pd.Series, + unsupplied_demand_after: pd.Series, + unsupplied_demand_before: pd.Series, + total_demand: int, ) -> pd.Series: """ Function calculates index effects marks for provided objects @@ -61,21 +55,18 @@ def _calculate_index( """ result = ( - ( - supplied_demand_after - supplied_demand_before - ) - ( - unsupplied_demand_after - unsupplied_demand_before - ) - ) / total_demand + (supplied_demand_after - supplied_demand_before) + - (unsupplied_demand_after - unsupplied_demand_before) + ) / total_demand return result # ToDo fix is_project attribute @staticmethod def _calculate_absolute( - supplied_demand_after: pd.Series, - supplied_demand_before: pd.Series, - unsupplied_demand_after: pd.Series, - unsupplied_demand_before: pd.Series, + supplied_demand_after: pd.Series, + supplied_demand_before: pd.Series, + unsupplied_demand_after: pd.Series, + unsupplied_demand_before: pd.Series, ) -> pd.Series: """ Function calculates absolute effects marks for provided objects @@ -86,17 +77,16 @@ def _calculate_absolute( unsupplied_demand_before (pd.Series): unsupplied demand for base scenario """ - result = ( - supplied_demand_after - supplied_demand_before - ).apply(lambda x: max(0, x)) - ( - unsupplied_demand_after - unsupplied_demand_before - ).apply(lambda x: max(0, x) - ) + result = (supplied_demand_after - supplied_demand_before).apply( + lambda x: max(0, x) + ) - (unsupplied_demand_after - unsupplied_demand_before).apply( + lambda x: max(0, x) + ) return result def _calculate_effects( - self, - effects: pd.DataFrame | gpd.GeoDataFrame, + self, + effects: pd.DataFrame | gpd.GeoDataFrame, ) -> pd.DataFrame | gpd.GeoDataFrame: """ Function calculates provision effects @@ -106,12 +96,20 @@ def _calculate_effects( pd.Series: effects results """ - #ToDo fix calculation without/before + # ToDo fix calculation without/before effects = effects.copy() - supplied_demand_within_before = effects["supplyed_demands_within_before"].fillna(0) - supplied_demand_without_before = effects["supplyed_demands_without_before"].fillna(0) - supplied_demand_within_after = effects["supplyed_demands_within_after"].fillna(0) - supplied_demand_without_after = effects["supplyed_demands_without_after"].fillna(0) + supplied_demand_within_before = effects[ + "supplyed_demands_within_before" + ].fillna(0) + supplied_demand_without_before = effects[ + "supplyed_demands_without_before" + ].fillna(0) + supplied_demand_within_after = effects["supplyed_demands_within_after"].fillna( + 0 + ) + supplied_demand_without_after = effects[ + "supplyed_demands_without_after" + ].fillna(0) unsupplied_demand_within_before = effects["us_demands_within_before"].fillna(0) unsupplied_demand_within_after = effects["us_demands_within_after"].fillna(0) total_supplied_demands_before = supplied_demand_without_before @@ -122,21 +120,21 @@ def _calculate_effects( effects.dropna(subset="is_project", inplace=True) - project_total_supplied_demands_before = effects[ - effects["is_project"] - ]["supplyed_demands_without_before"].fillna(0) + project_total_supplied_demands_before = effects[effects["is_project"]][ + "supplyed_demands_without_before" + ].fillna(0) - project_total_supplied_demands_after = effects[ - effects["is_project"] - ]["supplyed_demands_without_after"].fillna(0) + project_total_supplied_demands_after = effects[effects["is_project"]][ + "supplyed_demands_without_after" + ].fillna(0) - project_total_us_demands_before = effects[ - effects["is_project"] - ]["us_demands_without_before"].fillna(0) + project_total_us_demands_before = effects[effects["is_project"]][ + "us_demands_without_before" + ].fillna(0) - project_total_us_demands_after = effects[ - effects["is_project"] - ]["us_demands_without_after"].fillna(0) + project_total_us_demands_after = effects[effects["is_project"]][ + "us_demands_without_after" + ].fillna(0) project_total_demand = int(effects[effects["is_project"]]["demand"].sum()) @@ -154,33 +152,37 @@ def _calculate_effects( total_demand=total_demand, ) effects["absolute_scenario_project"] = None - effects.loc[effects["is_project"], ["absolute_scenario_project"]] = self._calculate_absolute( - supplied_demand_before=project_total_supplied_demands_before, - supplied_demand_after=project_total_supplied_demands_after, - unsupplied_demand_after=project_total_us_demands_after, - unsupplied_demand_before=project_total_us_demands_before, + effects.loc[effects["is_project"], ["absolute_scenario_project"]] = ( + self._calculate_absolute( + supplied_demand_before=project_total_supplied_demands_before, + supplied_demand_after=project_total_supplied_demands_after, + unsupplied_demand_after=project_total_us_demands_after, + unsupplied_demand_before=project_total_us_demands_before, + ) ) effects["index_scenario_project"] = None - effects.loc[effects["is_project"], ["index_scenario_project"]] = self._calculate_index( - supplied_demand_after=project_total_supplied_demands_after, - supplied_demand_before=project_total_supplied_demands_before, - unsupplied_demand_after=project_total_us_demands_after, - unsupplied_demand_before=project_total_us_demands_before, - total_demand=project_total_demand, + effects.loc[effects["is_project"], ["index_scenario_project"]] = ( + self._calculate_index( + supplied_demand_after=project_total_supplied_demands_after, + supplied_demand_before=project_total_supplied_demands_before, + unsupplied_demand_after=project_total_us_demands_after, + unsupplied_demand_before=project_total_us_demands_before, + total_demand=project_total_demand, + ) ) effects["absolute_within"] = self._calculate_absolute( supplied_demand_before=supplied_demand_within_before, supplied_demand_after=supplied_demand_within_after, unsupplied_demand_before=unsupplied_demand_within_before, - unsupplied_demand_after=unsupplied_demand_within_after + unsupplied_demand_after=unsupplied_demand_within_after, ) return effects # ToDo split function def estimate_effects( - self, - provision_before: gpd.GeoDataFrame, - provision_after: gpd.GeoDataFrame, + self, + provision_before: gpd.GeoDataFrame, + provision_after: gpd.GeoDataFrame, ) -> pd.DataFrame | gpd.GeoDataFrame: """ Main function which calculates provision and estimates effects @@ -191,45 +193,56 @@ def estimate_effects( gpd.GeoDataFrame: layer with effects, provision before and after attributes """ - provision_before["supplyed_demands_within_before"] = provision_before["supplyed_demands_within"].copy() + provision_before["supplyed_demands_within_before"] = provision_before[ + "supplyed_demands_within" + ].copy() - provision_before[ - "us_demands_within_before" - ] = provision_before["demand"] - provision_before["supplyed_demands_within_before"] + provision_before["us_demands_within_before"] = ( + provision_before["demand"] + - provision_before["supplyed_demands_within_before"] + ) - provision_before[ - "supplyed_demands_without_before" - ] = provision_before["supplyed_demands_without"] + provision_before["supplyed_demands_within_before"] + provision_before["supplyed_demands_without_before"] = ( + provision_before["supplyed_demands_without"] + + provision_before["supplyed_demands_within_before"] + ) - provision_before[ - "us_demands_without_before" - ] = provision_before["demand"] - provision_before["supplyed_demands_within_before"] + provision_before["us_demands_without_before"] = ( + provision_before["demand"] + - provision_before["supplyed_demands_within_before"] + ) - provision_after["supplyed_demands_within_after"] = provision_after["supplyed_demands_within"].copy() + provision_after["supplyed_demands_within_after"] = provision_after[ + "supplyed_demands_within" + ].copy() - provision_after[ - "us_demands_within_after" - ] = provision_after["demand"] - provision_after["supplyed_demands_within_after"] + provision_after["us_demands_within_after"] = ( + provision_after["demand"] - provision_after["supplyed_demands_within_after"] + ) - provision_after[ - "supplyed_demands_without_after" - ] = provision_after["supplyed_demands_within_after"] + provision_after["supplyed_demands_without"].copy() + provision_after["supplyed_demands_without_after"] = ( + provision_after["supplyed_demands_within_after"] + + provision_after["supplyed_demands_without"].copy() + ) - provision_after[ - "us_demands_without_after" - ] = provision_after["demand"] - provision_after["supplyed_demands_without_after"] + provision_after["us_demands_without_after"] = ( + provision_after["demand"] + - provision_after["supplyed_demands_without_after"] + ) effects = provision_after.merge( - provision_before, - how="outer", - on=["building_id"] + provision_before, how="outer", on=["building_id"] ) effects["geometry"] = effects.apply( - lambda x: x["geometry_x"] if not pd.isna(x["geometry_x"]) else x["geometry_y"], - axis=1 + lambda x: ( + x["geometry_x"] if not pd.isna(x["geometry_x"]) else x["geometry_y"] + ), + axis=1, ) effects.drop(columns=["geometry_x", "geometry_y"], inplace=True) - effects["demand"] = effects["demand_x"].fillna(0) + effects["demand_y"].fillna(0) + effects["demand"] = effects["demand_x"].fillna(0) + effects["demand_y"].fillna( + 0 + ) effects.drop("is_project_y", axis=1, inplace=True) effects.rename(columns={"is_project_x": "is_project"}, inplace=True) effects = self._calculate_effects(effects) @@ -242,10 +255,12 @@ def estimate_effects( "index_scenario_project", "absolute_within", "demand", - "is_project" + "is_project", ] ] - effects = gpd.GeoDataFrame(effects, geometry="geometry", crs=provision_before.crs) + effects = gpd.GeoDataFrame( + effects, geometry="geometry", crs=provision_before.crs + ) return effects diff --git a/app/effects/shemas/effects_base_schema.py b/app/effects/shemas/effects_base_schema.py index 81dedeb..14ddcc8 100644 --- a/app/effects/shemas/effects_base_schema.py +++ b/app/effects/shemas/effects_base_schema.py @@ -1,11 +1,18 @@ -from typing import Literal, Optional, Any +from typing import Any, Literal, Optional from pydantic import BaseModel class GeometrySchema(BaseModel): - type: Literal["Polygon", "MultiPolygon", "LineString", "MultiLineString", "Point", "MultiPoint"] + type: Literal[ + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + "Point", + "MultiPoint", + ] coordinates: list[Any] @@ -29,21 +36,22 @@ class ProvisionSchema(BaseModel): services: FeatureCollectionSchema links: FeatureCollectionSchema + class PivotSchema(BaseModel): - sum_absolute_total: int - average_absolute_total: int | float - median_absolute_total: int - average_index_total: int | float - median_index_total: int - sum_absolute_scenario_project: Optional[int] = None - average_absolute_scenario_project: Optional[int | float] = None - median_absolute_scenario_project: Optional[int] = None - average_index_scenario_project: Optional[int | float] = None - median_index_scenario_project: Optional[int] = None - sum_absolute_within: int - average_absolute_within: int | float - median_absolute_within: int + sum_absolute_total: int + average_absolute_total: int | float + median_absolute_total: int + average_index_total: int | float + median_index_total: int + sum_absolute_scenario_project: Optional[int] = None + average_absolute_scenario_project: Optional[int | float] = None + median_absolute_scenario_project: Optional[int] = None + average_index_scenario_project: Optional[int | float] = None + median_index_scenario_project: Optional[int] = None + sum_absolute_within: int + average_absolute_within: int | float + median_absolute_within: int class EffectsSchema(BaseModel): diff --git a/app/main.py b/app/main.py index e13d528..a1114f6 100644 --- a/app/main.py +++ b/app/main.py @@ -1,12 +1,12 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse, FileResponse +from fastapi.responses import FileResponse, RedirectResponse from loguru import logger +from .common.exceptions.exception_handler import ExceptionHandlerMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router - log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" logger.add( @@ -30,15 +30,19 @@ allow_methods=["*"], allow_headers=["*"], ) +app.add_middleware(ExceptionHandlerMiddleware) + @app.get("/", response_model=dict[str, str]) def read_root(): - return RedirectResponse(url='/docs') + return RedirectResponse(url="/docs") + @app.get("/status") async def read_root(): return {"status": "OK"} + @app.get("/logs") async def get_logs(): """ @@ -48,7 +52,7 @@ async def get_logs(): try: return FileResponse( ".log", - media_type='application/octet-stream', + media_type="application/octet-stream", filename=f"ObjectEffects.log", ) except FileNotFoundError as e: @@ -56,15 +60,15 @@ async def get_logs(): status_code=404, msg="Log file not found", _input={"lof_file_name": ".log"}, - _detail={"error": e.__str__()} + _detail={"error": e.__str__()}, ) except Exception as e: raise http_exception( status_code=500, msg="Internal server error during reading logs", _input={"log_file_name": ".log"}, - _detail={"error": e.__str__()} + _detail={"error": e.__str__()}, ) -app.include_router(effects_router) \ No newline at end of file +app.include_router(effects_router) diff --git a/requirements-dev.txt b/requirements-dev.txt index d5591ee..92c2d65 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,4 +12,5 @@ scipy~=1.15.1 requests~=2.32.3 uvicorn~=0.34.0 gunicorn~=23.0.0 -requests~=2.32.5 \ No newline at end of file +requests~=2.32.5 +pre-commit~=4.3.0 \ No newline at end of file From 2ae9bac63b086c7f07e8b1bcc5719d916f5b29f2 Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Tue, 21 Oct 2025 12:29:43 +0300 Subject: [PATCH 24/61] fix(effects_controller): - removed depreciated function call --- app/effects/effects_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index 3cf4eb4..50d1fac 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -21,4 +21,4 @@ async def calculate_effects( scenario ID: Scenario ID """ - return await effects_service.handle_effects_calculation(params) + return await effects_service.calculate_effects(params) From 87ab6ec186ebe9f5636c6faf8cb33c56332e068b Mon Sep 17 00:00:00 2001 From: Dmitry-Grachev Date: Fri, 31 Oct 2025 15:59:21 +0300 Subject: [PATCH 25/61] pypi mirror config changed --- pip.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip.conf b/pip.conf index 774069a..d16d3ff 100644 --- a/pip.conf +++ b/pip.conf @@ -1,4 +1,4 @@ [global] -index-url=http://10.32.1.108:3141/root/pypi/+simple/ -trusted-host=10.32.1.108 +index-url=http://10.32.11.13:3141/root/pypi/+simple/ +trusted-host=10.32.11.13 timeout=120 From 70b735a83f9e1ac656bd45420af527ed7e3ef6ba Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Mon, 10 Nov 2025 16:55:07 +0300 Subject: [PATCH 26/61] fix(auth): - added optional auth --- app/common/auth/__init__.py | 0 app/common/auth/bearer.py | 14 +++++++ app/effects/effects_controller.py | 5 ++- app/effects/effects_service.py | 23 +++++++---- app/effects/modules/effects_api_gateway.py | 45 ++++++++++++++-------- 5 files changed, 63 insertions(+), 24 deletions(-) create mode 100644 app/common/auth/__init__.py create mode 100644 app/common/auth/bearer.py diff --git a/app/common/auth/__init__.py b/app/common/auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/common/auth/bearer.py b/app/common/auth/bearer.py new file mode 100644 index 0000000..788136f --- /dev/null +++ b/app/common/auth/bearer.py @@ -0,0 +1,14 @@ +from typing import Optional + +from fastapi import Depends +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +http_bearer = HTTPBearer() + + +async def verify_bearer_token( + credentials: HTTPAuthorizationCredentials = Depends(http_bearer), +) -> str | None: + + token = credentials.credentials + return token if token not in ["''", '""'] or not token else None diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index 50d1fac..507422d 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -2,6 +2,8 @@ from fastapi import APIRouter, Depends +from app.common.auth.bearer import verify_bearer_token + from .dto.effects_dto import EffectsDTO from .effects_service import effects_service from .shemas.effects_base_schema import EffectsSchema @@ -12,6 +14,7 @@ @effects_router.get("/evaluate_provision", response_model=EffectsSchema) async def calculate_effects( params: Annotated[EffectsDTO, Depends(EffectsDTO)], + token: str = Depends(verify_bearer_token), ) -> EffectsSchema: """ Get method for retrieving effects with objectnat @@ -21,4 +24,4 @@ async def calculate_effects( scenario ID: Scenario ID """ - return await effects_service.calculate_effects(params) + return await effects_service.calculate_effects(params, token) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 44fce9e..20bcc74 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -77,11 +77,14 @@ async def _get_pivot( # ToDo Add population retrievement by year # ToDo Split function # ToDo Rewrite to context ids normal handling - async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: + async def calculate_effects( + self, effects_params: EffectsDTO, token: str + ) -> EffectsSchema: """ Calculate provision effects by project data and target scenario Args: effects_params (EffectsDTO): Project data + token (str): Authorization token Returns: gpd.GeoDataFrame: Provision effects """ @@ -90,21 +93,22 @@ async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: f"Started calculating effects for {effects_params.scenario_id} and service{effects_params.service_type_id}" ) project_data = await effects_api_gateway.get_project_data( - effects_params.project_id + effects_params.project_id, token ) project_territory = await effects_api_gateway.get_project_territory( - effects_params.project_id + effects_params.project_id, token ) normative_data = await effects_api_gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=effects_params.service_type_id, + token=token, ) context_population = await effects_api_gateway.get_context_population( - territory_ids_list=project_data["properties"]["context"] + territory_ids_list=project_data["properties"]["context"], token=token ) context_buildings = await effects_api_gateway.get_project_context_buildings( - scenario_id=project_data["base_scenario"]["id"], + scenario_id=project_data["base_scenario"]["id"], token=token ) context_buildings.drop( index=context_buildings.sjoin(project_territory).index, inplace=True @@ -123,6 +127,7 @@ async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: context_services = await effects_api_gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, + token=token, ) if context_services.empty: # ToDo Revise to another code @@ -137,11 +142,11 @@ async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: ) target_scenario_population = ( await effects_api_gateway.get_scenario_population_data( - scenario_id=effects_params.scenario_id, + scenario_id=effects_params.scenario_id, token=token ) ) target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( - scenario_id=effects_params.scenario_id + scenario_id=effects_params.scenario_id, token=token ) target_scenario_buildings = await attribute_parser.parse_all_from_buildings( living_buildings=target_scenario_buildings, @@ -157,12 +162,13 @@ async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: target_scenario_services = await effects_api_gateway.get_scenario_services( scenario_id=effects_params.scenario_id, service_type_id=effects_params.service_type_id, + token=token, ) target_scenario_services = await attribute_parser.parse_all_from_services( services=target_scenario_services, ) base_scenario_buildings = await effects_api_gateway.get_scenario_buildings( - scenario_id=project_data["base_scenario"]["id"] + scenario_id=project_data["base_scenario"]["id"], token=token ) base_scenario_buildings = await attribute_parser.parse_all_from_buildings( living_buildings=base_scenario_buildings, @@ -177,6 +183,7 @@ async def calculate_effects(self, effects_params: EffectsDTO) -> EffectsSchema: base_scenario_services = await effects_api_gateway.get_scenario_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, + token=token, ) base_scenario_services = await attribute_parser.parse_all_from_services( services=base_scenario_services, diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index e548cf7..fce428c 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -12,9 +12,7 @@ class EffectsAPIGateway: @staticmethod async def get_service_normative( - territory_id: int, - context_ids: list[int], - service_type_id: int, + territory_id: int, context_ids: list[int], service_type_id: int, token: str ) -> dict[str, int | str]: """ Function retrieves normative data from urban_api @@ -22,6 +20,7 @@ async def get_service_normative( territory_id: territory id to get normative from context_ids: context id to get normative from service_type_id: service to get normative from + token: auth token to get normative from Returns: dict[str, int | str]: normative data with normative value and normative type (Literal["time", "dist"]) Raises: @@ -31,11 +30,13 @@ async def get_service_normative( if len(context_ids) == 1: response = await urban_api_handler.get( f"/api/v1/territory/{context_ids[0]}/normatives", + headers={"Authorization": f"Bearer {token}"} if token else None, ) request_ter_id = context_ids[0] else: response = await urban_api_handler.get( f"/api/v1/territory/{territory_id}/normatives", + headers={"Authorization": f"Bearer {token}"} if token else None, ) request_ter_id = territory_id response_df = pd.DataFrame.from_records(response) @@ -115,29 +116,30 @@ async def get_service_normative( ) @staticmethod - async def get_project_data(project_id: int) -> dict[str, int | dict]: + async def get_project_data(project_id: int, token: str) -> dict[str, int | dict]: """ Function retrieves project territory data from urban_api Args: project_id: project id to get territory from + token: authentication token to retrieve data Returns: dict with "geometry" field as dict with "type" and "coordinates" fields and field "base_scenario_id" """ response = await urban_api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}", + headers={"Authorization": f"Bearer {token}"} if token else None, ) return response @staticmethod - async def get_scenario_buildings( - scenario_id: int, - ) -> gpd.GeoDataFrame: + async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFrame: """ Function retrieves scenario buildings data from urban_api Args: scenario_id: scenario id to get buildings from + token: authentication token to retrieve data Returns: gpd.GeoDataFrame: buildings layer, can be empty """ @@ -145,6 +147,7 @@ async def get_scenario_buildings( buildings = await urban_api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={"physical_object_type_id": 4}, + headers={"Authorization": f"Bearer {token}"} if token else None, ) buildings_gdf = gpd.GeoDataFrame.from_features(buildings) if buildings_gdf.empty: @@ -154,12 +157,13 @@ async def get_scenario_buildings( @staticmethod async def get_project_context_buildings( - scenario_id: int, + scenario_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario context buildings data from urban_api Args: scenario_id: scenario id to get buildings from + token: Authorization token to retrieve data Returns: gpd.GeoDataFrame: buildings layer Raises: @@ -171,6 +175,7 @@ async def get_project_context_buildings( params={ "physical_object_type_id": 4, }, + headers={"Authorization": f"Bearer {token}"} if token else None, ) context_buildings_gdf = gpd.GeoDataFrame.from_features(context_buildings) if context_buildings_gdf.empty: @@ -180,14 +185,14 @@ async def get_project_context_buildings( @staticmethod async def get_scenario_services( - scenario_id: int, - service_type_id: int, + scenario_id: int, service_type_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario services data from urban_api Args: scenario_id: scenario id to get services from service_type_id: service to get services from + token: Authorization token to retrieve data Returns: gpd.GeoDataFrame: services layer, can be empty """ @@ -197,6 +202,7 @@ async def get_scenario_services( params={ "service_type_id": service_type_id, }, + headers={"Authorization": f"Bearer {token}"} if token else None, ) services_gdf = gpd.GeoDataFrame.from_features(services) if services_gdf.empty: @@ -208,12 +214,14 @@ async def get_scenario_services( async def get_project_context_services( scenario_id: int, service_type_id: int, + token: str, ) -> gpd.GeoDataFrame: """ Function retrieves scenario context services data from urban_api Args: scenario_id: scenario id to get services from service_type_id: service to get services from + token: Authorization token to retrieve data Returns: gpd.GeoDataFrame: context services layer. Can be empty """ @@ -223,6 +231,7 @@ async def get_project_context_services( params={ "service_type_id": service_type_id, }, + headers={"Authorization": f"Bearer {token}"} if token else None, ) context_services_gdf = gpd.GeoDataFrame.from_features(context_services) if context_services_gdf.empty: @@ -231,11 +240,14 @@ async def get_project_context_services( return context_services_gdf @staticmethod - async def get_scenario_population_data(scenario_id: int | None) -> int | None: + async def get_scenario_population_data( + scenario_id: int | None, token: str + ) -> int | None: """ Function retrieves population data from urban_api Args: scenario_id: scenario id to get population data from + token: Authorization token to retrieve data Returns: int | none: population data layer, if < 1 returns None """ @@ -245,6 +257,7 @@ async def get_scenario_population_data(scenario_id: int | None) -> int | None: params={ "indicator_ids": 1, }, + headers={"Authorization": f"Bearer {token}"} if token else None, ) if len(population) < 1 or (value := population[0]["value"]) < 1: @@ -252,13 +265,12 @@ async def get_scenario_population_data(scenario_id: int | None) -> int | None: return value @staticmethod - async def get_context_population( - territory_ids_list: list[int], - ) -> int: + async def get_context_population(territory_ids_list: list[int], token: str) -> int: """ Function retrieves territory population data from urban_api by territory id Args: territory_ids_list: list[int]: territory ids list to get population data from + token: Authorization token to retrieve data Returns: gpd.GeoDataFrame: territory population data layer """ @@ -267,6 +279,7 @@ async def get_context_population( urban_api_handler.get( endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", params={"indicator_ids": 1}, + headers={"Authorization": f"Bearer {token}"} if token else None, ) for territory_id in territory_ids_list ] @@ -275,17 +288,19 @@ async def get_context_population( return sum([item[0]["value"] for item in result]) @staticmethod - async def get_project_territory(project_id: int) -> gpd.GeoDataFrame: + async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame: """ Function retrieves territory data from urban_api Args: project_id: project id to get territory data from + token: Authorization token to retrieve data Returns: gpd.GeoDataFrame: territory data layer """ territory = await urban_api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}/territory", + headers={"Authorization": f"Bearer {token}"} if token else None, ) territory_gdf = gpd.GeoDataFrame( geometry=[shape(territory["geometry"])], crs=4326 From 49f639fd81b4ad4402b2183135e72da0b93f47e2 Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Tue, 11 Nov 2025 17:31:38 +0300 Subject: [PATCH 27/61] fix(calculate effects): - fixed duplicated services from project and context --- app/effects/effects_service.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 20bcc74..a71ec05 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -205,11 +205,17 @@ async def calculate_effects( after_buildings.drop_duplicates("building_id", keep="first", inplace=True) after_buildings.set_index("building_id", inplace=True) after_services.set_index("service_id", inplace=True) + after_services = after_services[ + ~after_services.index.duplicated(keep="first") + ].copy() before_buildings.sort_values("is_project", ascending=False, inplace=True) before_buildings.drop_duplicates("building_id", keep="first", inplace=True) before_buildings.set_index("building_id", inplace=True) before_services.set_index("service_id", inplace=True) before_services.drop_duplicates("geometry", inplace=True) + before_services = before_services[ + ~before_services.index.duplicated(keep="first") + ].copy() if target_scenario_buildings.empty: local_crs = context_buildings.estimate_utm_crs() else: @@ -240,14 +246,14 @@ async def calculate_effects( before_prove_data = await asyncio.to_thread( objectnat_calculator.evaluate_provision, buildings=before_buildings, - services=before_services, + services=before_services[~before_services.index.duplicated(keep="first")], matrix=before_matrix, service_normative=normative_data["normative_value"], ) after_prove_data = await asyncio.to_thread( objectnat_calculator.evaluate_provision, buildings=after_buildings, - services=after_services, + services=after_services[~after_services.index.duplicated(keep="first")], matrix=after_matrix, service_normative=normative_data["normative_value"], ) From bf42f658da4f9178db71cbfc8d65cb67b4e0e252 Mon Sep 17 00:00:00 2001 From: Leon <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 14 Nov 2025 19:26:18 +0300 Subject: [PATCH 28/61] fix(api_handler): - fixed unexpected error handling --- app/common/api_handler/api_handler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index 3f188e9..b83ef1f 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -38,8 +38,15 @@ async def _check_response_status( elif response.status == 500: if response.content_type == "application/json": response_info = await response.json() - if "reset by peer" in await response_info["error"]: + if "reset by peer" in response_info: return None + else: + raise http_exception( + 500, + "Couldn't get data from API", + _input=repr(response.url), + _detail=response_info, + ) else: response_info = await response.text() raise http_exception( From 27397cb6b06127bdafa51a9bf3fac5d30379d5b8 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 22 Dec 2025 22:34:29 +0300 Subject: [PATCH 29/61] fix(effects_service): - added none capacity handling --- app/effects/effects_service.py | 5 +++++ app/effects/modules/attribute_parser.py | 10 +++++++--- app/effects/modules/data_restorator.py | 4 ++++ app/effects/modules/effects_api_gateway.py | 18 +++++++++++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index a71ec05..c2ec911 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -16,6 +16,7 @@ objectnat_calculator, ) from .shemas.effects_base_schema import EffectsSchema +from ..dependencies import urban_api_handler class EffectsService: @@ -98,6 +99,7 @@ async def calculate_effects( project_territory = await effects_api_gateway.get_project_territory( effects_params.project_id, token ) + service_default_capacity = await effects_api_gateway.get_default_capacity(service_type_id=effects_params.service_type_id) normative_data = await effects_api_gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], @@ -139,6 +141,7 @@ async def calculate_effects( ) context_services = await attribute_parser.parse_all_from_services( services=context_services, + service_default_capacity=service_default_capacity ) target_scenario_population = ( await effects_api_gateway.get_scenario_population_data( @@ -166,6 +169,7 @@ async def calculate_effects( ) target_scenario_services = await attribute_parser.parse_all_from_services( services=target_scenario_services, + service_default_capacity=service_default_capacity ) base_scenario_buildings = await effects_api_gateway.get_scenario_buildings( scenario_id=project_data["base_scenario"]["id"], token=token @@ -187,6 +191,7 @@ async def calculate_effects( ) base_scenario_services = await attribute_parser.parse_all_from_services( services=base_scenario_services, + service_default_capacity=service_default_capacity ) after_buildings = await asyncio.to_thread( pd.concat, objs=[context_buildings, target_scenario_buildings] diff --git a/app/effects/modules/attribute_parser.py b/app/effects/modules/attribute_parser.py index 94f745b..091500e 100644 --- a/app/effects/modules/attribute_parser.py +++ b/app/effects/modules/attribute_parser.py @@ -53,19 +53,20 @@ async def parse_all_from_buildings( @staticmethod def _parse_service_capacity( services: gpd.GeoDataFrame, + service_default_capacity: int ) -> gpd.GeoDataFrame: """ Function parses capacity attributes from nested response Args: services (gpd.GeoDataFrame): nested response from api as feature collection + service_default_capacity (int): default capacity to fill Returns: gpd.GeoDataFrame: service capacity with parsed storeys data. Can be empty """ services["capacity"] = ( - services["services"].apply(lambda x: x[0].get("capacity")).astype(int) + services["services"].apply(lambda x: x[0].get("capacity")).fillna(service_default_capacity).astype(int) ) - services["capacity"] = services["capacity"].fillna(0) return services @staticmethod @@ -86,14 +87,17 @@ def _parse_service_id(services: gpd.GeoDataFrame) -> gpd.GeoDataFrame: async def parse_all_from_services( self, services: gpd.GeoDataFrame, + service_default_capacity: int ) -> gpd.GeoDataFrame: """ Function parses all required data from service request data Args: services (gpd.GeoDataFrame): nested response from api as feature collection + service_default_capacity(int): service default capacity value Returns: gpd.GeoDataFrame: service capacity with parsed storeys data. Can be empty """ + services = services.copy() if services.empty: return services @@ -102,7 +106,7 @@ async def parse_all_from_services( services=services, ) services = await asyncio.to_thread( - self._parse_service_capacity, services=services + self._parse_service_capacity, services=services, service_default_capacity=service_default_capacity ) services = services.drop( [ diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index 62a9196..65238e4 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -152,5 +152,9 @@ def restore_demands( _detail={"available_demand_type": ["unit", "capacity"]}, ) + def restore_capacity(services: gpd.GeoDataFrame, service_capacity_normative: int) -> gpd.GeoDataFrame: + + services["capacity"] = service_capacity_normative + data_restorator = DataRestorator() diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index fce428c..55e75da 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -41,7 +41,7 @@ async def get_service_normative( request_ter_id = territory_id response_df = pd.DataFrame.from_records(response) response_df["service_type_id"] = response_df["service_type"].apply( - lambda x: x["id"] + lambda x: x["id"] if x else None ) service_type = response_df[ response_df["service_type_id"] == service_type_id @@ -307,5 +307,21 @@ async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame ) return territory_gdf + @staticmethod + async def get_default_capacity(service_type_id: int) -> int: + """ + Function retrieves default capacity data from urban_api + Args: + service_type_id (int): service type id to get default capacity data from + Returns: + int: default capacity value + """ + + service_types = await urban_api_handler.get( + endpoint_url="/api/v1/service_types" + ) + service_types_df = pd.DataFrame.from_records(service_types).fillna(0) + return service_types_df[service_types_df["service_type_id"] == service_type_id].iloc[0]["capacity_modeled"] + effects_api_gateway = EffectsAPIGateway() From 514062dae0435aaa0f69ad16c26040c86dae5d43 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 22 Dec 2025 22:35:46 +0300 Subject: [PATCH 30/61] style: - ran pre-commit with black and isort for all files --- app/effects/effects_service.py | 13 +++++++------ app/effects/modules/attribute_parser.py | 16 +++++++++------- app/effects/modules/data_restorator.py | 4 +++- app/effects/modules/effects_api_gateway.py | 4 +++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index c2ec911..bd252ce 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -7,6 +7,7 @@ from app.common.exceptions.http_exception_wrapper import http_exception +from ..dependencies import urban_api_handler from .dto.effects_dto import EffectsDTO from .modules import ( attribute_parser, @@ -16,7 +17,6 @@ objectnat_calculator, ) from .shemas.effects_base_schema import EffectsSchema -from ..dependencies import urban_api_handler class EffectsService: @@ -99,7 +99,9 @@ async def calculate_effects( project_territory = await effects_api_gateway.get_project_territory( effects_params.project_id, token ) - service_default_capacity = await effects_api_gateway.get_default_capacity(service_type_id=effects_params.service_type_id) + service_default_capacity = await effects_api_gateway.get_default_capacity( + service_type_id=effects_params.service_type_id + ) normative_data = await effects_api_gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], @@ -140,8 +142,7 @@ async def calculate_effects( _detail={}, ) context_services = await attribute_parser.parse_all_from_services( - services=context_services, - service_default_capacity=service_default_capacity + services=context_services, service_default_capacity=service_default_capacity ) target_scenario_population = ( await effects_api_gateway.get_scenario_population_data( @@ -169,7 +170,7 @@ async def calculate_effects( ) target_scenario_services = await attribute_parser.parse_all_from_services( services=target_scenario_services, - service_default_capacity=service_default_capacity + service_default_capacity=service_default_capacity, ) base_scenario_buildings = await effects_api_gateway.get_scenario_buildings( scenario_id=project_data["base_scenario"]["id"], token=token @@ -191,7 +192,7 @@ async def calculate_effects( ) base_scenario_services = await attribute_parser.parse_all_from_services( services=base_scenario_services, - service_default_capacity=service_default_capacity + service_default_capacity=service_default_capacity, ) after_buildings = await asyncio.to_thread( pd.concat, objs=[context_buildings, target_scenario_buildings] diff --git a/app/effects/modules/attribute_parser.py b/app/effects/modules/attribute_parser.py index 091500e..581e4ee 100644 --- a/app/effects/modules/attribute_parser.py +++ b/app/effects/modules/attribute_parser.py @@ -52,8 +52,7 @@ async def parse_all_from_buildings( @staticmethod def _parse_service_capacity( - services: gpd.GeoDataFrame, - service_default_capacity: int + services: gpd.GeoDataFrame, service_default_capacity: int ) -> gpd.GeoDataFrame: """ Function parses capacity attributes from nested response @@ -65,7 +64,10 @@ def _parse_service_capacity( """ services["capacity"] = ( - services["services"].apply(lambda x: x[0].get("capacity")).fillna(service_default_capacity).astype(int) + services["services"] + .apply(lambda x: x[0].get("capacity")) + .fillna(service_default_capacity) + .astype(int) ) return services @@ -85,9 +87,7 @@ def _parse_service_id(services: gpd.GeoDataFrame) -> gpd.GeoDataFrame: return services async def parse_all_from_services( - self, - services: gpd.GeoDataFrame, - service_default_capacity: int + self, services: gpd.GeoDataFrame, service_default_capacity: int ) -> gpd.GeoDataFrame: """ Function parses all required data from service request data @@ -106,7 +106,9 @@ async def parse_all_from_services( services=services, ) services = await asyncio.to_thread( - self._parse_service_capacity, services=services, service_default_capacity=service_default_capacity + self._parse_service_capacity, + services=services, + service_default_capacity=service_default_capacity, ) services = services.drop( [ diff --git a/app/effects/modules/data_restorator.py b/app/effects/modules/data_restorator.py index 65238e4..33fb0a1 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/effects/modules/data_restorator.py @@ -152,7 +152,9 @@ def restore_demands( _detail={"available_demand_type": ["unit", "capacity"]}, ) - def restore_capacity(services: gpd.GeoDataFrame, service_capacity_normative: int) -> gpd.GeoDataFrame: + def restore_capacity( + services: gpd.GeoDataFrame, service_capacity_normative: int + ) -> gpd.GeoDataFrame: services["capacity"] = service_capacity_normative diff --git a/app/effects/modules/effects_api_gateway.py b/app/effects/modules/effects_api_gateway.py index 55e75da..447c277 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/effects/modules/effects_api_gateway.py @@ -321,7 +321,9 @@ async def get_default_capacity(service_type_id: int) -> int: endpoint_url="/api/v1/service_types" ) service_types_df = pd.DataFrame.from_records(service_types).fillna(0) - return service_types_df[service_types_df["service_type_id"] == service_type_id].iloc[0]["capacity_modeled"] + return service_types_df[ + service_types_df["service_type_id"] == service_type_id + ].iloc[0]["capacity_modeled"] effects_api_gateway = EffectsAPIGateway() From c6024b592fd758e4d7101e3c6c022335d80fa7fb Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 25 Dec 2025 00:35:55 +0300 Subject: [PATCH 31/61] feat(effects_service): - added name mappings - upgraded dev requirements --- app/effects/effects_service.py | 83 +++++++++++++++++++++++++--- app/effects/modules/__init__.py | 6 ++ app/effects/modules/name_mappings.py | 41 ++++++++++++++ requirements-dev.txt | 15 ----- 4 files changed, 122 insertions(+), 23 deletions(-) create mode 100644 app/effects/modules/name_mappings.py diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index bd252ce..1f961d0 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -7,9 +7,12 @@ from app.common.exceptions.http_exception_wrapper import http_exception -from ..dependencies import urban_api_handler from .dto.effects_dto import EffectsDTO from .modules import ( + ATTRIBUTES_MAP, + BUILDINGS_DROP_COLUMNS, + EFFECTS_MAP, + SERVICE_DROP_COLUMNS, attribute_parser, data_restorator, effects_api_gateway, @@ -276,23 +279,87 @@ async def calculate_effects( result = { "before_prove_data": { "buildings": json.loads( - before_prove_data["buildings"].to_crs(4326).to_json() + before_prove_data["buildings"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in before_prove_data["buildings"].columns + } + ) + .drop(columns=BUILDINGS_DROP_COLUMNS) + .to_crs(4326) + .to_json() ), "services": json.loads( - before_prove_data["services"].to_crs(4326).to_json() + before_prove_data["services"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in before_prove_data["services"].columns + } + ) + .drop(columns=SERVICE_DROP_COLUMNS) + .to_crs(4326) + .to_json() + ), + "links": json.loads( + before_prove_data["links"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in before_prove_data["links"].columns + } + ) + .to_crs(4326) + .to_json() ), - "links": json.loads(before_prove_data["links"].to_crs(4326).to_json()), }, "after_prove_data": { "buildings": json.loads( - after_prove_data["buildings"].to_crs(4326).to_json() + after_prove_data["buildings"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in after_prove_data["buildings"].columns + } + ) + .drop(columns=BUILDINGS_DROP_COLUMNS) + .to_crs(4326) + .to_json() ), "services": json.loads( - after_prove_data["services"].to_crs(4326).to_json() + after_prove_data["services"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in after_prove_data["services"].columns + } + ) + .drop(columns=SERVICE_DROP_COLUMNS) + .to_crs(4326) + .to_json() + ), + "links": json.loads( + after_prove_data["links"] + .rename( + columns={ + k: v + for k, v in ATTRIBUTES_MAP.items() + if k in after_prove_data["links"].columns + } + ) + .to_crs(4326) + .to_json() ), - "links": json.loads(after_prove_data["links"].to_crs(4326).to_json()), }, - "effects": json.loads(effects.to_crs(4326).to_json()), + "effects": json.loads( + effects.rename(columns=EFFECTS_MAP).to_crs(4326).to_json() + ), "pivot": pivot, } return EffectsSchema(**result) diff --git a/app/effects/modules/__init__.py b/app/effects/modules/__init__.py index 40d2f2d..44b199a 100644 --- a/app/effects/modules/__init__.py +++ b/app/effects/modules/__init__.py @@ -2,4 +2,10 @@ from .data_restorator import data_restorator from .effects_api_gateway import effects_api_gateway from .matrix_builder import matrix_builder +from .name_mappings import ( + ATTRIBUTES_MAP, + BUILDINGS_DROP_COLUMNS, + EFFECTS_MAP, + SERVICE_DROP_COLUMNS, +) from .objectnat_calculator import objectnat_calculator diff --git a/app/effects/modules/name_mappings.py b/app/effects/modules/name_mappings.py new file mode 100644 index 0000000..15632ec --- /dev/null +++ b/app/effects/modules/name_mappings.py @@ -0,0 +1,41 @@ +ATTRIBUTES_MAP = { + "storeys_count": "Количество этажей", + "population": "Население (чел)", + "demand": "Спрос (чел)", + "demand_left": "Неудовлетворённый спрос (чел)", + "distance": "Расстояние (м)", + "avg_dist": "Средняя доступность до сервиса (м)", + "capacity": "Вместимость (чел)", + "capacity_left": "Профицит мест (чел)", + "living_area": "Жилая площадь (кв.м)", + "service_load": "Нагрузка на сервис", + "min_dist": "Минмиальное расстояне до сервиса (м)", + "building_index": "ID здания", + "service_index": "ID сервиса", + "supplyed_demands_within": "Удовлетворённый спрос в нормативной доступности (чел)", + "supplyed_demands_without": "Удовлетворённый спрос вне нормативной доступности (чел)", + "carried_capacity_within": "Обеспечено в радиусе нормативной доступности (чел)", + "carried_capacity_without": "Обеспечено вне радиуса нормативной доступности (чел)", + "provison_value": "Оценка обеспеченности", + "supplyed_demands_within_before": "Удовлетворённый спрос в нормативной доступности (до) (чел)", + "us_demands_within_before": "Неудовлетворённый спрос в нормативной доступности (до) (чел)", + "supplyed_demands_without_before": "Удовлетворённый спрос вне нормативной доступности (до) (чел)", + "us_demands_without_before": "Неудовлетворённый спрос вне нормативной доступности (до) (чел)", + "supplyed_demands_within_after": "Удовлетворённый спрос в нормативной доступности (после) (чел)", + "us_demands_within_after": "Неудовлетворённый спрос в нормативной доступности (после) (чел)", + "supplyed_demands_without_after": "Удовлетворённый спрос вне нормативной доступности (после) (чел)", + "us_demands_without_after": "Неудовлетворённый спрос вне нормативной доступности (после) (чел)", +} + +EFFECTS_MAP = { + "absolute_total": "Абсолютный эффект (чел)", + "index_total": "Индексный эффект", + "absolute_scenario_project": "Абсолютный эффект на территории проекта", + "index_scenario_project": "Индексный эффект на территории проекта", + "absolute_within": "Абсолютный эффект в нормативной доступности", + "demand": "Спрос (чел)", + "is_project": "Проектный объект", +} + +SERVICE_DROP_COLUMNS = ["is_scenario_object", "is_locked"] +BUILDINGS_DROP_COLUMNS = SERVICE_DROP_COLUMNS + ["is_project"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 92c2d65..ec9f058 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1 @@ -aiohttp~=3.11.12 -fastapi~=0.115.8 -geopandas~=0.14.4 -IDU-config~=1.0.2 -loguru~=0.7.3 -numba~=0.60.0 -numpy~=1.26.4 -ObjectNat~=0.2.6 -pandas~=2.2.3 -pydantic~=2.10.6 -scipy~=1.15.1 -requests~=2.32.3 -uvicorn~=0.34.0 -gunicorn~=23.0.0 -requests~=2.32.5 pre-commit~=4.3.0 \ No newline at end of file From d2ce484526012799146591fc0b7f29517cf7c588 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 19 Jan 2026 13:35:16 +0300 Subject: [PATCH 32/61] feat(otel_agent): - added prometheus to api --- Dockerfile | 2 +- app/__version__.py | 1 + app/main.py | 22 ++++- app/observability/__init__.py | 2 + app/observability/config.py | 14 +++ app/observability/metrics.py | 129 ++++++++++++++++++++++++++++ app/observability/metrics_server.py | 22 +++++ app/observability/otel_agent.py | 53 ++++++++++++ docker-compose.actions.yml | 3 +- docker-compose.yml | 3 +- requirements.txt | Bin 474 -> 752 bytes 11 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 app/__version__.py create mode 100644 app/observability/__init__.py create mode 100644 app/observability/config.py create mode 100644 app/observability/metrics.py create mode 100644 app/observability/metrics_server.py create mode 100644 app/observability/otel_agent.py diff --git a/Dockerfile b/Dockerfile index 3520f4c..5904046 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED=1 # Enables env file ENV APP_ENV=development -#add pyppi mirror to config +# add pyppi mirror to config COPY pip.conf /etc/xdg/pip/pip.conf # Install pip requirements COPY requirements.txt . diff --git a/app/__version__.py b/app/__version__.py new file mode 100644 index 0000000..5fccf13 --- /dev/null +++ b/app/__version__.py @@ -0,0 +1 @@ +APP_VERSION = "0.1.0" diff --git a/app/main.py b/app/main.py index a1114f6..2f8d7c5 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,16 @@ +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, RedirectResponse from loguru import logger +from .__version__ import APP_VERSION from .common.exceptions.exception_handler import ExceptionHandlerMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router +from .observability import OpenTelemetryAgent, PrometheusConfig +from .observability.metrics import setup_metrics log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" @@ -16,10 +21,25 @@ ) +@asynccontextmanager +async def lifespan(): + otel_agent = OpenTelemetryAgent( + prometheus_config=PrometheusConfig( + host="0.0.0.0", + port=int(config.get("PROMETHEUS_PORT")), + ), + ) + setup_metrics() + logger.info(f"Prometheus server started on {config.get('PROMETHEUS_PORT')}") + yield + otel_agent.shutdown() + logger.info("Prometheus server was shut down") + + app = FastAPI( title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", - version=config.get("APP_VERSION"), + version=APP_VERSION, ) # Add CORS middleware diff --git a/app/observability/__init__.py b/app/observability/__init__.py new file mode 100644 index 0000000..0bfec9a --- /dev/null +++ b/app/observability/__init__.py @@ -0,0 +1,2 @@ +from .config import PrometheusConfig +from .otel_agent import OpenTelemetryAgent diff --git a/app/observability/config.py b/app/observability/config.py new file mode 100644 index 0000000..10bc273 --- /dev/null +++ b/app/observability/config.py @@ -0,0 +1,14 @@ +"""Observability config is defined here.""" + +from dataclasses import dataclass + + +@dataclass +class PrometheusConfig: + host: str + port: int + + +@dataclass +class ObservabilityConfig: + prometheus: PrometheusConfig | None = None diff --git a/app/observability/metrics.py b/app/observability/metrics.py new file mode 100644 index 0000000..3d20bbe --- /dev/null +++ b/app/observability/metrics.py @@ -0,0 +1,129 @@ +"""Application metrics are defined here.""" + +import threading +import time +from dataclasses import dataclass +from typing import Callable + +import psutil +from opentelemetry import metrics +from opentelemetry.metrics import CallbackOptions, Observation +from opentelemetry.sdk.metrics import Counter, Histogram, UpDownCounter + +from app.__version__ import APP_VERSION as VERSION + + +@dataclass +class HTTPMetrics: + request_processing_duration: Histogram + """Processing time histogram in seconds by `["method", "path"]`.""" + requests_started: Counter + """Total started requests counter by `["method", "path"]`.""" + requests_finished: Counter + """Total finished requests counter by `["method", "path", "status_code"]`.""" + errors: Counter + """Total errors (exceptions) counter by `["method", "path", "error_type", "status_code"]`.""" + inflight_requests: UpDownCounter + """Current number of requests handled simultaniously.""" + + +@dataclass +class Metrics: + http: HTTPMetrics + + +def setup_metrics() -> Metrics: + meter = metrics.get_meter("{{project_name}}") + + _setup_callback_metrics(meter) + + return Metrics( + http=HTTPMetrics( + request_processing_duration=meter.create_histogram( + "request_processing_duration", + "sec", + "Request processing duration time in seconds", + explicit_bucket_boundaries_advisory=[ + 0.05, + 0.2, + 0.3, + 0.7, + 1.0, + 1.5, + 2.5, + 5.0, + 10.0, + 20.0, + 40.0, + 60.0, + 120.0, + ], + ), + requests_started=meter.create_counter( + "requests_started_total", "1", "Total number of started requests" + ), + requests_finished=meter.create_counter( + "request_finished_total", "1", "Total number of finished requests" + ), + errors=meter.create_counter( + "request_errors_total", + "1", + "Total number of errors (exceptions) in requests", + ), + inflight_requests=meter.create_up_down_counter( + "inflight_requests", + "1", + "Current number of requests handled simultaniously", + ), + ) + ) + + +def _setup_callback_metrics(meter: metrics.Meter) -> None: + # Create observable gauge + meter.create_observable_gauge( + name="system_resource_usage", + description="System resource utilization", + unit="1", + callbacks=[_get_system_metrics_callback()], + ) + meter.create_observable_gauge( + name="application_metrics", + description="Application-specific metrics", + unit="1", + callbacks=[_get_application_metrics_callback()], + ) + + +def _get_system_metrics_callback() -> Callable[[CallbackOptions], None]: + def system_metrics_callback( + options: CallbackOptions, + ): # pylint: disable=unused-argument + """Callback function to collect system metrics""" + + # Process CPU time, a bit more information than `process_cpu_seconds_total` + cpu_times = psutil.Process().cpu_times() + yield Observation(cpu_times.user, {"resource": "cpu", "mode": "user"}) + yield Observation(cpu_times.system, {"resource": "cpu", "mode": "system"}) + + return system_metrics_callback + + +def _get_application_metrics_callback() -> Callable[[CallbackOptions], None]: + startup_time = time.time() + + def application_metrics_callback( + options: CallbackOptions, + ): # pylint: disable=unused-argument + """Callback function to collect application-specific metrics""" + # Current timestamp + yield Observation(startup_time, {"metric": "startup_time", "version": VERSION}) + yield Observation( + time.time(), {"metric": "last_update_time", "version": VERSION} + ) + + # Active threads + active_threads = threading.active_count() + yield Observation(active_threads, {"metric": "active_threads"}) + + return application_metrics_callback diff --git a/app/observability/metrics_server.py b/app/observability/metrics_server.py new file mode 100644 index 0000000..520159a --- /dev/null +++ b/app/observability/metrics_server.py @@ -0,0 +1,22 @@ +"""Prometheus server configuration class is defined here.""" + +from threading import Thread +from wsgiref.simple_server import WSGIServer + +from prometheus_client import start_http_server + + +class PrometheusServer: # pylint: disable=too-few-public-methods + + def __init__(self, port: int = 9464, host: str = "0.0.0.0"): + self._host = host + self._port = port + self._server: WSGIServer + self._thread: Thread + + self._server, self._thread = start_http_server(self._port) + + def shutdown(self): + if self._server is not None: + self._server.shutdown() + self._server = None diff --git a/app/observability/otel_agent.py b/app/observability/otel_agent.py new file mode 100644 index 0000000..9c5c232 --- /dev/null +++ b/app/observability/otel_agent.py @@ -0,0 +1,53 @@ +"""Open Telemetry agent initialization is defined here""" + +import platform +from functools import cache + +from opentelemetry import metrics +from opentelemetry.exporter.prometheus import PrometheusMetricReader +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.resources import ( + SERVICE_INSTANCE_ID, + SERVICE_NAME, + SERVICE_VERSION, + Resource, +) + +from app.__version__ import APP_VERSION + +from .config import PrometheusConfig +from .metrics_server import PrometheusServer + + +@cache +def get_resource() -> Resource: + return Resource.create( + attributes={ + SERVICE_NAME: "Sirtep API", + SERVICE_VERSION: APP_VERSION, + SERVICE_INSTANCE_ID: platform.node(), + } + ) + + +class OpenTelemetryAgent: # pylint: disable=too-few-public-methods + def __init__( + self, + prometheus_config: PrometheusConfig | None, + ): + self._resource = get_resource() + self._prometheus: PrometheusServer | None = None + + if prometheus_config is not None: + self._prometheus = PrometheusServer( + port=prometheus_config.port, host=prometheus_config.host + ) + + reader = PrometheusMetricReader() + provider = MeterProvider(resource=self._resource, metric_readers=[reader]) + metrics.set_meter_provider(provider) + + def shutdown(self) -> None: + """Stop metrics and tracing services if they were started.""" + if self._prometheus is not None: + self._prometheus.shutdown() diff --git a/docker-compose.actions.yml b/docker-compose.actions.yml index ffeb3c4..f11bb9a 100644 --- a/docker-compose.actions.yml +++ b/docker-compose.actions.yml @@ -3,7 +3,8 @@ services: image: ${IMAGE} container_name: ${CONTAINER_NAME} ports: - - 5080:80 + - "5080:80" + - "9464:9464" env_file: - .env.development restart: always diff --git a/docker-compose.yml b/docker-compose.yml index 374af27..3ac7f79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,4 +7,5 @@ services: context: . dockerfile: ./Dockerfile ports: - - 80:80 \ No newline at end of file + - "80:80" + - "9464:9464" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 428b92aa313e74bada2017eea6778b4575716bab..bbad3d6b2f6050bc43c089123273fa661e9d3650 100644 GIT binary patch delta 288 zcma)%NeaS15JjJe8^OJoh;*VTEE1Jj delta 7 OcmeysdW(6(Ek*zi$OB#g From edfb83d7136b74b66c8fb76f65cbfdbb4731d433 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 19 Feb 2026 19:49:42 +0300 Subject: [PATCH 33/61] feat(prometheus): - added prometheus --- app/__version__.py | 2 +- app/common/exceptions/exception_handler.py | 85 ------------------ app/common/middlewares/__init__.py | 0 app/common/middlewares/exception_handler.py | 90 +++++++++++++++++++ app/common/middlewares/middleware_utils.py | 12 +++ app/common/middlewares/prometheus_handler.py | 41 +++++++++ app/main.py | 8 +- app/observability/metrics.py | 2 +- app/observability/otel_agent.py | 2 +- requirements.txt | Bin 752 -> 840 bytes 10 files changed, 152 insertions(+), 90 deletions(-) delete mode 100644 app/common/exceptions/exception_handler.py create mode 100644 app/common/middlewares/__init__.py create mode 100644 app/common/middlewares/exception_handler.py create mode 100644 app/common/middlewares/middleware_utils.py create mode 100644 app/common/middlewares/prometheus_handler.py diff --git a/app/__version__.py b/app/__version__.py index 5fccf13..b87a9e6 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.0" +APP_VERSION = "0.1.1" diff --git a/app/common/exceptions/exception_handler.py b/app/common/exceptions/exception_handler.py deleted file mode 100644 index b42acfc..0000000 --- a/app/common/exceptions/exception_handler.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Exception handling middleware is defined here.""" - -import itertools -import json -import traceback - -from fastapi import FastAPI, HTTPException, Request -from loguru import logger -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.responses import JSONResponse - -from .http_exception_wrapper import http_exception - - -class ExceptionHandlerMiddleware( - BaseHTTPMiddleware -): # pylint: disable=too-few-public-methods - """Handle exceptions, so they become http response code 500 - Internal Server Error if not handled as HTTPException - previously. - Attributes: - app (FastAPI): The FastAPI application instance. - """ - - def __init__(self, app: FastAPI): - """ - Universal exception handler middleware init function. - Args: - app (FastAPI): The FastAPI application instance. - """ - - super().__init__(app) - - async def dispatch(self, request: Request, call_next): - """ - Dispatch function for sending errors to user from API - Args: - request (Request): The incoming request object. - call_next: function to extract. - """ - - try: - return await call_next(request) - except Exception as e: - request_info = { - "method": request.method, - "url": str(request.url), - "path_params": dict(request.path_params), - "query_params": dict(request.query_params), - "headers": dict(request.headers), - } - try: - request_info["body"] = await request.json() - except: - try: - request_info["body"] = str(await request.body()) - except: - request_info["body"] = "Could not read request body" - if isinstance(e, HTTPException): - return JSONResponse( - status_code=e.status_code, - content={ - "message": ( - e.detail.get("msg") - if isinstance(e.detail, dict) - else str(e.detail) - ), - "error_type": e.__class__.__name__, - "request": request_info, - "detail": ( - e.detail.get("detail") - if isinstance(e.detail, dict) - else None - ), - }, - ) - return JSONResponse( - status_code=500, - content={ - "message": "Internal server error", - "error_type": e.__class__.__name__, - "request": request_info, - "detail": str(e), - "traceback": traceback.format_exc().splitlines(), - }, - ) diff --git a/app/common/middlewares/__init__.py b/app/common/middlewares/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/common/middlewares/exception_handler.py b/app/common/middlewares/exception_handler.py new file mode 100644 index 0000000..527f72c --- /dev/null +++ b/app/common/middlewares/exception_handler.py @@ -0,0 +1,90 @@ +"""Exception handling middleware is defined here.""" + +import traceback + +from fastapi import FastAPI, Request +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import JSONResponse + +from app.common.middlewares.middleware_utils import _normalize_path +from app.observability.metrics import Metrics + + +class ExceptionHandlerMiddleware( + BaseHTTPMiddleware +): # pylint: disable=too-few-public-methods + """Handle exceptions, so they become http response code 500 - Internal Server Error if not handled as HTTPException + previously. + Attributes: + app (FastAPI): The FastAPI application instance. + """ + + def __init__(self, app: FastAPI, metrics: Metrics): + """ + Universal exception handler middleware init function. + Args: + app (FastAPI): The FastAPI application instance. + """ + + super().__init__(app) + self.metrics = metrics + + @staticmethod + async def prepare_request_info(request: Request) -> dict: + """ + Function prepares request input data + Args: + request (Request): Request instance. + Returns: + dict: Request input data. + """ + + request_info = { + "method": request.method, + "url": str(request.url), + "path_params": dict(request.path_params), + "query_params": dict(request.query_params), + "headers": dict(request.headers), + } + + try: + request_info["body"] = await request.json() + return request_info + except: + try: + request_info["body"] = str(await request.body()) + return request_info + except: + request_info["body"] = "Could not read request body" + return request_info + + async def dispatch(self, request: Request, call_next): + """ + Dispatch function for sending errors to user from API + Args: + request (Request): The incoming request object. + call_next: function to extract. + """ + + try: + return await call_next(request) + except Exception as e: + request_info = await self.prepare_request_info(request) + self.metrics.http.errors.add( + 1, + { + "method": request.method, + "path": _normalize_path(request), + "error_type": type(e).__name__, + }, + ) + return JSONResponse( + status_code=500, + content={ + "message": "Internal server error", + "error_type": e.__class__.__name__, + "request": request_info, + "detail": str(e), + "traceback": traceback.format_exc().splitlines(), + }, + ) diff --git a/app/common/middlewares/middleware_utils.py b/app/common/middlewares/middleware_utils.py new file mode 100644 index 0000000..5e3f5d0 --- /dev/null +++ b/app/common/middlewares/middleware_utils.py @@ -0,0 +1,12 @@ +from fastapi import Request + + +def _normalize_path(request: Request) -> str: + """ + Normalize path to avoid high-cardinality metrics. + """ + + route = request.scope.get("route") + if route and hasattr(route, "path"): + return route.path + return request.url.path diff --git a/app/common/middlewares/prometheus_handler.py b/app/common/middlewares/prometheus_handler.py new file mode 100644 index 0000000..b7cf9c1 --- /dev/null +++ b/app/common/middlewares/prometheus_handler.py @@ -0,0 +1,41 @@ +"""Observability middleware is defined here.""" + +import time + +from fastapi import FastAPI, Request +from starlette.middleware.base import BaseHTTPMiddleware + +from app.common.middlewares.middleware_utils import _normalize_path +from app.observability.metrics import Metrics + + +class ObservabilityMiddleware(BaseHTTPMiddleware): + + def __init__(self, app: FastAPI, metrics: Metrics): + """Obervability middleware class for http metrics with prometheus + + Args: + app (FastAPI): FastAPI app instance + metrics (Metrics): Metrics with http field connectable with prometheus + """ + super().__init__(app) + self._http_metrics = metrics.http + + async def dispatch(self, request: Request, call_next): + + path = _normalize_path(request) + method = request.method + self._http_metrics.requests_started.add(1, {"method": method, "path": path}) + self._http_metrics.inflight_requests.add(1) + start = time.monotonic() + response = await call_next(request) + duration = time.monotonic() - start + self._http_metrics.requests_finished.add( + 1, + {"method": method, "path": path, "status_code": response.status_code}, + ) + self._http_metrics.request_processing_duration.record( + duration, {"method": method, "path": path} + ) + self._http_metrics.inflight_requests.add(-1) + return response diff --git a/app/main.py b/app/main.py index 2f8d7c5..751d3d8 100644 --- a/app/main.py +++ b/app/main.py @@ -6,7 +6,8 @@ from loguru import logger from .__version__ import APP_VERSION -from .common.exceptions.exception_handler import ExceptionHandlerMiddleware +from .common.middlewares.exception_handler import ExceptionHandlerMiddleware +from .common.middlewares.prometheus_handler import ObservabilityMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router from .observability import OpenTelemetryAgent, PrometheusConfig @@ -20,6 +21,8 @@ level="INFO", ) +metrics = setup_metrics() + @asynccontextmanager async def lifespan(): @@ -50,7 +53,8 @@ async def lifespan(): allow_methods=["*"], allow_headers=["*"], ) -app.add_middleware(ExceptionHandlerMiddleware) +app.add_middleware(ExceptionHandlerMiddleware, metrics=metrics) +app.add_middleware(ObservabilityMiddleware, metrics=metrics) @app.get("/", response_model=dict[str, str]) diff --git a/app/observability/metrics.py b/app/observability/metrics.py index 3d20bbe..86c818e 100644 --- a/app/observability/metrics.py +++ b/app/observability/metrics.py @@ -33,7 +33,7 @@ class Metrics: def setup_metrics() -> Metrics: - meter = metrics.get_meter("{{project_name}}") + meter = metrics.get_meter("sirtep-api") _setup_callback_metrics(meter) diff --git a/app/observability/otel_agent.py b/app/observability/otel_agent.py index 9c5c232..509a0f3 100644 --- a/app/observability/otel_agent.py +++ b/app/observability/otel_agent.py @@ -23,7 +23,7 @@ def get_resource() -> Resource: return Resource.create( attributes={ - SERVICE_NAME: "Sirtep API", + SERVICE_NAME: "sirtep-api", SERVICE_VERSION: APP_VERSION, SERVICE_INSTANCE_ID: platform.node(), } diff --git a/requirements.txt b/requirements.txt index bbad3d6b2f6050bc43c089123273fa661e9d3650..cd740c4706d7e2ed1c9496885ea2ee25740a8184 100644 GIT binary patch delta 60 zcmeysdV+1k0;b6#OmfPp47m)640#MC44Dkc47xy^59F0Gq(a4jyvdzR`iw@CcQTnW H88QF>+_Mgc delta 20 ccmX@X_JMW70;b77m?RhtCkry0G8r%c08u#w`v3p{ From e58d186d901515af6821405777686de69fedd66d Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 3 Apr 2026 14:49:53 +0300 Subject: [PATCH 34/61] fix(prometheus): - fixed prometheus on startup --- app/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 751d3d8..99147af 100644 --- a/app/main.py +++ b/app/main.py @@ -25,7 +25,7 @@ @asynccontextmanager -async def lifespan(): +async def lifespan(app: FastAPI): otel_agent = OpenTelemetryAgent( prometheus_config=PrometheusConfig( host="0.0.0.0", @@ -43,6 +43,7 @@ async def lifespan(): title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", version=APP_VERSION, + lifespan=lifespan, ) # Add CORS middleware From c3c545201b28c8b0c85bf034e228aca86ec493e0 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:56:18 +0300 Subject: [PATCH 35/61] fix(prometheus): (#27) - fixed prometheus on startup --- app/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 751d3d8..99147af 100644 --- a/app/main.py +++ b/app/main.py @@ -25,7 +25,7 @@ @asynccontextmanager -async def lifespan(): +async def lifespan(app: FastAPI): otel_agent = OpenTelemetryAgent( prometheus_config=PrometheusConfig( host="0.0.0.0", @@ -43,6 +43,7 @@ async def lifespan(): title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", version=APP_VERSION, + lifespan=lifespan, ) # Add CORS middleware From a3a038852731cb9a7c11da31b0d1f090d13a9708 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 3 Apr 2026 15:07:33 +0300 Subject: [PATCH 36/61] fix(prometheus): - fixed decreased nm workers to 1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5904046..2a8fb14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,4 +23,4 @@ WORKDIR /app COPY . /app # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug -CMD ["gunicorn", "--bind", "0.0.0.0:80", "-k", "uvicorn.workers.UvicornWorker", "--workers", "4", "app.main:app"] +CMD ["gunicorn", "--bind", "0.0.0.0:80", "-k", "uvicorn.workers.UvicornWorker", "--workers", "1", "app.main:app"] From 8ce6ed36340489af2849fb938b4b8c690348c436 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 20 Apr 2026 13:08:25 +0300 Subject: [PATCH 37/61] feat(dependencies): - updated requirements.txt, requirements-dev.txt and .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- requirements-dev.txt | 2 +- requirements.txt | Bin 840 -> 798 bytes 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9703de4..54e2bba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/psf/black - rev: 25.1.0 + rev: 26.3.1 hooks: - id: black language_version: python3.11 - repo: https://github.com/pycqa/isort - rev: 6.0.1 + rev: 8.0.1 hooks: - id: isort name: isort (python) diff --git a/requirements-dev.txt b/requirements-dev.txt index ec9f058..3351d0b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -pre-commit~=4.3.0 \ No newline at end of file +pre-commit~=4.5.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cd740c4706d7e2ed1c9496885ea2ee25740a8184..a7a181ca6c5dcf687a5d0ae765d5d5d736865273 100644 GIT binary patch delta 91 zcmX@XHjizB6ss|V9)rn5MOk%Y1~VYjfPt5Riy@66k)fEOgdvw9nW2E8j=`1zq|6Ac i)b;J;Fh)7?Qid{yOrUf=LlHwBScw5hdh>e5nT!A-)Dl|& delta 107 zcmbQoc7kn!6uTjV9)lr+(L{M!Ak!4cv6z_rR=$9tk|Bj5ks*(vgdvk5nW2usmcayRWhi7Q1(L-;RmE^M#$c7klQS8m8I3o$GR|ZK0GrztUjP6A From 6be14a68e72a02bfb5ed505ee1b3f94ee2d501af Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 20 Apr 2026 18:38:48 +0300 Subject: [PATCH 38/61] feat(mcp): - mcp server in progress --- app/effects/effects_mcp.py | 62 +++++++++++++++++++++++ app/effects/effects_service.py | 30 ++++++++++- app/effects/shemas/effects_base_schema.py | 1 + 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 app/effects/effects_mcp.py diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py new file mode 100644 index 0000000..2de0bae --- /dev/null +++ b/app/effects/effects_mcp.py @@ -0,0 +1,62 @@ +from fastmcp import FastMCP +from fastmcp.server.dependencies import CurrentContext, get_access_token + +from app.effects.dto.effects_dto import EffectsDTO + +from .effects_service import effects_service + +effects_mcp = FastMCP("Object Effects MCP server") + + +@effects_mcp.tool( + name="CalculateObjectEffects", + title="Get provision effects for service", + description=""" + Retrieve service provision effects by service id. + If total population is provided, demand is restored from it. Otherwise, population is restored from living square. + + Args to select: + + Returns effects layers with estimated pivot info for llm analyses. + Response format: + { + "before_prove_data": { + "buildings": FeatureCollection, + "services": FeatureCollection, + "": FeatureCollection + }, + "after_prove_data": { + "buildings": FeatureCollection, + "services": FeatureCollection, + "links": FeatureCollection + }, + "effects": FeatureCollection, + "pivot": { + "sum_absolute_total": int, + "average_absolute_total": float, + "median_absolute_total": int, + "average_index_total": float, + "median_index_total": int, + "sum_absolute_within": int, + "average_absolute_within": float, + "median_absolute_within": int, + }, + "": str + } + """, +) +async def calc_provision( + service_type_id: int, target_population: int | None = None, ctx=CurrentContext() +): + + project_id = int(ctx.request_context.meta.project_id) + scenario_id = int(ctx.request_context.meta.scenario_id) + token = get_access_token() + effects_dto = EffectsDTO( + project_id=project_id, + scenario_id=scenario_id, + service_type_id=service_type_id, + target_population=target_population, + ) + result = await effects_service.calculate_effects(effects_dto, token, for_mcp=True) + return result diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 1f961d0..bab210c 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -82,13 +82,14 @@ async def _get_pivot( # ToDo Split function # ToDo Rewrite to context ids normal handling async def calculate_effects( - self, effects_params: EffectsDTO, token: str + self, effects_params: EffectsDTO, token: str, for_mcp: bool = False ) -> EffectsSchema: """ Calculate provision effects by project data and target scenario Args: effects_params (EffectsDTO): Project data token (str): Authorization token + for_mcp (bool): If flag enabled adds string description for llm. Default to false. Returns: gpd.GeoDataFrame: Provision effects """ @@ -362,7 +363,34 @@ async def calculate_effects( ), "pivot": pivot, } + if for_mcp: + result["text_pivot"] = await self.form_llm_context( + before_prove_data["buildings"], + after_prove_data["buildings"], + before_prove_data["services"], + after_prove_data["services"], + ) return EffectsSchema(**result) + async def form_llm_context( + self, + buildings_before: gpd.GeoDataFrame, + buildings_after: gpd.GeoDataFrame, + services_before: gpd.GeoDataFrame, + services_after: gpd.GeoDataFrame, + ) -> str: + """ + Function forms text repr stats from calculated provision data for llm. + Args: + buildings_before (gpd.GeoDataFrame): Buildings provision layers before. + buildings_after (gpd.GeoDataFrame): Buildings provision layers after. + services_before (gpd.GeoDataFrame): Services provision layers before. + services_after (gpd.GeoDataFrame): Services provision layers after. + Returns: + str: Text representation for formed stats in json string. + """ + + {""} + effects_service = EffectsService() diff --git a/app/effects/shemas/effects_base_schema.py b/app/effects/shemas/effects_base_schema.py index 14ddcc8..d8b69e2 100644 --- a/app/effects/shemas/effects_base_schema.py +++ b/app/effects/shemas/effects_base_schema.py @@ -60,3 +60,4 @@ class EffectsSchema(BaseModel): after_prove_data: ProvisionSchema effects: FeatureCollectionSchema pivot: PivotSchema + text_pivot: str | None = None From 0ac78bdd21fa0a5b0ce46fcd10ff64120ccf3ca4 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 22 Apr 2026 15:12:50 +0300 Subject: [PATCH 39/61] feat(provision, mcp): - mcp server added - added provision endpoints --- app/{effects => common}/modules/__init__.py | 0 .../modules/attribute_parser.py | 0 .../modules/data_restorator.py | 6 - .../modules/effects_api_gateway.py | 48 +++ .../modules/matrix_builder.py | 0 .../modules/name_mappings.py | 3 +- .../modules/objectnat_calculator.py | 0 app/{effects => }/dto/__init__.py | 0 .../effects_dto.py => dto/provision_dto.py} | 8 +- app/effects/effects_controller.py | 11 +- app/effects/effects_mcp.py | 6 +- app/effects/effects_service.py | 389 +++++++++++++++++- app/effects/shemas/effects_base_schema.py | 36 +- app/main.py | 7 +- app/mcp.py | 3 + app/provision/__init__.py | 0 app/provision/provision_controller.py | 19 + app/provision/provision_service.py | 166 ++++++++ app/schemas/__init__.py | 0 app/schemas/provision_base_schema.py | 37 ++ requirements.txt | Bin 798 -> 992 bytes 21 files changed, 665 insertions(+), 74 deletions(-) rename app/{effects => common}/modules/__init__.py (100%) rename app/{effects => common}/modules/attribute_parser.py (100%) rename app/{effects => common}/modules/data_restorator.py (96%) rename app/{effects => common}/modules/effects_api_gateway.py (85%) rename app/{effects => common}/modules/matrix_builder.py (100%) rename app/{effects => common}/modules/name_mappings.py (97%) rename app/{effects => common}/modules/objectnat_calculator.py (100%) rename app/{effects => }/dto/__init__.py (100%) rename app/{effects/dto/effects_dto.py => dto/provision_dto.py} (62%) create mode 100644 app/mcp.py create mode 100644 app/provision/__init__.py create mode 100644 app/provision/provision_controller.py create mode 100644 app/provision/provision_service.py create mode 100644 app/schemas/__init__.py create mode 100644 app/schemas/provision_base_schema.py diff --git a/app/effects/modules/__init__.py b/app/common/modules/__init__.py similarity index 100% rename from app/effects/modules/__init__.py rename to app/common/modules/__init__.py diff --git a/app/effects/modules/attribute_parser.py b/app/common/modules/attribute_parser.py similarity index 100% rename from app/effects/modules/attribute_parser.py rename to app/common/modules/attribute_parser.py diff --git a/app/effects/modules/data_restorator.py b/app/common/modules/data_restorator.py similarity index 96% rename from app/effects/modules/data_restorator.py rename to app/common/modules/data_restorator.py index 33fb0a1..62a9196 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/common/modules/data_restorator.py @@ -152,11 +152,5 @@ def restore_demands( _detail={"available_demand_type": ["unit", "capacity"]}, ) - def restore_capacity( - services: gpd.GeoDataFrame, service_capacity_normative: int - ) -> gpd.GeoDataFrame: - - services["capacity"] = service_capacity_normative - data_restorator = DataRestorator() diff --git a/app/effects/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py similarity index 85% rename from app/effects/modules/effects_api_gateway.py rename to app/common/modules/effects_api_gateway.py index 447c277..829f4f5 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -325,5 +325,53 @@ async def get_default_capacity(service_type_id: int) -> int: service_types_df["service_type_id"] == service_type_id ].iloc[0]["capacity_modeled"] + @staticmethod + async def get_services_with_context( + scenario_id: int, service_type_id: int, token: str | None = None + ) -> gpd.GeoDataFrame: + """ + Function retrieves service by service_type_id for scenario ID from urban api with context. + Args: + scenario_id (int): Scenario ID from Urban API. + service_type_id (int): Service type ID from Urban API. + token (str | None): Auth token to retrieve data from Urban API. Default to None + Returns: + gpd.GeoDataFrame: layer with services in 4326 crs. + """ + + services = await urban_api_handler.get( + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", + params={ + "service_type_id": service_type_id, + "include_scenario_objects": True, + }, + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return gpd.GeoDataFrame.from_features(services, crs=4326) + + @staticmethod + async def get_physical_objects_with_context( + scenario_id: int, physical_object_type_id: int, token: str | None = None + ): + """ + Function retrieves physical objects by physical_object_type_id for scenario ID from urban api with context. + Args: + scenario_id (int): Scenario ID from Urban API. + physical_object_type_id (int): Physical object type ID from Urban API. + token (str | None): Auth token to retrieve data from Urban API. Default to None + Returns: + gpd.GeoDataFrame: layer with physical_objects in 4326 crs. + """ + + physical_objects = await urban_api_handler.get( + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", + params={ + "physical_object_type_id": physical_object_type_id, + "include_scenario_objects": True, + }, + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return gpd.GeoDataFrame.from_features(physical_objects, crs=4326) + effects_api_gateway = EffectsAPIGateway() diff --git a/app/effects/modules/matrix_builder.py b/app/common/modules/matrix_builder.py similarity index 100% rename from app/effects/modules/matrix_builder.py rename to app/common/modules/matrix_builder.py diff --git a/app/effects/modules/name_mappings.py b/app/common/modules/name_mappings.py similarity index 97% rename from app/effects/modules/name_mappings.py rename to app/common/modules/name_mappings.py index 15632ec..1a185e4 100644 --- a/app/effects/modules/name_mappings.py +++ b/app/common/modules/name_mappings.py @@ -25,6 +25,7 @@ "us_demands_within_after": "Неудовлетворённый спрос в нормативной доступности (после) (чел)", "supplyed_demands_without_after": "Удовлетворённый спрос вне нормативной доступности (после) (чел)", "us_demands_without_after": "Неудовлетворённый спрос вне нормативной доступности (после) (чел)", + "is_scenario_object": "Сценарный объект", } EFFECTS_MAP = { @@ -37,5 +38,5 @@ "is_project": "Проектный объект", } -SERVICE_DROP_COLUMNS = ["is_scenario_object", "is_locked"] +SERVICE_DROP_COLUMNS = ["is_locked"] BUILDINGS_DROP_COLUMNS = SERVICE_DROP_COLUMNS + ["is_project"] diff --git a/app/effects/modules/objectnat_calculator.py b/app/common/modules/objectnat_calculator.py similarity index 100% rename from app/effects/modules/objectnat_calculator.py rename to app/common/modules/objectnat_calculator.py diff --git a/app/effects/dto/__init__.py b/app/dto/__init__.py similarity index 100% rename from app/effects/dto/__init__.py rename to app/dto/__init__.py diff --git a/app/effects/dto/effects_dto.py b/app/dto/provision_dto.py similarity index 62% rename from app/effects/dto/effects_dto.py rename to app/dto/provision_dto.py index 93dbffe..5fc9375 100644 --- a/app/effects/dto/effects_dto.py +++ b/app/dto/provision_dto.py @@ -1,14 +1,12 @@ -from typing import Optional - from pydantic import BaseModel, Field -class EffectsDTO(BaseModel): +class ProvisionDTO(BaseModel): project_id: int = Field(..., examples=[72], description="Project ID") scenario_id: int = Field(..., examples=[192], description="Scenario ID") - service_type_id: int = Field(..., examples=[7], description="Service type ID") - target_population: Optional[int] = Field( + service_type_id: int = Field(..., examples=[22], description="Service type ID") + target_population: int | None = Field( default=None, examples=[200], description="Target population for project territory", diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index 507422d..d4e92e9 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dto.provision_dto import ProvisionDTO -from .dto.effects_dto import EffectsDTO from .effects_service import effects_service from .shemas.effects_base_schema import EffectsSchema @@ -13,15 +13,8 @@ @effects_router.get("/evaluate_provision", response_model=EffectsSchema) async def calculate_effects( - params: Annotated[EffectsDTO, Depends(EffectsDTO)], + params: Annotated[ProvisionDTO, Depends(ProvisionDTO)], token: str = Depends(verify_bearer_token), ) -> EffectsSchema: - """ - Get method for retrieving effects with objectnat - Params: - - project ID: Project ID - scenario ID: Scenario ID - """ return await effects_service.calculate_effects(params, token) diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index 2de0bae..0676cf4 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -1,7 +1,7 @@ from fastmcp import FastMCP from fastmcp.server.dependencies import CurrentContext, get_access_token -from app.effects.dto.effects_dto import EffectsDTO +from app.dto.provision_dto import ProvisionDTO from .effects_service import effects_service @@ -45,14 +45,14 @@ } """, ) -async def calc_provision( +async def calc_provision_effects( service_type_id: int, target_population: int | None = None, ctx=CurrentContext() ): project_id = int(ctx.request_context.meta.project_id) scenario_id = int(ctx.request_context.meta.scenario_id) token = get_access_token() - effects_dto = EffectsDTO( + effects_dto = ProvisionDTO( project_id=project_id, scenario_id=scenario_id, service_type_id=service_type_id, diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index bab210c..8d1c67e 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -6,9 +6,7 @@ from loguru import logger from app.common.exceptions.http_exception_wrapper import http_exception - -from .dto.effects_dto import EffectsDTO -from .modules import ( +from app.common.modules import ( ATTRIBUTES_MAP, BUILDINGS_DROP_COLUMNS, EFFECTS_MAP, @@ -19,6 +17,8 @@ matrix_builder, objectnat_calculator, ) +from app.dto.provision_dto import ProvisionDTO + from .shemas.effects_base_schema import EffectsSchema @@ -82,12 +82,12 @@ async def _get_pivot( # ToDo Split function # ToDo Rewrite to context ids normal handling async def calculate_effects( - self, effects_params: EffectsDTO, token: str, for_mcp: bool = False + self, effects_params: ProvisionDTO, token: str, for_mcp: bool = False ) -> EffectsSchema: """ Calculate provision effects by project data and target scenario Args: - effects_params (EffectsDTO): Project data + effects_params (ProvisionDTO): Project data token (str): Authorization token for_mcp (bool): If flag enabled adds string description for llm. Default to false. Returns: @@ -372,25 +372,384 @@ async def calculate_effects( ) return EffectsSchema(**result) + @staticmethod async def form_llm_context( - self, - buildings_before: gpd.GeoDataFrame, - buildings_after: gpd.GeoDataFrame, - services_before: gpd.GeoDataFrame, - services_after: gpd.GeoDataFrame, + before_buildings: gpd.GeoDataFrame, + after_buildings: gpd.GeoDataFrame, + before_services: gpd.GeoDataFrame, + after_services: gpd.GeoDataFrame, ) -> str: """ Function forms text repr stats from calculated provision data for llm. Args: - buildings_before (gpd.GeoDataFrame): Buildings provision layers before. - buildings_after (gpd.GeoDataFrame): Buildings provision layers after. - services_before (gpd.GeoDataFrame): Services provision layers before. - services_after (gpd.GeoDataFrame): Services provision layers after. + before_buildings (gpd.GeoDataFrame): Buildings provision layers before. + after_buildings (gpd.GeoDataFrame): Buildings provision layers after. + before_services (gpd.GeoDataFrame): Services provision layers before. + after_services (gpd.GeoDataFrame): Services provision layers after. Returns: str: Text representation for formed stats in json string. """ - {""} + before_buildings_all = before_buildings.copy() + after_buildings_all = after_buildings.copy() + before_services_all = before_services.copy() + after_services_all = after_services.copy() + before_buildings_context = before_buildings[ + before_buildings["is_scenario_object"] == False + ] + after_buildings_context = after_buildings[ + after_buildings["is_scenario_object"] == False + ] + before_services_context = before_services[ + before_services["is_scenario_object"] == False + ] + after_services_context = after_services[ + after_services["is_scenario_object"] == False + ] + before_buildings_project = before_buildings[ + before_buildings["is_scenario_object"] == True + ] + after_buildings_project = after_buildings[ + after_buildings["is_scenario_object"] == True + ] + before_services_project = before_services[ + before_services["is_scenario_object"] == True + ] + after_services_project = after_services[ + after_services["is_scenario_object"] == True + ] + all_provision_before = int( + before_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_after = int( + after_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_provision_within_before = int( + before_buildings_all[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_within_after = int( + after_buildings_all[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + all_provision_without_before = int( + before_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_without_after = int( + after_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_total_capacity_before = int(before_services_all["Вместимость (чел)"].sum()) + all_total_capacity_after = int(after_services_all["Вместимость (чел)"].sum()) + all_demand_before = int(before_buildings_all["Спрос (чел)"].sum()) + all_demand_after = int(after_buildings_all["Спрос (чел)"].sum()) + all_unmet_demand_before = int( + before_buildings_all["Неудовлетворённый спрос (чел)"].sum() + ) + all_unmet_demand_after = int( + after_buildings_all["Неудовлетворённый спрос (чел)"].sum() + ) + all_unmet_demand_within_before = int( + before_buildings_all[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + all_unmet_demand_within_after = int( + after_buildings_all[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + all_unmet_demand_without_before = int( + before_buildings_all[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_unmet_demand_without_after = int( + after_buildings_all[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_balance_before = all_total_capacity_before - all_demand_before + all_balance_after = all_total_capacity_after - all_demand_after + all_deficit_before = min(0, all_balance_before) + all_deficit_after = min(0, all_balance_after) + all_surplus_before = max(0, all_balance_before) + all_surplus_after = max(0, all_balance_after) + context_provision_before = int( + before_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_after = int( + after_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_provision_within_before = int( + before_buildings_context[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_within_after = int( + after_buildings_context[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + context_provision_without_before = int( + before_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_without_after = int( + after_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_total_capacity_before = int( + before_services_context["Вместимость (чел)"].sum() + ) + context_total_capacity_after = int( + after_services_context["Вместимость (чел)"].sum() + ) + context_demand_before = int(before_buildings_context["Спрос (чел)"].sum()) + context_demand_after = int(after_buildings_context["Спрос (чел)"].sum()) + context_unmet_demand_before = int( + before_buildings_context["Неудовлетворённый спрос (чел)"].sum() + ) + context_unmet_demand_after = int( + after_buildings_context["Неудовлетворённый спрос (чел)"].sum() + ) + context_unmet_demand_within_before = int( + before_buildings_context[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + context_unmet_demand_within_after = int( + after_buildings_context[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + context_unmet_demand_without_before = int( + before_buildings_context[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_unmet_demand_without_after = int( + after_buildings_context[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_balance_before = context_total_capacity_before - context_demand_before + context_balance_after = context_total_capacity_after - context_demand_after + context_deficit_before = min(0, context_balance_before) + context_deficit_after = min(0, context_balance_after) + context_surplus_before = max(0, context_balance_before) + context_surplus_after = max(0, context_balance_after) + project_provision_before = int( + before_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_after = int( + after_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_provision_within_before = int( + before_buildings_project[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_within_after = int( + after_buildings_project[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + project_provision_without_before = int( + before_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_without_after = int( + after_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_total_capacity_before = int( + before_services_project["Вместимость (чел)"].sum() + ) + project_total_capacity_after = int( + after_services_project["Вместимость (чел)"].sum() + ) + project_demand_before = int(before_buildings_project["Спрос (чел)"].sum()) + project_demand_after = int(after_buildings_project["Спрос (чел)"].sum()) + project_unmet_demand_before = int( + before_buildings_project["Неудовлетворённый спрос (чел)"].sum() + ) + project_unmet_demand_after = int( + after_buildings_project["Неудовлетворённый спрос (чел)"].sum() + ) + project_unmet_demand_within_before = int( + before_buildings_project[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + project_unmet_demand_within_after = int( + after_buildings_project[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + project_unmet_demand_without_before = int( + before_buildings_project[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_unmet_demand_without_after = int( + after_buildings_project[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_balance_before = project_total_capacity_before - project_demand_before + project_balance_after = project_total_capacity_after - project_demand_after + project_deficit_before = min(0, project_balance_before) + project_deficit_after = min(0, project_balance_after) + project_surplus_before = max(0, project_balance_before) + project_surplus_after = max(0, project_balance_after) + + result = { + "all": { + "provision_before": all_provision_before, + "provision_after": all_provision_after, + "provision_delta": all_provision_after - all_provision_before, + "provision_within_before": all_provision_within_before, + "provision_within_after": all_provision_within_after, + "provision_within_delta": all_provision_within_after + - all_provision_within_before, + "provision_without_before": all_provision_without_before, + "provision_without_after": all_provision_without_after, + "provision_without_delta": all_provision_without_after + - all_provision_without_before, + "total_capacity_before": all_total_capacity_before, + "total_capacity_after": all_total_capacity_after, + "total_capacity_delta": all_total_capacity_after + - all_total_capacity_before, + "balance_before": all_balance_before, + "balance_after": all_balance_after, + "balance_delta": all_balance_after - all_balance_before, + "deficit_before": all_deficit_before, + "deficit_after": all_deficit_after, + "deficit_delta": all_deficit_after - all_deficit_before, + "surplus_before": all_surplus_before, + "surplus_after": all_surplus_after, + "surplus_delta": all_surplus_after - all_surplus_before, + "demand_before": all_demand_before, + "demand_after": all_demand_after, + "demand_delta": all_demand_after - all_demand_before, + "unmet_demand_before": all_unmet_demand_before, + "unmet_demand_after": all_unmet_demand_after, + "unmet_demand_delta": all_unmet_demand_after - all_unmet_demand_before, + "unmet_demand_within_before": all_unmet_demand_within_before, + "unmet_demand_within_after": all_unmet_demand_within_after, + "unmet_demand_within_delta": all_unmet_demand_within_after + - all_unmet_demand_within_before, + "unmet_demand_without_before": all_unmet_demand_without_before, + "unmet_demand_without_after": all_unmet_demand_without_after, + "unmet_demand_without_delta": all_unmet_demand_without_after + - all_unmet_demand_without_before, + }, + "context": { + "provision_before": context_provision_before, + "provision_after": context_provision_after, + "provision_delta": context_provision_after - context_provision_before, + "provision_within_before": context_provision_within_before, + "provision_within_after": context_provision_within_after, + "provision_within_delta": context_provision_within_after + - context_provision_within_before, + "provision_without_before": context_provision_without_before, + "provision_without_after": context_provision_without_after, + "provision_without_delta": context_provision_without_after + - context_provision_without_before, + "total_capacity_before": context_total_capacity_before, + "total_capacity_after": context_total_capacity_after, + "total_capacity_delta": context_total_capacity_after + - context_total_capacity_before, + "balance_before": context_balance_before, + "balance_after": context_balance_after, + "balance_delta": context_balance_after - context_balance_before, + "deficit_before": context_deficit_before, + "deficit_after": context_deficit_after, + "deficit_delta": context_deficit_after - context_deficit_before, + "surplus_before": context_surplus_before, + "surplus_after": context_surplus_after, + "surplus_delta": context_surplus_after - context_surplus_before, + "demand_before": context_demand_before, + "demand_after": context_demand_after, + "demand_delta": context_demand_after - context_demand_before, + "unmet_demand_before": context_unmet_demand_before, + "unmet_demand_after": context_unmet_demand_after, + "unmet_demand_delta": context_unmet_demand_after + - context_unmet_demand_before, + "unmet_demand_within_before": context_unmet_demand_within_before, + "unmet_demand_within_after": context_unmet_demand_within_after, + "unmet_demand_within_delta": context_unmet_demand_within_after + - context_unmet_demand_within_before, + "unmet_demand_without_before": context_unmet_demand_without_before, + "unmet_demand_without_after": context_unmet_demand_without_after, + "unmet_demand_without_delta": context_unmet_demand_without_after + - context_unmet_demand_without_before, + }, + "project": { + "provision_before": project_provision_before, + "provision_after": project_provision_after, + "provision_delta": project_provision_after - project_provision_before, + "provision_within_before": project_provision_within_before, + "provision_within_after": project_provision_within_after, + "provision_within_delta": project_provision_within_after + - project_provision_within_before, + "provision_without_before": project_provision_without_before, + "provision_without_after": project_provision_without_after, + "provision_without_delta": project_provision_without_after + - project_provision_without_before, + "total_capacity_before": project_total_capacity_before, + "total_capacity_after": project_total_capacity_after, + "total_capacity_delta": project_total_capacity_after + - project_total_capacity_before, + "balance_before": project_balance_before, + "balance_after": project_balance_after, + "balance_delta": project_balance_after - project_balance_before, + "deficit_before": project_deficit_before, + "deficit_after": project_deficit_after, + "deficit_delta": project_deficit_after - project_deficit_before, + "surplus_before": project_surplus_before, + "surplus_after": project_surplus_after, + "surplus_delta": project_surplus_after - project_surplus_before, + "demand_before": project_demand_before, + "demand_after": project_demand_after, + "demand_delta": project_demand_after - project_demand_before, + "unmet_demand_before": project_unmet_demand_before, + "unmet_demand_after": project_unmet_demand_after, + "unmet_demand_delta": project_unmet_demand_after + - project_unmet_demand_before, + "unmet_demand_within_before": project_unmet_demand_within_before, + "unmet_demand_within_after": project_unmet_demand_within_after, + "unmet_demand_within_delta": project_unmet_demand_within_after + - project_unmet_demand_within_before, + "unmet_demand_without_before": project_unmet_demand_without_before, + "unmet_demand_without_after": project_unmet_demand_without_after, + "unmet_demand_without_delta": project_unmet_demand_without_after + - project_unmet_demand_without_before, + }, + } + return json.dumps(result) effects_service = EffectsService() diff --git a/app/effects/shemas/effects_base_schema.py b/app/effects/shemas/effects_base_schema.py index d8b69e2..2029720 100644 --- a/app/effects/shemas/effects_base_schema.py +++ b/app/effects/shemas/effects_base_schema.py @@ -1,40 +1,8 @@ -from typing import Any, Literal, Optional +from typing import Optional from pydantic import BaseModel - -class GeometrySchema(BaseModel): - - type: Literal[ - "Polygon", - "MultiPolygon", - "LineString", - "MultiLineString", - "Point", - "MultiPoint", - ] - coordinates: list[Any] - - -class FeatureSchema(BaseModel): - - id: Optional[int | None] - type: Literal["Feature"] - geometry: GeometrySchema - properties: dict - - -class FeatureCollectionSchema(BaseModel): - - type: Literal["FeatureCollection"] - features: list[FeatureSchema] - - -class ProvisionSchema(BaseModel): - - buildings: FeatureCollectionSchema - services: FeatureCollectionSchema - links: FeatureCollectionSchema +from app.schemas.provision_base_schema import FeatureCollectionSchema, ProvisionSchema class PivotSchema(BaseModel): diff --git a/app/main.py b/app/main.py index 99147af..b96e4cc 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, RedirectResponse +from fastmcp.utilities.lifespan import combine_lifespans from loguru import logger from .__version__ import APP_VERSION @@ -10,8 +11,10 @@ from .common.middlewares.prometheus_handler import ObservabilityMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router +from .mcp import effects_mcp_app from .observability import OpenTelemetryAgent, PrometheusConfig from .observability.metrics import setup_metrics +from .provision.provision_controller import provision_router log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" @@ -43,8 +46,9 @@ async def lifespan(app: FastAPI): title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", version=APP_VERSION, - lifespan=lifespan, + lifespan=combine_lifespans(lifespan, effects_mcp_app.lifespan), ) +app.mount("/effects", effects_mcp_app) # Add CORS middleware app.add_middleware( @@ -97,3 +101,4 @@ async def get_logs(): app.include_router(effects_router) +app.include_router(provision_router) diff --git a/app/mcp.py b/app/mcp.py new file mode 100644 index 0000000..28b2b03 --- /dev/null +++ b/app/mcp.py @@ -0,0 +1,3 @@ +from app.effects.effects_mcp import effects_mcp + +effects_mcp_app = effects_mcp.http_app(path="/mcp") diff --git a/app/provision/__init__.py b/app/provision/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py new file mode 100644 index 0000000..357a008 --- /dev/null +++ b/app/provision/provision_controller.py @@ -0,0 +1,19 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends + +from app.common.auth.bearer import verify_bearer_token +from app.dto.provision_dto import ProvisionDTO +from app.provision.provision_service import provision_service +from app.schemas.provision_base_schema import ProvisionSchema + +provision_router = APIRouter(prefix="/provision", tags=["provision"]) + + +@provision_router.get("/calc_provision", response_model=ProvisionSchema) +async def calculate_provision( + provision_dto: Annotated[ProvisionDTO, Depends(ProvisionDTO)], + token: str = Depends(verify_bearer_token), +) -> ProvisionSchema: + + return await provision_service.calculate_provision(provision_dto, token) diff --git a/app/provision/provision_service.py b/app/provision/provision_service.py new file mode 100644 index 0000000..964421e --- /dev/null +++ b/app/provision/provision_service.py @@ -0,0 +1,166 @@ +import asyncio +import json + +import pandas as pd +from loguru import logger + +from app.common.exceptions.http_exception_wrapper import http_exception +from app.common.modules import ( + attribute_parser, + data_restorator, + matrix_builder, + objectnat_calculator, +) +from app.common.modules.effects_api_gateway import effects_api_gateway +from app.dto.provision_dto import ProvisionDTO +from app.schemas.provision_base_schema import ProvisionSchema + +LIVING_BUILDINGS_ID = 4 + + +class ProvisionService: + + def __init__(self): + pass + + @staticmethod + async def calculate_provision( + provision_params: ProvisionDTO, token: str + ) -> ProvisionSchema: + """ + Calculate provision effects by project data and target scenario + Args: + provision_params (ProvisionDTO): Project data + token (str): Authorization token + Returns: + gpd.GeoDataFrame: Provision for scenario. + """ + + logger.info( + f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" + ) + project_data = await effects_api_gateway.get_project_data( + provision_params.project_id, token + ) + project_territory = await effects_api_gateway.get_project_territory( + provision_params.project_id, token + ) + service_default_capacity = await effects_api_gateway.get_default_capacity( + service_type_id=provision_params.service_type_id + ) + normative_data = await effects_api_gateway.get_service_normative( + territory_id=project_data["territory"]["id"], + context_ids=project_data["properties"]["context"], + service_type_id=provision_params.service_type_id, + token=token, + ) + context_population = await effects_api_gateway.get_context_population( + territory_ids_list=project_data["properties"]["context"], token=token + ) + context_buildings = await effects_api_gateway.get_project_context_buildings( + scenario_id=project_data["base_scenario"]["id"], token=token + ) + context_buildings.drop( + index=context_buildings.sjoin(project_territory).index, inplace=True + ) + context_buildings = await attribute_parser.parse_all_from_buildings( + living_buildings=context_buildings, + ) + context_buildings = await asyncio.to_thread( + data_restorator.restore_demands, + buildings=context_buildings, + service_normative=normative_data["services_capacity_per_1000_normative"], + service_normative_type=normative_data["capacity_type"], + target_population=context_population, + ) + context_buildings["is_project"] = False + context_services = await effects_api_gateway.get_project_context_services( + scenario_id=project_data["base_scenario"]["id"], + service_type_id=provision_params.service_type_id, + token=token, + ) + if context_services.empty: + # ToDo Revise to another code + raise http_exception( + status_code=404, + msg="No services of {service_type_id} type found in context", + _input={"service_type_id": provision_params.service_type_id}, + _detail={}, + ) + context_services = await attribute_parser.parse_all_from_services( + services=context_services, service_default_capacity=service_default_capacity + ) + target_scenario_population = ( + await effects_api_gateway.get_scenario_population_data( + scenario_id=provision_params.scenario_id, token=token + ) + ) + target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + scenario_id=provision_params.scenario_id, token=token + ) + target_scenario_buildings = await attribute_parser.parse_all_from_buildings( + living_buildings=target_scenario_buildings, + ) + target_scenario_buildings = await asyncio.to_thread( + data_restorator.restore_demands, + buildings=target_scenario_buildings, + service_normative=normative_data["services_capacity_per_1000_normative"], + service_normative_type=normative_data["capacity_type"], + target_population=target_scenario_population, + ) + target_scenario_buildings["is_project"] = True + target_scenario_services = await effects_api_gateway.get_scenario_services( + scenario_id=provision_params.scenario_id, + service_type_id=provision_params.service_type_id, + token=token, + ) + target_scenario_services = await attribute_parser.parse_all_from_services( + services=target_scenario_services, + service_default_capacity=service_default_capacity, + ) + before_buildings = await asyncio.to_thread( + pd.concat, + objs=[context_buildings, target_scenario_buildings], + ) + before_services = await asyncio.to_thread( + pd.concat, objs=[context_services, target_scenario_services] + ) + before_buildings.sort_values("is_project", ascending=False, inplace=True) + before_buildings.drop_duplicates("building_id", keep="first", inplace=True) + before_buildings.set_index("building_id", inplace=True) + before_services.set_index("service_id", inplace=True) + before_services.drop_duplicates("geometry", inplace=True) + before_services = before_services[ + ~before_services.index.duplicated(keep="first") + ].copy() + if target_scenario_buildings.empty: + local_crs = context_buildings.estimate_utm_crs() + else: + local_crs = target_scenario_buildings.estimate_utm_crs() + before_buildings.to_crs(local_crs, inplace=True) + before_services.to_crs(local_crs, inplace=True) + before_matrix = await asyncio.to_thread( + matrix_builder.calculate_availability_matrix, + buildings=before_buildings, + services=before_services, + normative_value=normative_data["normative_value"], + normative_type=normative_data["normative_type"], + ) + before_services["capacity"] = before_services["capacity"].fillna( + before_services["capacity"].mean() + ) + before_prove_data = await asyncio.to_thread( + objectnat_calculator.evaluate_provision, + buildings=before_buildings, + services=before_services[~before_services.index.duplicated(keep="first")], + matrix=before_matrix, + service_normative=normative_data["normative_value"], + ) + result = {k: json.loads(v.to_json()) for k, v in before_prove_data.items()} + logger.info( + f"Calculated PROVISION for {provision_params.scenario_id} and {provision_params.service_type_id}" + ) + return ProvisionSchema(**result) + + +provision_service = ProvisionService() diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/schemas/provision_base_schema.py b/app/schemas/provision_base_schema.py new file mode 100644 index 0000000..db2f88c --- /dev/null +++ b/app/schemas/provision_base_schema.py @@ -0,0 +1,37 @@ +from typing import Any, Literal, Optional + +from pydantic import BaseModel + + +class GeometrySchema(BaseModel): + + type: Literal[ + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + "Point", + "MultiPoint", + ] + coordinates: list[Any] + + +class FeatureSchema(BaseModel): + + id: Optional[int | None] + type: Literal["Feature"] + geometry: GeometrySchema + properties: dict + + +class FeatureCollectionSchema(BaseModel): + + type: Literal["FeatureCollection"] + features: list[FeatureSchema] + + +class ProvisionSchema(BaseModel): + + buildings: FeatureCollectionSchema + services: FeatureCollectionSchema + links: FeatureCollectionSchema diff --git a/requirements.txt b/requirements.txt index a7a181ca6c5dcf687a5d0ae765d5d5d736865273..2f5a88685119fbb8a019a0b2554beb3ee2373510 100644 GIT binary patch delta 179 zcmbQo_JDnZ9CJM{0~bRvLkUA7LlHv`Ln@F}0wn7gY#9t0^cW1l7^J3vp^_m5s45RA zpUIF6R$&B`HDoXbl12;=b25M`3V`}^fT}@efK-6YFbB&Q0PVIf~jB!6~hfM1{-6_008hK9h3k7 delta 7 OcmaFBK96mK95Vn5&H`Nk From 70842d4fcaa77fd77922c47849edbf55a8c5fed0 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:14:50 +0300 Subject: [PATCH 40/61] Dev (#30) * feat(dependencies): - updated requirements.txt, requirements-dev.txt and .pre-commit-config.yaml * feat(mcp): - mcp server in progress * feat(provision, mcp): - mcp server added - added provision endpoints --- .pre-commit-config.yaml | 4 +- app/{effects => common}/modules/__init__.py | 0 .../modules/attribute_parser.py | 0 .../modules/data_restorator.py | 6 - .../modules/effects_api_gateway.py | 48 +++ .../modules/matrix_builder.py | 0 .../modules/name_mappings.py | 3 +- .../modules/objectnat_calculator.py | 0 app/{effects => }/dto/__init__.py | 0 .../effects_dto.py => dto/provision_dto.py} | 8 +- app/effects/effects_controller.py | 11 +- app/effects/effects_mcp.py | 62 +++ app/effects/effects_service.py | 397 +++++++++++++++++- app/effects/shemas/effects_base_schema.py | 37 +- app/main.py | 7 +- app/mcp.py | 3 + app/provision/__init__.py | 0 app/provision/provision_controller.py | 19 + app/provision/provision_service.py | 166 ++++++++ app/schemas/__init__.py | 0 app/schemas/provision_base_schema.py | 37 ++ requirements-dev.txt | 2 +- requirements.txt | Bin 840 -> 992 bytes 23 files changed, 746 insertions(+), 64 deletions(-) rename app/{effects => common}/modules/__init__.py (100%) rename app/{effects => common}/modules/attribute_parser.py (100%) rename app/{effects => common}/modules/data_restorator.py (96%) rename app/{effects => common}/modules/effects_api_gateway.py (85%) rename app/{effects => common}/modules/matrix_builder.py (100%) rename app/{effects => common}/modules/name_mappings.py (97%) rename app/{effects => common}/modules/objectnat_calculator.py (100%) rename app/{effects => }/dto/__init__.py (100%) rename app/{effects/dto/effects_dto.py => dto/provision_dto.py} (62%) create mode 100644 app/effects/effects_mcp.py create mode 100644 app/mcp.py create mode 100644 app/provision/__init__.py create mode 100644 app/provision/provision_controller.py create mode 100644 app/provision/provision_service.py create mode 100644 app/schemas/__init__.py create mode 100644 app/schemas/provision_base_schema.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9703de4..54e2bba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/psf/black - rev: 25.1.0 + rev: 26.3.1 hooks: - id: black language_version: python3.11 - repo: https://github.com/pycqa/isort - rev: 6.0.1 + rev: 8.0.1 hooks: - id: isort name: isort (python) diff --git a/app/effects/modules/__init__.py b/app/common/modules/__init__.py similarity index 100% rename from app/effects/modules/__init__.py rename to app/common/modules/__init__.py diff --git a/app/effects/modules/attribute_parser.py b/app/common/modules/attribute_parser.py similarity index 100% rename from app/effects/modules/attribute_parser.py rename to app/common/modules/attribute_parser.py diff --git a/app/effects/modules/data_restorator.py b/app/common/modules/data_restorator.py similarity index 96% rename from app/effects/modules/data_restorator.py rename to app/common/modules/data_restorator.py index 33fb0a1..62a9196 100644 --- a/app/effects/modules/data_restorator.py +++ b/app/common/modules/data_restorator.py @@ -152,11 +152,5 @@ def restore_demands( _detail={"available_demand_type": ["unit", "capacity"]}, ) - def restore_capacity( - services: gpd.GeoDataFrame, service_capacity_normative: int - ) -> gpd.GeoDataFrame: - - services["capacity"] = service_capacity_normative - data_restorator = DataRestorator() diff --git a/app/effects/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py similarity index 85% rename from app/effects/modules/effects_api_gateway.py rename to app/common/modules/effects_api_gateway.py index 447c277..829f4f5 100644 --- a/app/effects/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -325,5 +325,53 @@ async def get_default_capacity(service_type_id: int) -> int: service_types_df["service_type_id"] == service_type_id ].iloc[0]["capacity_modeled"] + @staticmethod + async def get_services_with_context( + scenario_id: int, service_type_id: int, token: str | None = None + ) -> gpd.GeoDataFrame: + """ + Function retrieves service by service_type_id for scenario ID from urban api with context. + Args: + scenario_id (int): Scenario ID from Urban API. + service_type_id (int): Service type ID from Urban API. + token (str | None): Auth token to retrieve data from Urban API. Default to None + Returns: + gpd.GeoDataFrame: layer with services in 4326 crs. + """ + + services = await urban_api_handler.get( + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", + params={ + "service_type_id": service_type_id, + "include_scenario_objects": True, + }, + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return gpd.GeoDataFrame.from_features(services, crs=4326) + + @staticmethod + async def get_physical_objects_with_context( + scenario_id: int, physical_object_type_id: int, token: str | None = None + ): + """ + Function retrieves physical objects by physical_object_type_id for scenario ID from urban api with context. + Args: + scenario_id (int): Scenario ID from Urban API. + physical_object_type_id (int): Physical object type ID from Urban API. + token (str | None): Auth token to retrieve data from Urban API. Default to None + Returns: + gpd.GeoDataFrame: layer with physical_objects in 4326 crs. + """ + + physical_objects = await urban_api_handler.get( + endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", + params={ + "physical_object_type_id": physical_object_type_id, + "include_scenario_objects": True, + }, + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return gpd.GeoDataFrame.from_features(physical_objects, crs=4326) + effects_api_gateway = EffectsAPIGateway() diff --git a/app/effects/modules/matrix_builder.py b/app/common/modules/matrix_builder.py similarity index 100% rename from app/effects/modules/matrix_builder.py rename to app/common/modules/matrix_builder.py diff --git a/app/effects/modules/name_mappings.py b/app/common/modules/name_mappings.py similarity index 97% rename from app/effects/modules/name_mappings.py rename to app/common/modules/name_mappings.py index 15632ec..1a185e4 100644 --- a/app/effects/modules/name_mappings.py +++ b/app/common/modules/name_mappings.py @@ -25,6 +25,7 @@ "us_demands_within_after": "Неудовлетворённый спрос в нормативной доступности (после) (чел)", "supplyed_demands_without_after": "Удовлетворённый спрос вне нормативной доступности (после) (чел)", "us_demands_without_after": "Неудовлетворённый спрос вне нормативной доступности (после) (чел)", + "is_scenario_object": "Сценарный объект", } EFFECTS_MAP = { @@ -37,5 +38,5 @@ "is_project": "Проектный объект", } -SERVICE_DROP_COLUMNS = ["is_scenario_object", "is_locked"] +SERVICE_DROP_COLUMNS = ["is_locked"] BUILDINGS_DROP_COLUMNS = SERVICE_DROP_COLUMNS + ["is_project"] diff --git a/app/effects/modules/objectnat_calculator.py b/app/common/modules/objectnat_calculator.py similarity index 100% rename from app/effects/modules/objectnat_calculator.py rename to app/common/modules/objectnat_calculator.py diff --git a/app/effects/dto/__init__.py b/app/dto/__init__.py similarity index 100% rename from app/effects/dto/__init__.py rename to app/dto/__init__.py diff --git a/app/effects/dto/effects_dto.py b/app/dto/provision_dto.py similarity index 62% rename from app/effects/dto/effects_dto.py rename to app/dto/provision_dto.py index 93dbffe..5fc9375 100644 --- a/app/effects/dto/effects_dto.py +++ b/app/dto/provision_dto.py @@ -1,14 +1,12 @@ -from typing import Optional - from pydantic import BaseModel, Field -class EffectsDTO(BaseModel): +class ProvisionDTO(BaseModel): project_id: int = Field(..., examples=[72], description="Project ID") scenario_id: int = Field(..., examples=[192], description="Scenario ID") - service_type_id: int = Field(..., examples=[7], description="Service type ID") - target_population: Optional[int] = Field( + service_type_id: int = Field(..., examples=[22], description="Service type ID") + target_population: int | None = Field( default=None, examples=[200], description="Target population for project territory", diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index 507422d..d4e92e9 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dto.provision_dto import ProvisionDTO -from .dto.effects_dto import EffectsDTO from .effects_service import effects_service from .shemas.effects_base_schema import EffectsSchema @@ -13,15 +13,8 @@ @effects_router.get("/evaluate_provision", response_model=EffectsSchema) async def calculate_effects( - params: Annotated[EffectsDTO, Depends(EffectsDTO)], + params: Annotated[ProvisionDTO, Depends(ProvisionDTO)], token: str = Depends(verify_bearer_token), ) -> EffectsSchema: - """ - Get method for retrieving effects with objectnat - Params: - - project ID: Project ID - scenario ID: Scenario ID - """ return await effects_service.calculate_effects(params, token) diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py new file mode 100644 index 0000000..0676cf4 --- /dev/null +++ b/app/effects/effects_mcp.py @@ -0,0 +1,62 @@ +from fastmcp import FastMCP +from fastmcp.server.dependencies import CurrentContext, get_access_token + +from app.dto.provision_dto import ProvisionDTO + +from .effects_service import effects_service + +effects_mcp = FastMCP("Object Effects MCP server") + + +@effects_mcp.tool( + name="CalculateObjectEffects", + title="Get provision effects for service", + description=""" + Retrieve service provision effects by service id. + If total population is provided, demand is restored from it. Otherwise, population is restored from living square. + + Args to select: + + Returns effects layers with estimated pivot info for llm analyses. + Response format: + { + "before_prove_data": { + "buildings": FeatureCollection, + "services": FeatureCollection, + "": FeatureCollection + }, + "after_prove_data": { + "buildings": FeatureCollection, + "services": FeatureCollection, + "links": FeatureCollection + }, + "effects": FeatureCollection, + "pivot": { + "sum_absolute_total": int, + "average_absolute_total": float, + "median_absolute_total": int, + "average_index_total": float, + "median_index_total": int, + "sum_absolute_within": int, + "average_absolute_within": float, + "median_absolute_within": int, + }, + "": str + } + """, +) +async def calc_provision_effects( + service_type_id: int, target_population: int | None = None, ctx=CurrentContext() +): + + project_id = int(ctx.request_context.meta.project_id) + scenario_id = int(ctx.request_context.meta.scenario_id) + token = get_access_token() + effects_dto = ProvisionDTO( + project_id=project_id, + scenario_id=scenario_id, + service_type_id=service_type_id, + target_population=target_population, + ) + result = await effects_service.calculate_effects(effects_dto, token, for_mcp=True) + return result diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 1f961d0..8d1c67e 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -6,9 +6,7 @@ from loguru import logger from app.common.exceptions.http_exception_wrapper import http_exception - -from .dto.effects_dto import EffectsDTO -from .modules import ( +from app.common.modules import ( ATTRIBUTES_MAP, BUILDINGS_DROP_COLUMNS, EFFECTS_MAP, @@ -19,6 +17,8 @@ matrix_builder, objectnat_calculator, ) +from app.dto.provision_dto import ProvisionDTO + from .shemas.effects_base_schema import EffectsSchema @@ -82,13 +82,14 @@ async def _get_pivot( # ToDo Split function # ToDo Rewrite to context ids normal handling async def calculate_effects( - self, effects_params: EffectsDTO, token: str + self, effects_params: ProvisionDTO, token: str, for_mcp: bool = False ) -> EffectsSchema: """ Calculate provision effects by project data and target scenario Args: - effects_params (EffectsDTO): Project data + effects_params (ProvisionDTO): Project data token (str): Authorization token + for_mcp (bool): If flag enabled adds string description for llm. Default to false. Returns: gpd.GeoDataFrame: Provision effects """ @@ -362,7 +363,393 @@ async def calculate_effects( ), "pivot": pivot, } + if for_mcp: + result["text_pivot"] = await self.form_llm_context( + before_prove_data["buildings"], + after_prove_data["buildings"], + before_prove_data["services"], + after_prove_data["services"], + ) return EffectsSchema(**result) + @staticmethod + async def form_llm_context( + before_buildings: gpd.GeoDataFrame, + after_buildings: gpd.GeoDataFrame, + before_services: gpd.GeoDataFrame, + after_services: gpd.GeoDataFrame, + ) -> str: + """ + Function forms text repr stats from calculated provision data for llm. + Args: + before_buildings (gpd.GeoDataFrame): Buildings provision layers before. + after_buildings (gpd.GeoDataFrame): Buildings provision layers after. + before_services (gpd.GeoDataFrame): Services provision layers before. + after_services (gpd.GeoDataFrame): Services provision layers after. + Returns: + str: Text representation for formed stats in json string. + """ + + before_buildings_all = before_buildings.copy() + after_buildings_all = after_buildings.copy() + before_services_all = before_services.copy() + after_services_all = after_services.copy() + before_buildings_context = before_buildings[ + before_buildings["is_scenario_object"] == False + ] + after_buildings_context = after_buildings[ + after_buildings["is_scenario_object"] == False + ] + before_services_context = before_services[ + before_services["is_scenario_object"] == False + ] + after_services_context = after_services[ + after_services["is_scenario_object"] == False + ] + before_buildings_project = before_buildings[ + before_buildings["is_scenario_object"] == True + ] + after_buildings_project = after_buildings[ + after_buildings["is_scenario_object"] == True + ] + before_services_project = before_services[ + before_services["is_scenario_object"] == True + ] + after_services_project = after_services[ + after_services["is_scenario_object"] == True + ] + all_provision_before = int( + before_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_after = int( + after_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_provision_within_before = int( + before_buildings_all[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_within_after = int( + after_buildings_all[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + all_provision_without_before = int( + before_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_provision_without_after = int( + after_buildings_all[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_total_capacity_before = int(before_services_all["Вместимость (чел)"].sum()) + all_total_capacity_after = int(after_services_all["Вместимость (чел)"].sum()) + all_demand_before = int(before_buildings_all["Спрос (чел)"].sum()) + all_demand_after = int(after_buildings_all["Спрос (чел)"].sum()) + all_unmet_demand_before = int( + before_buildings_all["Неудовлетворённый спрос (чел)"].sum() + ) + all_unmet_demand_after = int( + after_buildings_all["Неудовлетворённый спрос (чел)"].sum() + ) + all_unmet_demand_within_before = int( + before_buildings_all[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + all_unmet_demand_within_after = int( + after_buildings_all[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + all_unmet_demand_without_before = int( + before_buildings_all[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + all_unmet_demand_without_after = int( + after_buildings_all[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + all_balance_before = all_total_capacity_before - all_demand_before + all_balance_after = all_total_capacity_after - all_demand_after + all_deficit_before = min(0, all_balance_before) + all_deficit_after = min(0, all_balance_after) + all_surplus_before = max(0, all_balance_before) + all_surplus_after = max(0, all_balance_after) + context_provision_before = int( + before_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_after = int( + after_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_provision_within_before = int( + before_buildings_context[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_within_after = int( + after_buildings_context[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + context_provision_without_before = int( + before_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_provision_without_after = int( + after_buildings_context[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_total_capacity_before = int( + before_services_context["Вместимость (чел)"].sum() + ) + context_total_capacity_after = int( + after_services_context["Вместимость (чел)"].sum() + ) + context_demand_before = int(before_buildings_context["Спрос (чел)"].sum()) + context_demand_after = int(after_buildings_context["Спрос (чел)"].sum()) + context_unmet_demand_before = int( + before_buildings_context["Неудовлетворённый спрос (чел)"].sum() + ) + context_unmet_demand_after = int( + after_buildings_context["Неудовлетворённый спрос (чел)"].sum() + ) + context_unmet_demand_within_before = int( + before_buildings_context[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + context_unmet_demand_within_after = int( + after_buildings_context[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + context_unmet_demand_without_before = int( + before_buildings_context[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + context_unmet_demand_without_after = int( + after_buildings_context[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + context_balance_before = context_total_capacity_before - context_demand_before + context_balance_after = context_total_capacity_after - context_demand_after + context_deficit_before = min(0, context_balance_before) + context_deficit_after = min(0, context_balance_after) + context_surplus_before = max(0, context_balance_before) + context_surplus_after = max(0, context_balance_after) + project_provision_before = int( + before_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_after = int( + after_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_provision_within_before = int( + before_buildings_project[ + "Удовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_within_after = int( + after_buildings_project[ + "Удовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + project_provision_without_before = int( + before_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_provision_without_after = int( + after_buildings_project[ + "Удовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_total_capacity_before = int( + before_services_project["Вместимость (чел)"].sum() + ) + project_total_capacity_after = int( + after_services_project["Вместимость (чел)"].sum() + ) + project_demand_before = int(before_buildings_project["Спрос (чел)"].sum()) + project_demand_after = int(after_buildings_project["Спрос (чел)"].sum()) + project_unmet_demand_before = int( + before_buildings_project["Неудовлетворённый спрос (чел)"].sum() + ) + project_unmet_demand_after = int( + after_buildings_project["Неудовлетворённый спрос (чел)"].sum() + ) + project_unmet_demand_within_before = int( + before_buildings_project[ + "Неудовлетворённый спрос в нормативной доступности (до) (чел)" + ].sum() + ) + project_unmet_demand_within_after = int( + after_buildings_project[ + "Неудовлетворённый спрос в нормативной доступности (после) (чел)" + ].sum() + ) + project_unmet_demand_without_before = int( + before_buildings_project[ + "Неудовлетворённый спрос вне нормативной доступности (до) (чел)" + ].sum() + ) + project_unmet_demand_without_after = int( + after_buildings_project[ + "Неудовлетворённый спрос вне нормативной доступности (после) (чел)" + ].sum() + ) + project_balance_before = project_total_capacity_before - project_demand_before + project_balance_after = project_total_capacity_after - project_demand_after + project_deficit_before = min(0, project_balance_before) + project_deficit_after = min(0, project_balance_after) + project_surplus_before = max(0, project_balance_before) + project_surplus_after = max(0, project_balance_after) + + result = { + "all": { + "provision_before": all_provision_before, + "provision_after": all_provision_after, + "provision_delta": all_provision_after - all_provision_before, + "provision_within_before": all_provision_within_before, + "provision_within_after": all_provision_within_after, + "provision_within_delta": all_provision_within_after + - all_provision_within_before, + "provision_without_before": all_provision_without_before, + "provision_without_after": all_provision_without_after, + "provision_without_delta": all_provision_without_after + - all_provision_without_before, + "total_capacity_before": all_total_capacity_before, + "total_capacity_after": all_total_capacity_after, + "total_capacity_delta": all_total_capacity_after + - all_total_capacity_before, + "balance_before": all_balance_before, + "balance_after": all_balance_after, + "balance_delta": all_balance_after - all_balance_before, + "deficit_before": all_deficit_before, + "deficit_after": all_deficit_after, + "deficit_delta": all_deficit_after - all_deficit_before, + "surplus_before": all_surplus_before, + "surplus_after": all_surplus_after, + "surplus_delta": all_surplus_after - all_surplus_before, + "demand_before": all_demand_before, + "demand_after": all_demand_after, + "demand_delta": all_demand_after - all_demand_before, + "unmet_demand_before": all_unmet_demand_before, + "unmet_demand_after": all_unmet_demand_after, + "unmet_demand_delta": all_unmet_demand_after - all_unmet_demand_before, + "unmet_demand_within_before": all_unmet_demand_within_before, + "unmet_demand_within_after": all_unmet_demand_within_after, + "unmet_demand_within_delta": all_unmet_demand_within_after + - all_unmet_demand_within_before, + "unmet_demand_without_before": all_unmet_demand_without_before, + "unmet_demand_without_after": all_unmet_demand_without_after, + "unmet_demand_without_delta": all_unmet_demand_without_after + - all_unmet_demand_without_before, + }, + "context": { + "provision_before": context_provision_before, + "provision_after": context_provision_after, + "provision_delta": context_provision_after - context_provision_before, + "provision_within_before": context_provision_within_before, + "provision_within_after": context_provision_within_after, + "provision_within_delta": context_provision_within_after + - context_provision_within_before, + "provision_without_before": context_provision_without_before, + "provision_without_after": context_provision_without_after, + "provision_without_delta": context_provision_without_after + - context_provision_without_before, + "total_capacity_before": context_total_capacity_before, + "total_capacity_after": context_total_capacity_after, + "total_capacity_delta": context_total_capacity_after + - context_total_capacity_before, + "balance_before": context_balance_before, + "balance_after": context_balance_after, + "balance_delta": context_balance_after - context_balance_before, + "deficit_before": context_deficit_before, + "deficit_after": context_deficit_after, + "deficit_delta": context_deficit_after - context_deficit_before, + "surplus_before": context_surplus_before, + "surplus_after": context_surplus_after, + "surplus_delta": context_surplus_after - context_surplus_before, + "demand_before": context_demand_before, + "demand_after": context_demand_after, + "demand_delta": context_demand_after - context_demand_before, + "unmet_demand_before": context_unmet_demand_before, + "unmet_demand_after": context_unmet_demand_after, + "unmet_demand_delta": context_unmet_demand_after + - context_unmet_demand_before, + "unmet_demand_within_before": context_unmet_demand_within_before, + "unmet_demand_within_after": context_unmet_demand_within_after, + "unmet_demand_within_delta": context_unmet_demand_within_after + - context_unmet_demand_within_before, + "unmet_demand_without_before": context_unmet_demand_without_before, + "unmet_demand_without_after": context_unmet_demand_without_after, + "unmet_demand_without_delta": context_unmet_demand_without_after + - context_unmet_demand_without_before, + }, + "project": { + "provision_before": project_provision_before, + "provision_after": project_provision_after, + "provision_delta": project_provision_after - project_provision_before, + "provision_within_before": project_provision_within_before, + "provision_within_after": project_provision_within_after, + "provision_within_delta": project_provision_within_after + - project_provision_within_before, + "provision_without_before": project_provision_without_before, + "provision_without_after": project_provision_without_after, + "provision_without_delta": project_provision_without_after + - project_provision_without_before, + "total_capacity_before": project_total_capacity_before, + "total_capacity_after": project_total_capacity_after, + "total_capacity_delta": project_total_capacity_after + - project_total_capacity_before, + "balance_before": project_balance_before, + "balance_after": project_balance_after, + "balance_delta": project_balance_after - project_balance_before, + "deficit_before": project_deficit_before, + "deficit_after": project_deficit_after, + "deficit_delta": project_deficit_after - project_deficit_before, + "surplus_before": project_surplus_before, + "surplus_after": project_surplus_after, + "surplus_delta": project_surplus_after - project_surplus_before, + "demand_before": project_demand_before, + "demand_after": project_demand_after, + "demand_delta": project_demand_after - project_demand_before, + "unmet_demand_before": project_unmet_demand_before, + "unmet_demand_after": project_unmet_demand_after, + "unmet_demand_delta": project_unmet_demand_after + - project_unmet_demand_before, + "unmet_demand_within_before": project_unmet_demand_within_before, + "unmet_demand_within_after": project_unmet_demand_within_after, + "unmet_demand_within_delta": project_unmet_demand_within_after + - project_unmet_demand_within_before, + "unmet_demand_without_before": project_unmet_demand_without_before, + "unmet_demand_without_after": project_unmet_demand_without_after, + "unmet_demand_without_delta": project_unmet_demand_without_after + - project_unmet_demand_without_before, + }, + } + return json.dumps(result) + effects_service = EffectsService() diff --git a/app/effects/shemas/effects_base_schema.py b/app/effects/shemas/effects_base_schema.py index 14ddcc8..2029720 100644 --- a/app/effects/shemas/effects_base_schema.py +++ b/app/effects/shemas/effects_base_schema.py @@ -1,40 +1,8 @@ -from typing import Any, Literal, Optional +from typing import Optional from pydantic import BaseModel - -class GeometrySchema(BaseModel): - - type: Literal[ - "Polygon", - "MultiPolygon", - "LineString", - "MultiLineString", - "Point", - "MultiPoint", - ] - coordinates: list[Any] - - -class FeatureSchema(BaseModel): - - id: Optional[int | None] - type: Literal["Feature"] - geometry: GeometrySchema - properties: dict - - -class FeatureCollectionSchema(BaseModel): - - type: Literal["FeatureCollection"] - features: list[FeatureSchema] - - -class ProvisionSchema(BaseModel): - - buildings: FeatureCollectionSchema - services: FeatureCollectionSchema - links: FeatureCollectionSchema +from app.schemas.provision_base_schema import FeatureCollectionSchema, ProvisionSchema class PivotSchema(BaseModel): @@ -60,3 +28,4 @@ class EffectsSchema(BaseModel): after_prove_data: ProvisionSchema effects: FeatureCollectionSchema pivot: PivotSchema + text_pivot: str | None = None diff --git a/app/main.py b/app/main.py index 99147af..b96e4cc 100644 --- a/app/main.py +++ b/app/main.py @@ -3,6 +3,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, RedirectResponse +from fastmcp.utilities.lifespan import combine_lifespans from loguru import logger from .__version__ import APP_VERSION @@ -10,8 +11,10 @@ from .common.middlewares.prometheus_handler import ObservabilityMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router +from .mcp import effects_mcp_app from .observability import OpenTelemetryAgent, PrometheusConfig from .observability.metrics import setup_metrics +from .provision.provision_controller import provision_router log_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}" @@ -43,8 +46,9 @@ async def lifespan(app: FastAPI): title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", version=APP_VERSION, - lifespan=lifespan, + lifespan=combine_lifespans(lifespan, effects_mcp_app.lifespan), ) +app.mount("/effects", effects_mcp_app) # Add CORS middleware app.add_middleware( @@ -97,3 +101,4 @@ async def get_logs(): app.include_router(effects_router) +app.include_router(provision_router) diff --git a/app/mcp.py b/app/mcp.py new file mode 100644 index 0000000..28b2b03 --- /dev/null +++ b/app/mcp.py @@ -0,0 +1,3 @@ +from app.effects.effects_mcp import effects_mcp + +effects_mcp_app = effects_mcp.http_app(path="/mcp") diff --git a/app/provision/__init__.py b/app/provision/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py new file mode 100644 index 0000000..357a008 --- /dev/null +++ b/app/provision/provision_controller.py @@ -0,0 +1,19 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends + +from app.common.auth.bearer import verify_bearer_token +from app.dto.provision_dto import ProvisionDTO +from app.provision.provision_service import provision_service +from app.schemas.provision_base_schema import ProvisionSchema + +provision_router = APIRouter(prefix="/provision", tags=["provision"]) + + +@provision_router.get("/calc_provision", response_model=ProvisionSchema) +async def calculate_provision( + provision_dto: Annotated[ProvisionDTO, Depends(ProvisionDTO)], + token: str = Depends(verify_bearer_token), +) -> ProvisionSchema: + + return await provision_service.calculate_provision(provision_dto, token) diff --git a/app/provision/provision_service.py b/app/provision/provision_service.py new file mode 100644 index 0000000..964421e --- /dev/null +++ b/app/provision/provision_service.py @@ -0,0 +1,166 @@ +import asyncio +import json + +import pandas as pd +from loguru import logger + +from app.common.exceptions.http_exception_wrapper import http_exception +from app.common.modules import ( + attribute_parser, + data_restorator, + matrix_builder, + objectnat_calculator, +) +from app.common.modules.effects_api_gateway import effects_api_gateway +from app.dto.provision_dto import ProvisionDTO +from app.schemas.provision_base_schema import ProvisionSchema + +LIVING_BUILDINGS_ID = 4 + + +class ProvisionService: + + def __init__(self): + pass + + @staticmethod + async def calculate_provision( + provision_params: ProvisionDTO, token: str + ) -> ProvisionSchema: + """ + Calculate provision effects by project data and target scenario + Args: + provision_params (ProvisionDTO): Project data + token (str): Authorization token + Returns: + gpd.GeoDataFrame: Provision for scenario. + """ + + logger.info( + f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" + ) + project_data = await effects_api_gateway.get_project_data( + provision_params.project_id, token + ) + project_territory = await effects_api_gateway.get_project_territory( + provision_params.project_id, token + ) + service_default_capacity = await effects_api_gateway.get_default_capacity( + service_type_id=provision_params.service_type_id + ) + normative_data = await effects_api_gateway.get_service_normative( + territory_id=project_data["territory"]["id"], + context_ids=project_data["properties"]["context"], + service_type_id=provision_params.service_type_id, + token=token, + ) + context_population = await effects_api_gateway.get_context_population( + territory_ids_list=project_data["properties"]["context"], token=token + ) + context_buildings = await effects_api_gateway.get_project_context_buildings( + scenario_id=project_data["base_scenario"]["id"], token=token + ) + context_buildings.drop( + index=context_buildings.sjoin(project_territory).index, inplace=True + ) + context_buildings = await attribute_parser.parse_all_from_buildings( + living_buildings=context_buildings, + ) + context_buildings = await asyncio.to_thread( + data_restorator.restore_demands, + buildings=context_buildings, + service_normative=normative_data["services_capacity_per_1000_normative"], + service_normative_type=normative_data["capacity_type"], + target_population=context_population, + ) + context_buildings["is_project"] = False + context_services = await effects_api_gateway.get_project_context_services( + scenario_id=project_data["base_scenario"]["id"], + service_type_id=provision_params.service_type_id, + token=token, + ) + if context_services.empty: + # ToDo Revise to another code + raise http_exception( + status_code=404, + msg="No services of {service_type_id} type found in context", + _input={"service_type_id": provision_params.service_type_id}, + _detail={}, + ) + context_services = await attribute_parser.parse_all_from_services( + services=context_services, service_default_capacity=service_default_capacity + ) + target_scenario_population = ( + await effects_api_gateway.get_scenario_population_data( + scenario_id=provision_params.scenario_id, token=token + ) + ) + target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + scenario_id=provision_params.scenario_id, token=token + ) + target_scenario_buildings = await attribute_parser.parse_all_from_buildings( + living_buildings=target_scenario_buildings, + ) + target_scenario_buildings = await asyncio.to_thread( + data_restorator.restore_demands, + buildings=target_scenario_buildings, + service_normative=normative_data["services_capacity_per_1000_normative"], + service_normative_type=normative_data["capacity_type"], + target_population=target_scenario_population, + ) + target_scenario_buildings["is_project"] = True + target_scenario_services = await effects_api_gateway.get_scenario_services( + scenario_id=provision_params.scenario_id, + service_type_id=provision_params.service_type_id, + token=token, + ) + target_scenario_services = await attribute_parser.parse_all_from_services( + services=target_scenario_services, + service_default_capacity=service_default_capacity, + ) + before_buildings = await asyncio.to_thread( + pd.concat, + objs=[context_buildings, target_scenario_buildings], + ) + before_services = await asyncio.to_thread( + pd.concat, objs=[context_services, target_scenario_services] + ) + before_buildings.sort_values("is_project", ascending=False, inplace=True) + before_buildings.drop_duplicates("building_id", keep="first", inplace=True) + before_buildings.set_index("building_id", inplace=True) + before_services.set_index("service_id", inplace=True) + before_services.drop_duplicates("geometry", inplace=True) + before_services = before_services[ + ~before_services.index.duplicated(keep="first") + ].copy() + if target_scenario_buildings.empty: + local_crs = context_buildings.estimate_utm_crs() + else: + local_crs = target_scenario_buildings.estimate_utm_crs() + before_buildings.to_crs(local_crs, inplace=True) + before_services.to_crs(local_crs, inplace=True) + before_matrix = await asyncio.to_thread( + matrix_builder.calculate_availability_matrix, + buildings=before_buildings, + services=before_services, + normative_value=normative_data["normative_value"], + normative_type=normative_data["normative_type"], + ) + before_services["capacity"] = before_services["capacity"].fillna( + before_services["capacity"].mean() + ) + before_prove_data = await asyncio.to_thread( + objectnat_calculator.evaluate_provision, + buildings=before_buildings, + services=before_services[~before_services.index.duplicated(keep="first")], + matrix=before_matrix, + service_normative=normative_data["normative_value"], + ) + result = {k: json.loads(v.to_json()) for k, v in before_prove_data.items()} + logger.info( + f"Calculated PROVISION for {provision_params.scenario_id} and {provision_params.service_type_id}" + ) + return ProvisionSchema(**result) + + +provision_service = ProvisionService() diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/schemas/provision_base_schema.py b/app/schemas/provision_base_schema.py new file mode 100644 index 0000000..db2f88c --- /dev/null +++ b/app/schemas/provision_base_schema.py @@ -0,0 +1,37 @@ +from typing import Any, Literal, Optional + +from pydantic import BaseModel + + +class GeometrySchema(BaseModel): + + type: Literal[ + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + "Point", + "MultiPoint", + ] + coordinates: list[Any] + + +class FeatureSchema(BaseModel): + + id: Optional[int | None] + type: Literal["Feature"] + geometry: GeometrySchema + properties: dict + + +class FeatureCollectionSchema(BaseModel): + + type: Literal["FeatureCollection"] + features: list[FeatureSchema] + + +class ProvisionSchema(BaseModel): + + buildings: FeatureCollectionSchema + services: FeatureCollectionSchema + links: FeatureCollectionSchema diff --git a/requirements-dev.txt b/requirements-dev.txt index ec9f058..3351d0b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -pre-commit~=4.3.0 \ No newline at end of file +pre-commit~=4.5.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cd740c4706d7e2ed1c9496885ea2ee25740a8184..2f5a88685119fbb8a019a0b2554beb3ee2373510 100644 GIT binary patch delta 186 zcmX@X_JDnY6ss|V9)rn5MOk%Y1~VYjfPt5Riy@66k)fEOgdvw9nW2E8j=`1zq|6Ac z)b;J;Fh)7?Qid{yOrUf=LlHwBScw5hdh>e5nT*OHGeG(ifs#24sX$fSiOG}!0Q<-y2mk;8 delta 59 zcmaFBeu8a+6uTjV9)lr+(L{M!Ak!4cv6z_rcCs6@6srM{V>a2FQF-zKW;sUV$(fAO MjK-T=8D}yA0GwJ50ssI2 From 669c845bf1acb77a23006cd6479e2f2aa8714f96 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Mon, 18 May 2026 03:06:48 +0300 Subject: [PATCH 41/61] fix(effects_service): - fixed llm context creation --- app/__version__.py | 2 +- app/effects/effects_mcp.py | 34 ++++++++---- app/effects/effects_service.py | 99 +++++++++++++++++++++++++++++----- docker-compose.yml | 12 ++++- 4 files changed, 121 insertions(+), 26 deletions(-) diff --git a/app/__version__.py b/app/__version__.py index b87a9e6..87f0a6a 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.1" +APP_VERSION = "0.1.2" diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index 0676cf4..e2fcfbd 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -1,5 +1,8 @@ +import traceback + from fastmcp import FastMCP from fastmcp.server.dependencies import CurrentContext, get_access_token +from loguru import logger from app.dto.provision_dto import ProvisionDTO @@ -49,14 +52,23 @@ async def calc_provision_effects( service_type_id: int, target_population: int | None = None, ctx=CurrentContext() ): - project_id = int(ctx.request_context.meta.project_id) - scenario_id = int(ctx.request_context.meta.scenario_id) - token = get_access_token() - effects_dto = ProvisionDTO( - project_id=project_id, - scenario_id=scenario_id, - service_type_id=service_type_id, - target_population=target_population, - ) - result = await effects_service.calculate_effects(effects_dto, token, for_mcp=True) - return result + try: + project_id = int(ctx.request_context.meta.project_id) + scenario_id = int(ctx.request_context.meta.scenario_id) + token = get_access_token() + effects_dto = ProvisionDTO( + project_id=project_id, + scenario_id=scenario_id, + service_type_id=service_type_id, + target_population=target_population, + ) + result = await effects_service.calculate_effects( + effects_dto, token, for_mcp=True + ) + return result + except Exception as e: + tb = traceback.format_exc() + logger.opt(exception=True).error( + f"Error in MCP tool 'CalculateObjectEffects': {type(e).__name__}: {e}" + ) + raise Exception(f"{type(e).__name__}: {e}\n\nTraceback:\n{tb}") from e diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 8d1c67e..c020ec0 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -390,34 +390,109 @@ async def form_llm_context( str: Text representation for formed stats in json string. """ - before_buildings_all = before_buildings.copy() - after_buildings_all = after_buildings.copy() - before_services_all = before_services.copy() - after_services_all = after_services.copy() + before_buildings_all = before_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) + after_buildings_all = after_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) + before_services_all = before_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) + after_services_all = after_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) + before_buildings_context = before_buildings[ before_buildings["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) after_buildings_context = after_buildings[ after_buildings["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) before_services_context = before_services[ before_services["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) after_services_context = after_services[ after_services["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) before_buildings_project = before_buildings[ before_buildings["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) after_buildings_project = after_buildings[ after_buildings["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) before_services_project = before_services[ before_services["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) after_services_project = after_services[ after_services["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) + + before_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + }, + inplace=True, + ) + after_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + }, + inplace=True, + ) + before_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + }, + inplace=True, + ) + after_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + }, + inplace=True, + ) + all_provision_before = int( before_buildings_all[ "Удовлетворённый спрос вне нормативной доступности (до) (чел)" diff --git a/docker-compose.yml b/docker-compose.yml index 3ac7f79..0740fb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,14 @@ services: build: context: . dockerfile: ./Dockerfile + env_file: + - .env.example ports: - - "80:80" - - "9464:9464" \ No newline at end of file + - "8080:80" + - "9464:9464" + networks: + - localnet + +networks: + localnet: + external: true From 5adbf60cffab97220105abe0fc426ef0dd3ca62d Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Mon, 18 May 2026 03:15:11 +0300 Subject: [PATCH 42/61] Dev (#32) * feat(dependencies): - updated requirements.txt, requirements-dev.txt and .pre-commit-config.yaml * feat(mcp): - mcp server in progress * feat(provision, mcp): - mcp server added - added provision endpoints * fix(effects_service): - fixed llm context creation --- app/__version__.py | 2 +- app/effects/effects_mcp.py | 34 ++++++++---- app/effects/effects_service.py | 99 +++++++++++++++++++++++++++++----- docker-compose.yml | 12 ++++- 4 files changed, 121 insertions(+), 26 deletions(-) diff --git a/app/__version__.py b/app/__version__.py index b87a9e6..87f0a6a 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.1" +APP_VERSION = "0.1.2" diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index 0676cf4..e2fcfbd 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -1,5 +1,8 @@ +import traceback + from fastmcp import FastMCP from fastmcp.server.dependencies import CurrentContext, get_access_token +from loguru import logger from app.dto.provision_dto import ProvisionDTO @@ -49,14 +52,23 @@ async def calc_provision_effects( service_type_id: int, target_population: int | None = None, ctx=CurrentContext() ): - project_id = int(ctx.request_context.meta.project_id) - scenario_id = int(ctx.request_context.meta.scenario_id) - token = get_access_token() - effects_dto = ProvisionDTO( - project_id=project_id, - scenario_id=scenario_id, - service_type_id=service_type_id, - target_population=target_population, - ) - result = await effects_service.calculate_effects(effects_dto, token, for_mcp=True) - return result + try: + project_id = int(ctx.request_context.meta.project_id) + scenario_id = int(ctx.request_context.meta.scenario_id) + token = get_access_token() + effects_dto = ProvisionDTO( + project_id=project_id, + scenario_id=scenario_id, + service_type_id=service_type_id, + target_population=target_population, + ) + result = await effects_service.calculate_effects( + effects_dto, token, for_mcp=True + ) + return result + except Exception as e: + tb = traceback.format_exc() + logger.opt(exception=True).error( + f"Error in MCP tool 'CalculateObjectEffects': {type(e).__name__}: {e}" + ) + raise Exception(f"{type(e).__name__}: {e}\n\nTraceback:\n{tb}") from e diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 8d1c67e..c020ec0 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -390,34 +390,109 @@ async def form_llm_context( str: Text representation for formed stats in json string. """ - before_buildings_all = before_buildings.copy() - after_buildings_all = after_buildings.copy() - before_services_all = before_services.copy() - after_services_all = after_services.copy() + before_buildings_all = before_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) + after_buildings_all = after_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) + before_services_all = before_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) + after_services_all = after_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) + before_buildings_context = before_buildings[ before_buildings["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) after_buildings_context = after_buildings[ after_buildings["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) before_services_context = before_services[ before_services["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) after_services_context = after_services[ after_services["is_scenario_object"] == False - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) before_buildings_project = before_buildings[ before_buildings["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + } + ) after_buildings_project = after_buildings[ after_buildings["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + } + ) before_services_project = before_services[ before_services["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + } + ) after_services_project = after_services[ after_services["is_scenario_object"] == True - ] + ].rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + } + ) + + before_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_buildings.columns + }, + inplace=True, + ) + after_buildings.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_buildings.columns + }, + inplace=True, + ) + before_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in before_services.columns + }, + inplace=True, + ) + after_services.rename( + columns={ + k: v for k, v in ATTRIBUTES_MAP.items() if k in after_services.columns + }, + inplace=True, + ) + all_provision_before = int( before_buildings_all[ "Удовлетворённый спрос вне нормативной доступности (до) (чел)" diff --git a/docker-compose.yml b/docker-compose.yml index 3ac7f79..0740fb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,14 @@ services: build: context: . dockerfile: ./Dockerfile + env_file: + - .env.example ports: - - "80:80" - - "9464:9464" \ No newline at end of file + - "8080:80" + - "9464:9464" + networks: + - localnet + +networks: + localnet: + external: true From 4a8824d69281f44b7ef9a4f1c00a1df56435232a Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 18 May 2026 18:45:48 +0300 Subject: [PATCH 43/61] refacto(oop-style-service): - added oop style service --- app/__dev_runner__.py | 6 ++ app/__version__.py | 2 +- app/common/modules/__init__.py | 2 +- app/common/modules/effects_api_gateway.py | 83 +++++++++++------------ app/dependencies.py | 12 ++++ app/effects/effects_controller.py | 2 +- app/effects/effects_mcp.py | 5 +- app/effects/effects_service.py | 36 +++++----- app/main.py | 10 +-- app/mcp.py | 2 +- app/provision/provision_controller.py | 2 +- app/provision/provision_service.py | 36 ++++------ docker-compose.yml | 2 +- 13 files changed, 104 insertions(+), 96 deletions(-) create mode 100644 app/__dev_runner__.py diff --git a/app/__dev_runner__.py b/app/__dev_runner__.py new file mode 100644 index 0000000..d85f2ff --- /dev/null +++ b/app/__dev_runner__.py @@ -0,0 +1,6 @@ +import uvicorn + +from app.main import app + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8020) diff --git a/app/__version__.py b/app/__version__.py index 87f0a6a..d1a212f 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.2" +APP_VERSION = "0.1.4" diff --git a/app/common/modules/__init__.py b/app/common/modules/__init__.py index 44b199a..d639e53 100644 --- a/app/common/modules/__init__.py +++ b/app/common/modules/__init__.py @@ -1,6 +1,6 @@ from .attribute_parser import attribute_parser from .data_restorator import data_restorator -from .effects_api_gateway import effects_api_gateway +from .effects_api_gateway import EffectsAPIGateway from .matrix_builder import matrix_builder from .name_mappings import ( ATTRIBUTES_MAP, diff --git a/app/common/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py index 829f4f5..11c396b 100644 --- a/app/common/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -4,15 +4,21 @@ import pandas as pd from shapely.geometry import shape +from app.common.api_handler.api_handler import APIHandler from app.common.exceptions.http_exception_wrapper import http_exception -from app.dependencies import urban_api_handler class EffectsAPIGateway: - @staticmethod + def __init__(self, api_handler: APIHandler) -> None: + self.api_handler = api_handler + async def get_service_normative( - territory_id: int, context_ids: list[int], service_type_id: int, token: str + self, + territory_id: int, + context_ids: list[int], + service_type_id: int, + token: str, ) -> dict[str, int | str]: """ Function retrieves normative data from urban_api @@ -28,13 +34,13 @@ async def get_service_normative( """ if len(context_ids) == 1: - response = await urban_api_handler.get( + response = await self.api_handler.get( f"/api/v1/territory/{context_ids[0]}/normatives", headers={"Authorization": f"Bearer {token}"} if token else None, ) request_ter_id = context_ids[0] else: - response = await urban_api_handler.get( + response = await self.api_handler.get( f"/api/v1/territory/{territory_id}/normatives", headers={"Authorization": f"Bearer {token}"} if token else None, ) @@ -115,8 +121,9 @@ async def get_service_normative( _detail={"Available service ids": response_df["service_type_id"].to_list()}, ) - @staticmethod - async def get_project_data(project_id: int, token: str) -> dict[str, int | dict]: + async def get_project_data( + self, project_id: int, token: str + ) -> dict[str, int | dict]: """ Function retrieves project territory data from urban_api Args: @@ -126,15 +133,16 @@ async def get_project_data(project_id: int, token: str) -> dict[str, int | dict] dict with "geometry" field as dict with "type" and "coordinates" fields and field "base_scenario_id" """ - response = await urban_api_handler.get( + response = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}", headers={"Authorization": f"Bearer {token}"} if token else None, ) return response - @staticmethod - async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFrame: + async def get_scenario_buildings( + self, scenario_id: int, token: str + ) -> gpd.GeoDataFrame: """ Function retrieves scenario buildings data from urban_api Args: @@ -144,7 +152,7 @@ async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFra gpd.GeoDataFrame: buildings layer, can be empty """ - buildings = await urban_api_handler.get( + buildings = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={"physical_object_type_id": 4}, headers={"Authorization": f"Bearer {token}"} if token else None, @@ -155,9 +163,8 @@ async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFra buildings_gdf.set_crs(4326, inplace=True) return buildings_gdf - @staticmethod async def get_project_context_buildings( - scenario_id: int, token: str + self, scenario_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario context buildings data from urban_api @@ -170,7 +177,7 @@ async def get_project_context_buildings( 404, http exception living buildings not found """ - context_buildings = await urban_api_handler.get( + context_buildings = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "physical_object_type_id": 4, @@ -183,9 +190,8 @@ async def get_project_context_buildings( context_buildings_gdf.set_crs(4326, inplace=True) return context_buildings_gdf - @staticmethod async def get_scenario_services( - scenario_id: int, service_type_id: int, token: str + self, scenario_id: int, service_type_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario services data from urban_api @@ -197,7 +203,7 @@ async def get_scenario_services( gpd.GeoDataFrame: services layer, can be empty """ - services = await urban_api_handler.get( + services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={ "service_type_id": service_type_id, @@ -210,8 +216,8 @@ async def get_scenario_services( services_gdf.set_crs(4326, inplace=True) return services_gdf - @staticmethod async def get_project_context_services( + self, scenario_id: int, service_type_id: int, token: str, @@ -226,7 +232,7 @@ async def get_project_context_services( gpd.GeoDataFrame: context services layer. Can be empty """ - context_services = await urban_api_handler.get( + context_services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "service_type_id": service_type_id, @@ -239,9 +245,8 @@ async def get_project_context_services( context_services_gdf.set_crs(4326, inplace=True) return context_services_gdf - @staticmethod async def get_scenario_population_data( - scenario_id: int | None, token: str + self, scenario_id: int | None, token: str ) -> int | None: """ Function retrieves population data from urban_api @@ -252,7 +257,7 @@ async def get_scenario_population_data( int | none: population data layer, if < 1 returns None """ - population = await urban_api_handler.get( + population = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/indicators_values", params={ "indicator_ids": 1, @@ -264,8 +269,9 @@ async def get_scenario_population_data( return None return value - @staticmethod - async def get_context_population(territory_ids_list: list[int], token: str) -> int: + async def get_context_population( + self, territory_ids_list: list[int], token: str + ) -> int: """ Function retrieves territory population data from urban_api by territory id Args: @@ -276,7 +282,7 @@ async def get_context_population(territory_ids_list: list[int], token: str) -> i """ task_list = [ - urban_api_handler.get( + self.api_handler.get( endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", params={"indicator_ids": 1}, headers={"Authorization": f"Bearer {token}"} if token else None, @@ -287,8 +293,9 @@ async def get_context_population(territory_ids_list: list[int], token: str) -> i result = await asyncio.gather(*task_list) return sum([item[0]["value"] for item in result]) - @staticmethod - async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame: + async def get_project_territory( + self, project_id: int, token: str + ) -> gpd.GeoDataFrame: """ Function retrieves territory data from urban_api Args: @@ -298,7 +305,7 @@ async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame gpd.GeoDataFrame: territory data layer """ - territory = await urban_api_handler.get( + territory = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}/territory", headers={"Authorization": f"Bearer {token}"} if token else None, ) @@ -307,8 +314,7 @@ async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame ) return territory_gdf - @staticmethod - async def get_default_capacity(service_type_id: int) -> int: + async def get_default_capacity(self, service_type_id: int) -> int: """ Function retrieves default capacity data from urban_api Args: @@ -317,17 +323,14 @@ async def get_default_capacity(service_type_id: int) -> int: int: default capacity value """ - service_types = await urban_api_handler.get( - endpoint_url="/api/v1/service_types" - ) + service_types = await self.api_handler.get(endpoint_url="/api/v1/service_types") service_types_df = pd.DataFrame.from_records(service_types).fillna(0) return service_types_df[ service_types_df["service_type_id"] == service_type_id ].iloc[0]["capacity_modeled"] - @staticmethod async def get_services_with_context( - scenario_id: int, service_type_id: int, token: str | None = None + self, scenario_id: int, service_type_id: int, token: str | None = None ) -> gpd.GeoDataFrame: """ Function retrieves service by service_type_id for scenario ID from urban api with context. @@ -339,7 +342,7 @@ async def get_services_with_context( gpd.GeoDataFrame: layer with services in 4326 crs. """ - services = await urban_api_handler.get( + services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", params={ "service_type_id": service_type_id, @@ -349,9 +352,8 @@ async def get_services_with_context( ) return gpd.GeoDataFrame.from_features(services, crs=4326) - @staticmethod async def get_physical_objects_with_context( - scenario_id: int, physical_object_type_id: int, token: str | None = None + self, scenario_id: int, physical_object_type_id: int, token: str | None = None ): """ Function retrieves physical objects by physical_object_type_id for scenario ID from urban api with context. @@ -363,7 +365,7 @@ async def get_physical_objects_with_context( gpd.GeoDataFrame: layer with physical_objects in 4326 crs. """ - physical_objects = await urban_api_handler.get( + physical_objects = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", params={ "physical_object_type_id": physical_object_type_id, @@ -372,6 +374,3 @@ async def get_physical_objects_with_context( headers={"Authorization": f"Bearer {token}"} if token else None, ) return gpd.GeoDataFrame.from_features(physical_objects, crs=4326) - - -effects_api_gateway = EffectsAPIGateway() diff --git a/app/dependencies.py b/app/dependencies.py index 81a1606..5200480 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -5,6 +5,9 @@ from app.common.api_handler.api_handler import APIHandler from app.common.exceptions.http_exception_wrapper import http_exception +from app.common.modules.effects_api_gateway import EffectsAPIGateway +from app.effects.effects_service import EffectsService +from app.provision.provision_service import ProvisionService logger.remove() logger.add(sys.stderr, level="INFO") @@ -21,3 +24,12 @@ ) urban_api_handler = APIHandler(config.get("URBAN_API")) +urban_api_mcp_handler = APIHandler(config.get("MCP_URBAN_API")) + +effects_api_gateway = EffectsAPIGateway(urban_api_handler) +effects_api_mcp_gateway = EffectsAPIGateway(urban_api_mcp_handler) + +effects_service = EffectsService(effects_api_gateway) +effects_mcp_service = EffectsService(effects_api_mcp_gateway) + +provision_service = ProvisionService(effects_api_gateway) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index d4e92e9..fdfc51e 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dependencies import effects_service from app.dto.provision_dto import ProvisionDTO -from .effects_service import effects_service from .shemas.effects_base_schema import EffectsSchema effects_router = APIRouter(prefix="/effects") diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index e2fcfbd..413be03 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -4,10 +4,9 @@ from fastmcp.server.dependencies import CurrentContext, get_access_token from loguru import logger +from app.dependencies import effects_mcp_service from app.dto.provision_dto import ProvisionDTO -from .effects_service import effects_service - effects_mcp = FastMCP("Object Effects MCP server") @@ -62,7 +61,7 @@ async def calc_provision_effects( service_type_id=service_type_id, target_population=target_population, ) - result = await effects_service.calculate_effects( + result = await effects_mcp_service.calculate_effects( effects_dto, token, for_mcp=True ) return result diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index c020ec0..6cc43e2 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -11,9 +11,9 @@ BUILDINGS_DROP_COLUMNS, EFFECTS_MAP, SERVICE_DROP_COLUMNS, + EffectsAPIGateway, attribute_parser, data_restorator, - effects_api_gateway, matrix_builder, objectnat_calculator, ) @@ -27,6 +27,9 @@ class EffectsService: Class for handling services calculation """ + def __init__(self, gateway: EffectsAPIGateway) -> None: + self.gateway = gateway + @staticmethod async def _get_pivot( effects: pd.DataFrame | gpd.GeoDataFrame, @@ -97,25 +100,25 @@ async def calculate_effects( logger.info( f"Started calculating effects for {effects_params.scenario_id} and service{effects_params.service_type_id}" ) - project_data = await effects_api_gateway.get_project_data( + project_data = await self.gateway.get_project_data( effects_params.project_id, token ) - project_territory = await effects_api_gateway.get_project_territory( + project_territory = await self.gateway.get_project_territory( effects_params.project_id, token ) - service_default_capacity = await effects_api_gateway.get_default_capacity( + service_default_capacity = await self.gateway.get_default_capacity( service_type_id=effects_params.service_type_id ) - normative_data = await effects_api_gateway.get_service_normative( + normative_data = await self.gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=effects_params.service_type_id, token=token, ) - context_population = await effects_api_gateway.get_context_population( + context_population = await self.gateway.get_context_population( territory_ids_list=project_data["properties"]["context"], token=token ) - context_buildings = await effects_api_gateway.get_project_context_buildings( + context_buildings = await self.gateway.get_project_context_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) context_buildings.drop( @@ -132,7 +135,7 @@ async def calculate_effects( target_population=context_population, ) context_buildings["is_project"] = False - context_services = await effects_api_gateway.get_project_context_services( + context_services = await self.gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, token=token, @@ -148,12 +151,10 @@ async def calculate_effects( context_services = await attribute_parser.parse_all_from_services( services=context_services, service_default_capacity=service_default_capacity ) - target_scenario_population = ( - await effects_api_gateway.get_scenario_population_data( - scenario_id=effects_params.scenario_id, token=token - ) + target_scenario_population = await self.gateway.get_scenario_population_data( + scenario_id=effects_params.scenario_id, token=token ) - target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + target_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=effects_params.scenario_id, token=token ) target_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -167,7 +168,7 @@ async def calculate_effects( target_population=target_scenario_population, ) target_scenario_buildings["is_project"] = True - target_scenario_services = await effects_api_gateway.get_scenario_services( + target_scenario_services = await self.gateway.get_scenario_services( scenario_id=effects_params.scenario_id, service_type_id=effects_params.service_type_id, token=token, @@ -176,7 +177,7 @@ async def calculate_effects( services=target_scenario_services, service_default_capacity=service_default_capacity, ) - base_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + base_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) base_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -189,7 +190,7 @@ async def calculate_effects( service_normative_type=normative_data["capacity_type"], ) base_scenario_buildings["is_project"] = True - base_scenario_services = await effects_api_gateway.get_scenario_services( + base_scenario_services = await self.gateway.get_scenario_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, token=token, @@ -825,6 +826,3 @@ async def form_llm_context( }, } return json.dumps(result) - - -effects_service = EffectsService() diff --git a/app/main.py b/app/main.py index b96e4cc..9611348 100644 --- a/app/main.py +++ b/app/main.py @@ -48,7 +48,11 @@ async def lifespan(app: FastAPI): version=APP_VERSION, lifespan=combine_lifespans(lifespan, effects_mcp_app.lifespan), ) -app.mount("/effects", effects_mcp_app) + +app.include_router(effects_router) +app.include_router(provision_router) + +app.mount("/effects/mcp", effects_mcp_app) # Add CORS middleware app.add_middleware( @@ -98,7 +102,3 @@ async def get_logs(): _input={"log_file_name": ".log"}, _detail={"error": e.__str__()}, ) - - -app.include_router(effects_router) -app.include_router(provision_router) diff --git a/app/mcp.py b/app/mcp.py index 28b2b03..4313aba 100644 --- a/app/mcp.py +++ b/app/mcp.py @@ -1,3 +1,3 @@ from app.effects.effects_mcp import effects_mcp -effects_mcp_app = effects_mcp.http_app(path="/mcp") +effects_mcp_app = effects_mcp.http_app(path="/") diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py index 357a008..7c8c081 100644 --- a/app/provision/provision_controller.py +++ b/app/provision/provision_controller.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dependencies import provision_service from app.dto.provision_dto import ProvisionDTO -from app.provision.provision_service import provision_service from app.schemas.provision_base_schema import ProvisionSchema provision_router = APIRouter(prefix="/provision", tags=["provision"]) diff --git a/app/provision/provision_service.py b/app/provision/provision_service.py index 964421e..8d3d8ff 100644 --- a/app/provision/provision_service.py +++ b/app/provision/provision_service.py @@ -6,12 +6,12 @@ from app.common.exceptions.http_exception_wrapper import http_exception from app.common.modules import ( + EffectsAPIGateway, attribute_parser, data_restorator, matrix_builder, objectnat_calculator, ) -from app.common.modules.effects_api_gateway import effects_api_gateway from app.dto.provision_dto import ProvisionDTO from app.schemas.provision_base_schema import ProvisionSchema @@ -20,12 +20,11 @@ class ProvisionService: - def __init__(self): - pass + def __init__(self, gateway: EffectsAPIGateway) -> None: + self.gateway = gateway - @staticmethod async def calculate_provision( - provision_params: ProvisionDTO, token: str + self, provision_params: ProvisionDTO, token: str ) -> ProvisionSchema: """ Calculate provision effects by project data and target scenario @@ -39,25 +38,25 @@ async def calculate_provision( logger.info( f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" ) - project_data = await effects_api_gateway.get_project_data( + project_data = await self.gateway.get_project_data( provision_params.project_id, token ) - project_territory = await effects_api_gateway.get_project_territory( + project_territory = await self.gateway.get_project_territory( provision_params.project_id, token ) - service_default_capacity = await effects_api_gateway.get_default_capacity( + service_default_capacity = await self.gateway.get_default_capacity( service_type_id=provision_params.service_type_id ) - normative_data = await effects_api_gateway.get_service_normative( + normative_data = await self.gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=provision_params.service_type_id, token=token, ) - context_population = await effects_api_gateway.get_context_population( + context_population = await self.gateway.get_context_population( territory_ids_list=project_data["properties"]["context"], token=token ) - context_buildings = await effects_api_gateway.get_project_context_buildings( + context_buildings = await self.gateway.get_project_context_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) context_buildings.drop( @@ -74,7 +73,7 @@ async def calculate_provision( target_population=context_population, ) context_buildings["is_project"] = False - context_services = await effects_api_gateway.get_project_context_services( + context_services = await self.gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=provision_params.service_type_id, token=token, @@ -90,12 +89,10 @@ async def calculate_provision( context_services = await attribute_parser.parse_all_from_services( services=context_services, service_default_capacity=service_default_capacity ) - target_scenario_population = ( - await effects_api_gateway.get_scenario_population_data( - scenario_id=provision_params.scenario_id, token=token - ) + target_scenario_population = await self.gateway.get_scenario_population_data( + scenario_id=provision_params.scenario_id, token=token ) - target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + target_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=provision_params.scenario_id, token=token ) target_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -109,7 +106,7 @@ async def calculate_provision( target_population=target_scenario_population, ) target_scenario_buildings["is_project"] = True - target_scenario_services = await effects_api_gateway.get_scenario_services( + target_scenario_services = await self.gateway.get_scenario_services( scenario_id=provision_params.scenario_id, service_type_id=provision_params.service_type_id, token=token, @@ -161,6 +158,3 @@ async def calculate_provision( f"Calculated PROVISION for {provision_params.scenario_id} and {provision_params.service_type_id}" ) return ProvisionSchema(**result) - - -provision_service = ProvisionService() diff --git a/docker-compose.yml b/docker-compose.yml index 0740fb3..cd9a07a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: context: . dockerfile: ./Dockerfile env_file: - - .env.example + - .env.development ports: - "8080:80" - "9464:9464" From 804015f9e4779c6b5e860a9ffd4b24fcdec5a20a Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 18 May 2026 18:58:26 +0300 Subject: [PATCH 44/61] feat(env): - added env example --- .env.example | 4 ++++ .gitignore | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..98456e2 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +LOGS_FILE="object_effects" +URBAN_API="https://urban-api.testing" +MCP_URBAN_API="https://urban-api.testing" +PROMETHEUS_PORT=9464 diff --git a/.gitignore b/.gitignore index 4b74b34..7a79b92 100644 --- a/.gitignore +++ b/.gitignore @@ -120,7 +120,8 @@ celerybeat.pid *.sage.py # Environments -.env* +.env.development +.env.production .venv env/ venv/ @@ -160,4 +161,4 @@ cython_debug/ .idea/ # Notebooks -*.ipynb \ No newline at end of file +*.ipynb From 8dcbf6811f56b98ecbd37528b8ba5f3c0d2534db Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Mon, 18 May 2026 19:00:11 +0300 Subject: [PATCH 45/61] v0.1.4 (#34) * refactor(oop-style-service): - added oop style service * feat(env): - added env example --- .env.example | 4 ++ .gitignore | 5 +- app/__dev_runner__.py | 6 ++ app/__version__.py | 2 +- app/common/modules/__init__.py | 2 +- app/common/modules/effects_api_gateway.py | 83 +++++++++++------------ app/dependencies.py | 12 ++++ app/effects/effects_controller.py | 2 +- app/effects/effects_mcp.py | 5 +- app/effects/effects_service.py | 36 +++++----- app/main.py | 10 +-- app/mcp.py | 2 +- app/provision/provision_controller.py | 2 +- app/provision/provision_service.py | 36 ++++------ docker-compose.yml | 2 +- 15 files changed, 111 insertions(+), 98 deletions(-) create mode 100644 .env.example create mode 100644 app/__dev_runner__.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..98456e2 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +LOGS_FILE="object_effects" +URBAN_API="https://urban-api.testing" +MCP_URBAN_API="https://urban-api.testing" +PROMETHEUS_PORT=9464 diff --git a/.gitignore b/.gitignore index 4b74b34..7a79b92 100644 --- a/.gitignore +++ b/.gitignore @@ -120,7 +120,8 @@ celerybeat.pid *.sage.py # Environments -.env* +.env.development +.env.production .venv env/ venv/ @@ -160,4 +161,4 @@ cython_debug/ .idea/ # Notebooks -*.ipynb \ No newline at end of file +*.ipynb diff --git a/app/__dev_runner__.py b/app/__dev_runner__.py new file mode 100644 index 0000000..d85f2ff --- /dev/null +++ b/app/__dev_runner__.py @@ -0,0 +1,6 @@ +import uvicorn + +from app.main import app + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8020) diff --git a/app/__version__.py b/app/__version__.py index 87f0a6a..d1a212f 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.2" +APP_VERSION = "0.1.4" diff --git a/app/common/modules/__init__.py b/app/common/modules/__init__.py index 44b199a..d639e53 100644 --- a/app/common/modules/__init__.py +++ b/app/common/modules/__init__.py @@ -1,6 +1,6 @@ from .attribute_parser import attribute_parser from .data_restorator import data_restorator -from .effects_api_gateway import effects_api_gateway +from .effects_api_gateway import EffectsAPIGateway from .matrix_builder import matrix_builder from .name_mappings import ( ATTRIBUTES_MAP, diff --git a/app/common/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py index 829f4f5..11c396b 100644 --- a/app/common/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -4,15 +4,21 @@ import pandas as pd from shapely.geometry import shape +from app.common.api_handler.api_handler import APIHandler from app.common.exceptions.http_exception_wrapper import http_exception -from app.dependencies import urban_api_handler class EffectsAPIGateway: - @staticmethod + def __init__(self, api_handler: APIHandler) -> None: + self.api_handler = api_handler + async def get_service_normative( - territory_id: int, context_ids: list[int], service_type_id: int, token: str + self, + territory_id: int, + context_ids: list[int], + service_type_id: int, + token: str, ) -> dict[str, int | str]: """ Function retrieves normative data from urban_api @@ -28,13 +34,13 @@ async def get_service_normative( """ if len(context_ids) == 1: - response = await urban_api_handler.get( + response = await self.api_handler.get( f"/api/v1/territory/{context_ids[0]}/normatives", headers={"Authorization": f"Bearer {token}"} if token else None, ) request_ter_id = context_ids[0] else: - response = await urban_api_handler.get( + response = await self.api_handler.get( f"/api/v1/territory/{territory_id}/normatives", headers={"Authorization": f"Bearer {token}"} if token else None, ) @@ -115,8 +121,9 @@ async def get_service_normative( _detail={"Available service ids": response_df["service_type_id"].to_list()}, ) - @staticmethod - async def get_project_data(project_id: int, token: str) -> dict[str, int | dict]: + async def get_project_data( + self, project_id: int, token: str + ) -> dict[str, int | dict]: """ Function retrieves project territory data from urban_api Args: @@ -126,15 +133,16 @@ async def get_project_data(project_id: int, token: str) -> dict[str, int | dict] dict with "geometry" field as dict with "type" and "coordinates" fields and field "base_scenario_id" """ - response = await urban_api_handler.get( + response = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}", headers={"Authorization": f"Bearer {token}"} if token else None, ) return response - @staticmethod - async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFrame: + async def get_scenario_buildings( + self, scenario_id: int, token: str + ) -> gpd.GeoDataFrame: """ Function retrieves scenario buildings data from urban_api Args: @@ -144,7 +152,7 @@ async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFra gpd.GeoDataFrame: buildings layer, can be empty """ - buildings = await urban_api_handler.get( + buildings = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={"physical_object_type_id": 4}, headers={"Authorization": f"Bearer {token}"} if token else None, @@ -155,9 +163,8 @@ async def get_scenario_buildings(scenario_id: int, token: str) -> gpd.GeoDataFra buildings_gdf.set_crs(4326, inplace=True) return buildings_gdf - @staticmethod async def get_project_context_buildings( - scenario_id: int, token: str + self, scenario_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario context buildings data from urban_api @@ -170,7 +177,7 @@ async def get_project_context_buildings( 404, http exception living buildings not found """ - context_buildings = await urban_api_handler.get( + context_buildings = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "physical_object_type_id": 4, @@ -183,9 +190,8 @@ async def get_project_context_buildings( context_buildings_gdf.set_crs(4326, inplace=True) return context_buildings_gdf - @staticmethod async def get_scenario_services( - scenario_id: int, service_type_id: int, token: str + self, scenario_id: int, service_type_id: int, token: str ) -> gpd.GeoDataFrame: """ Function retrieves scenario services data from urban_api @@ -197,7 +203,7 @@ async def get_scenario_services( gpd.GeoDataFrame: services layer, can be empty """ - services = await urban_api_handler.get( + services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={ "service_type_id": service_type_id, @@ -210,8 +216,8 @@ async def get_scenario_services( services_gdf.set_crs(4326, inplace=True) return services_gdf - @staticmethod async def get_project_context_services( + self, scenario_id: int, service_type_id: int, token: str, @@ -226,7 +232,7 @@ async def get_project_context_services( gpd.GeoDataFrame: context services layer. Can be empty """ - context_services = await urban_api_handler.get( + context_services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/geometries_with_all_objects", params={ "service_type_id": service_type_id, @@ -239,9 +245,8 @@ async def get_project_context_services( context_services_gdf.set_crs(4326, inplace=True) return context_services_gdf - @staticmethod async def get_scenario_population_data( - scenario_id: int | None, token: str + self, scenario_id: int | None, token: str ) -> int | None: """ Function retrieves population data from urban_api @@ -252,7 +257,7 @@ async def get_scenario_population_data( int | none: population data layer, if < 1 returns None """ - population = await urban_api_handler.get( + population = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/indicators_values", params={ "indicator_ids": 1, @@ -264,8 +269,9 @@ async def get_scenario_population_data( return None return value - @staticmethod - async def get_context_population(territory_ids_list: list[int], token: str) -> int: + async def get_context_population( + self, territory_ids_list: list[int], token: str + ) -> int: """ Function retrieves territory population data from urban_api by territory id Args: @@ -276,7 +282,7 @@ async def get_context_population(territory_ids_list: list[int], token: str) -> i """ task_list = [ - urban_api_handler.get( + self.api_handler.get( endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", params={"indicator_ids": 1}, headers={"Authorization": f"Bearer {token}"} if token else None, @@ -287,8 +293,9 @@ async def get_context_population(territory_ids_list: list[int], token: str) -> i result = await asyncio.gather(*task_list) return sum([item[0]["value"] for item in result]) - @staticmethod - async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame: + async def get_project_territory( + self, project_id: int, token: str + ) -> gpd.GeoDataFrame: """ Function retrieves territory data from urban_api Args: @@ -298,7 +305,7 @@ async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame gpd.GeoDataFrame: territory data layer """ - territory = await urban_api_handler.get( + territory = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}/territory", headers={"Authorization": f"Bearer {token}"} if token else None, ) @@ -307,8 +314,7 @@ async def get_project_territory(project_id: int, token: str) -> gpd.GeoDataFrame ) return territory_gdf - @staticmethod - async def get_default_capacity(service_type_id: int) -> int: + async def get_default_capacity(self, service_type_id: int) -> int: """ Function retrieves default capacity data from urban_api Args: @@ -317,17 +323,14 @@ async def get_default_capacity(service_type_id: int) -> int: int: default capacity value """ - service_types = await urban_api_handler.get( - endpoint_url="/api/v1/service_types" - ) + service_types = await self.api_handler.get(endpoint_url="/api/v1/service_types") service_types_df = pd.DataFrame.from_records(service_types).fillna(0) return service_types_df[ service_types_df["service_type_id"] == service_type_id ].iloc[0]["capacity_modeled"] - @staticmethod async def get_services_with_context( - scenario_id: int, service_type_id: int, token: str | None = None + self, scenario_id: int, service_type_id: int, token: str | None = None ) -> gpd.GeoDataFrame: """ Function retrieves service by service_type_id for scenario ID from urban api with context. @@ -339,7 +342,7 @@ async def get_services_with_context( gpd.GeoDataFrame: layer with services in 4326 crs. """ - services = await urban_api_handler.get( + services = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", params={ "service_type_id": service_type_id, @@ -349,9 +352,8 @@ async def get_services_with_context( ) return gpd.GeoDataFrame.from_features(services, crs=4326) - @staticmethod async def get_physical_objects_with_context( - scenario_id: int, physical_object_type_id: int, token: str | None = None + self, scenario_id: int, physical_object_type_id: int, token: str | None = None ): """ Function retrieves physical objects by physical_object_type_id for scenario ID from urban api with context. @@ -363,7 +365,7 @@ async def get_physical_objects_with_context( gpd.GeoDataFrame: layer with physical_objects in 4326 crs. """ - physical_objects = await urban_api_handler.get( + physical_objects = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/context/services_with_geometry", params={ "physical_object_type_id": physical_object_type_id, @@ -372,6 +374,3 @@ async def get_physical_objects_with_context( headers={"Authorization": f"Bearer {token}"} if token else None, ) return gpd.GeoDataFrame.from_features(physical_objects, crs=4326) - - -effects_api_gateway = EffectsAPIGateway() diff --git a/app/dependencies.py b/app/dependencies.py index 81a1606..5200480 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -5,6 +5,9 @@ from app.common.api_handler.api_handler import APIHandler from app.common.exceptions.http_exception_wrapper import http_exception +from app.common.modules.effects_api_gateway import EffectsAPIGateway +from app.effects.effects_service import EffectsService +from app.provision.provision_service import ProvisionService logger.remove() logger.add(sys.stderr, level="INFO") @@ -21,3 +24,12 @@ ) urban_api_handler = APIHandler(config.get("URBAN_API")) +urban_api_mcp_handler = APIHandler(config.get("MCP_URBAN_API")) + +effects_api_gateway = EffectsAPIGateway(urban_api_handler) +effects_api_mcp_gateway = EffectsAPIGateway(urban_api_mcp_handler) + +effects_service = EffectsService(effects_api_gateway) +effects_mcp_service = EffectsService(effects_api_mcp_gateway) + +provision_service = ProvisionService(effects_api_gateway) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index d4e92e9..fdfc51e 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dependencies import effects_service from app.dto.provision_dto import ProvisionDTO -from .effects_service import effects_service from .shemas.effects_base_schema import EffectsSchema effects_router = APIRouter(prefix="/effects") diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index e2fcfbd..413be03 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -4,10 +4,9 @@ from fastmcp.server.dependencies import CurrentContext, get_access_token from loguru import logger +from app.dependencies import effects_mcp_service from app.dto.provision_dto import ProvisionDTO -from .effects_service import effects_service - effects_mcp = FastMCP("Object Effects MCP server") @@ -62,7 +61,7 @@ async def calc_provision_effects( service_type_id=service_type_id, target_population=target_population, ) - result = await effects_service.calculate_effects( + result = await effects_mcp_service.calculate_effects( effects_dto, token, for_mcp=True ) return result diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index c020ec0..6cc43e2 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -11,9 +11,9 @@ BUILDINGS_DROP_COLUMNS, EFFECTS_MAP, SERVICE_DROP_COLUMNS, + EffectsAPIGateway, attribute_parser, data_restorator, - effects_api_gateway, matrix_builder, objectnat_calculator, ) @@ -27,6 +27,9 @@ class EffectsService: Class for handling services calculation """ + def __init__(self, gateway: EffectsAPIGateway) -> None: + self.gateway = gateway + @staticmethod async def _get_pivot( effects: pd.DataFrame | gpd.GeoDataFrame, @@ -97,25 +100,25 @@ async def calculate_effects( logger.info( f"Started calculating effects for {effects_params.scenario_id} and service{effects_params.service_type_id}" ) - project_data = await effects_api_gateway.get_project_data( + project_data = await self.gateway.get_project_data( effects_params.project_id, token ) - project_territory = await effects_api_gateway.get_project_territory( + project_territory = await self.gateway.get_project_territory( effects_params.project_id, token ) - service_default_capacity = await effects_api_gateway.get_default_capacity( + service_default_capacity = await self.gateway.get_default_capacity( service_type_id=effects_params.service_type_id ) - normative_data = await effects_api_gateway.get_service_normative( + normative_data = await self.gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=effects_params.service_type_id, token=token, ) - context_population = await effects_api_gateway.get_context_population( + context_population = await self.gateway.get_context_population( territory_ids_list=project_data["properties"]["context"], token=token ) - context_buildings = await effects_api_gateway.get_project_context_buildings( + context_buildings = await self.gateway.get_project_context_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) context_buildings.drop( @@ -132,7 +135,7 @@ async def calculate_effects( target_population=context_population, ) context_buildings["is_project"] = False - context_services = await effects_api_gateway.get_project_context_services( + context_services = await self.gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, token=token, @@ -148,12 +151,10 @@ async def calculate_effects( context_services = await attribute_parser.parse_all_from_services( services=context_services, service_default_capacity=service_default_capacity ) - target_scenario_population = ( - await effects_api_gateway.get_scenario_population_data( - scenario_id=effects_params.scenario_id, token=token - ) + target_scenario_population = await self.gateway.get_scenario_population_data( + scenario_id=effects_params.scenario_id, token=token ) - target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + target_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=effects_params.scenario_id, token=token ) target_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -167,7 +168,7 @@ async def calculate_effects( target_population=target_scenario_population, ) target_scenario_buildings["is_project"] = True - target_scenario_services = await effects_api_gateway.get_scenario_services( + target_scenario_services = await self.gateway.get_scenario_services( scenario_id=effects_params.scenario_id, service_type_id=effects_params.service_type_id, token=token, @@ -176,7 +177,7 @@ async def calculate_effects( services=target_scenario_services, service_default_capacity=service_default_capacity, ) - base_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + base_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) base_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -189,7 +190,7 @@ async def calculate_effects( service_normative_type=normative_data["capacity_type"], ) base_scenario_buildings["is_project"] = True - base_scenario_services = await effects_api_gateway.get_scenario_services( + base_scenario_services = await self.gateway.get_scenario_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=effects_params.service_type_id, token=token, @@ -825,6 +826,3 @@ async def form_llm_context( }, } return json.dumps(result) - - -effects_service = EffectsService() diff --git a/app/main.py b/app/main.py index b96e4cc..9611348 100644 --- a/app/main.py +++ b/app/main.py @@ -48,7 +48,11 @@ async def lifespan(app: FastAPI): version=APP_VERSION, lifespan=combine_lifespans(lifespan, effects_mcp_app.lifespan), ) -app.mount("/effects", effects_mcp_app) + +app.include_router(effects_router) +app.include_router(provision_router) + +app.mount("/effects/mcp", effects_mcp_app) # Add CORS middleware app.add_middleware( @@ -98,7 +102,3 @@ async def get_logs(): _input={"log_file_name": ".log"}, _detail={"error": e.__str__()}, ) - - -app.include_router(effects_router) -app.include_router(provision_router) diff --git a/app/mcp.py b/app/mcp.py index 28b2b03..4313aba 100644 --- a/app/mcp.py +++ b/app/mcp.py @@ -1,3 +1,3 @@ from app.effects.effects_mcp import effects_mcp -effects_mcp_app = effects_mcp.http_app(path="/mcp") +effects_mcp_app = effects_mcp.http_app(path="/") diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py index 357a008..7c8c081 100644 --- a/app/provision/provision_controller.py +++ b/app/provision/provision_controller.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends from app.common.auth.bearer import verify_bearer_token +from app.dependencies import provision_service from app.dto.provision_dto import ProvisionDTO -from app.provision.provision_service import provision_service from app.schemas.provision_base_schema import ProvisionSchema provision_router = APIRouter(prefix="/provision", tags=["provision"]) diff --git a/app/provision/provision_service.py b/app/provision/provision_service.py index 964421e..8d3d8ff 100644 --- a/app/provision/provision_service.py +++ b/app/provision/provision_service.py @@ -6,12 +6,12 @@ from app.common.exceptions.http_exception_wrapper import http_exception from app.common.modules import ( + EffectsAPIGateway, attribute_parser, data_restorator, matrix_builder, objectnat_calculator, ) -from app.common.modules.effects_api_gateway import effects_api_gateway from app.dto.provision_dto import ProvisionDTO from app.schemas.provision_base_schema import ProvisionSchema @@ -20,12 +20,11 @@ class ProvisionService: - def __init__(self): - pass + def __init__(self, gateway: EffectsAPIGateway) -> None: + self.gateway = gateway - @staticmethod async def calculate_provision( - provision_params: ProvisionDTO, token: str + self, provision_params: ProvisionDTO, token: str ) -> ProvisionSchema: """ Calculate provision effects by project data and target scenario @@ -39,25 +38,25 @@ async def calculate_provision( logger.info( f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" ) - project_data = await effects_api_gateway.get_project_data( + project_data = await self.gateway.get_project_data( provision_params.project_id, token ) - project_territory = await effects_api_gateway.get_project_territory( + project_territory = await self.gateway.get_project_territory( provision_params.project_id, token ) - service_default_capacity = await effects_api_gateway.get_default_capacity( + service_default_capacity = await self.gateway.get_default_capacity( service_type_id=provision_params.service_type_id ) - normative_data = await effects_api_gateway.get_service_normative( + normative_data = await self.gateway.get_service_normative( territory_id=project_data["territory"]["id"], context_ids=project_data["properties"]["context"], service_type_id=provision_params.service_type_id, token=token, ) - context_population = await effects_api_gateway.get_context_population( + context_population = await self.gateway.get_context_population( territory_ids_list=project_data["properties"]["context"], token=token ) - context_buildings = await effects_api_gateway.get_project_context_buildings( + context_buildings = await self.gateway.get_project_context_buildings( scenario_id=project_data["base_scenario"]["id"], token=token ) context_buildings.drop( @@ -74,7 +73,7 @@ async def calculate_provision( target_population=context_population, ) context_buildings["is_project"] = False - context_services = await effects_api_gateway.get_project_context_services( + context_services = await self.gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], service_type_id=provision_params.service_type_id, token=token, @@ -90,12 +89,10 @@ async def calculate_provision( context_services = await attribute_parser.parse_all_from_services( services=context_services, service_default_capacity=service_default_capacity ) - target_scenario_population = ( - await effects_api_gateway.get_scenario_population_data( - scenario_id=provision_params.scenario_id, token=token - ) + target_scenario_population = await self.gateway.get_scenario_population_data( + scenario_id=provision_params.scenario_id, token=token ) - target_scenario_buildings = await effects_api_gateway.get_scenario_buildings( + target_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=provision_params.scenario_id, token=token ) target_scenario_buildings = await attribute_parser.parse_all_from_buildings( @@ -109,7 +106,7 @@ async def calculate_provision( target_population=target_scenario_population, ) target_scenario_buildings["is_project"] = True - target_scenario_services = await effects_api_gateway.get_scenario_services( + target_scenario_services = await self.gateway.get_scenario_services( scenario_id=provision_params.scenario_id, service_type_id=provision_params.service_type_id, token=token, @@ -161,6 +158,3 @@ async def calculate_provision( f"Calculated PROVISION for {provision_params.scenario_id} and {provision_params.service_type_id}" ) return ProvisionSchema(**result) - - -provision_service = ProvisionService() diff --git a/docker-compose.yml b/docker-compose.yml index 0740fb3..cd9a07a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: context: . dockerfile: ./Dockerfile env_file: - - .env.example + - .env.development ports: - "8080:80" - "9464:9464" From 5ae31c48906288cf9b16205a6d3b176babfb5a22 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 28 May 2026 20:48:24 +0300 Subject: [PATCH 46/61] feat(env): - added env example --- app/__version__.py | 2 +- app/common/modules/effects_api_gateway.py | 18 ++++++++++++++++++ app/effects/effects_mcp.py | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/__version__.py b/app/__version__.py index d1a212f..a9e42e4 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.4" +APP_VERSION = "0.2.4" diff --git a/app/common/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py index 11c396b..fa9204a 100644 --- a/app/common/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -13,6 +13,24 @@ class EffectsAPIGateway: def __init__(self, api_handler: APIHandler) -> None: self.api_handler = api_handler + async def get_project_id_by_scenario(self, scenario_id: int, token: str) -> int: + """ + Function retrieves project ID based on scenario ID from Urban API. + Args: + scenario_id (int): Scenario ID from Urban API. + token (str): User access token. + Returns: + int: Project ID from Urban API. + Raises: + Any: HTTP from Urban API. + """ + + proj_resp = await self.api_handler.get( + f"/api/v1/scenarios/{scenario_id}", + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return proj_resp["project"]["project_id"] + async def get_service_normative( self, territory_id: int, diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index 413be03..ee77134 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -52,9 +52,11 @@ async def calc_provision_effects( ): try: - project_id = int(ctx.request_context.meta.project_id) scenario_id = int(ctx.request_context.meta.scenario_id) token = get_access_token() + project_id = await effects_mcp_service.gateway.get_project_id_by_scenario( + scenario_id, token + ) effects_dto = ProvisionDTO( project_id=project_id, scenario_id=scenario_id, From 79562faeb4d88d0820d878dfb68f67997f5bad1d Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Thu, 28 May 2026 20:52:56 +0300 Subject: [PATCH 47/61] v0.2.4 (#36) * refactor(project_id removed from mcp) --- app/__version__.py | 2 +- app/common/modules/effects_api_gateway.py | 18 ++++++++++++++++++ app/effects/effects_mcp.py | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/__version__.py b/app/__version__.py index d1a212f..a9e42e4 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.1.4" +APP_VERSION = "0.2.4" diff --git a/app/common/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py index 11c396b..fa9204a 100644 --- a/app/common/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -13,6 +13,24 @@ class EffectsAPIGateway: def __init__(self, api_handler: APIHandler) -> None: self.api_handler = api_handler + async def get_project_id_by_scenario(self, scenario_id: int, token: str) -> int: + """ + Function retrieves project ID based on scenario ID from Urban API. + Args: + scenario_id (int): Scenario ID from Urban API. + token (str): User access token. + Returns: + int: Project ID from Urban API. + Raises: + Any: HTTP from Urban API. + """ + + proj_resp = await self.api_handler.get( + f"/api/v1/scenarios/{scenario_id}", + headers={"Authorization": f"Bearer {token}"} if token else None, + ) + return proj_resp["project"]["project_id"] + async def get_service_normative( self, territory_id: int, diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index 413be03..ee77134 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -52,9 +52,11 @@ async def calc_provision_effects( ): try: - project_id = int(ctx.request_context.meta.project_id) scenario_id = int(ctx.request_context.meta.scenario_id) token = get_access_token() + project_id = await effects_mcp_service.gateway.get_project_id_by_scenario( + scenario_id, token + ) effects_dto = ProvisionDTO( project_id=project_id, scenario_id=scenario_id, From 0fe9ff049f2cfbb4343f0b6513fb1b24e15794b5 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 29 May 2026 16:59:40 +0300 Subject: [PATCH 48/61] refactor(meta): - changed meta params --- app/effects/effects_mcp.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index ee77134..a5b88fd 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -1,7 +1,7 @@ import traceback from fastmcp import FastMCP -from fastmcp.server.dependencies import CurrentContext, get_access_token +from fastmcp.server.dependencies import get_access_token from loguru import logger from app.dependencies import effects_mcp_service @@ -14,10 +14,14 @@ name="CalculateObjectEffects", title="Get provision effects for service", description=""" - Retrieve service provision effects by service id. + Retrieve service provision effects by service id for scenario id. If total population is provided, demand is restored from it. Otherwise, population is restored from living square. - Args to select: + Args to select: + - scenario_id (int): Scenario ID from Urban API to calculate effects for. + - service_type_id (int): Service type ID to calculate provision effects for. + - target_population (int, optional): Total population for demand calculation. If not provided, population is restored from living square. + Returns effects layers with estimated pivot info for llm analyses. Response format: @@ -48,11 +52,10 @@ """, ) async def calc_provision_effects( - service_type_id: int, target_population: int | None = None, ctx=CurrentContext() + scenario_id: int, service_type_id: int, target_population: int | None = None ): try: - scenario_id = int(ctx.request_context.meta.scenario_id) token = get_access_token() project_id = await effects_mcp_service.gateway.get_project_id_by_scenario( scenario_id, token From 6be8c24824b8a4897106184b83890e9bfdf92936 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 29 May 2026 17:00:32 +0300 Subject: [PATCH 49/61] version(0.3.4): - upgraded app version --- app/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__version__.py b/app/__version__.py index a9e42e4..2b9da61 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.2.4" +APP_VERSION = "0.3.4" From 04f9534c5b8a5d1c36afb11e42a060b30c9a0638 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 29 May 2026 17:02:15 +0300 Subject: [PATCH 50/61] v0.3.4 (#38) * refactor(moved scenario_id from meta for mcp) From 41c99f6dae94860f68edd01314f448bd661f2eba Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:05:22 +0300 Subject: [PATCH 51/61] Feat/objectnamt v1 (#39) * feat(objectnat): - objectnat v1 in-progress * feat(provision_mcp): - added provision mcp endpoints - upgraded app version to 0.4.0 --- .gitignore | 4 + app/__version__.py | 2 +- app/common/config/__init__.py | 1 + app/common/config/config.py | 63 ++++++ app/common/modules/data_restorator.py | 30 ++- app/common/modules/name_mappings.py | 14 +- app/common/modules/objectnat_calculator.py | 40 ++-- app/dependencies.py | 3 +- app/effects/effects_mcp.py | 4 +- app/effects/effects_service.py | 4 + app/main.py | 7 +- app/mcp.py | 7 + app/provision/provision_controller.py | 17 +- app/provision/provision_mcp.py | 133 +++++++++++ app/provision/provision_service.py | 250 +++++++++++++++++---- app/schemas/provision_base_schema.py | 63 +++++- requirements.txt | Bin 992 -> 924 bytes 17 files changed, 560 insertions(+), 82 deletions(-) create mode 100644 app/common/config/__init__.py create mode 100644 app/common/config/config.py create mode 100644 app/provision/provision_mcp.py diff --git a/.gitignore b/.gitignore index 7a79b92..46a37a2 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,7 @@ cython_debug/ # Notebooks *.ipynb + +# Agents +CLAUDE.md +AGENTS.md diff --git a/app/__version__.py b/app/__version__.py index 2b9da61..e3ed1f4 100644 --- a/app/__version__.py +++ b/app/__version__.py @@ -1 +1 @@ -APP_VERSION = "0.3.4" +APP_VERSION = "0.4.0" diff --git a/app/common/config/__init__.py b/app/common/config/__init__.py new file mode 100644 index 0000000..cca5d9b --- /dev/null +++ b/app/common/config/__init__.py @@ -0,0 +1 @@ +from .config import Config diff --git a/app/common/config/config.py b/app/common/config/config.py new file mode 100644 index 0000000..271e678 --- /dev/null +++ b/app/common/config/config.py @@ -0,0 +1,63 @@ +import os +from pathlib import Path + + +class Config: + """ + Class for loading environment variables from .env.{APP_ENV} file + """ + + def __init__(self): + app_env = os.getenv("APP_ENV") + if not app_env: + raise ValueError("APP_ENV variable is not present") + env_file = Path().absolute() / f".env.{app_env}" + if not env_file.is_file(): + raise FileNotFoundError(f"Couldn't find file with .env.{app_env} name") + self._load_env_file(env_file) + + @staticmethod + def _load_env_file(env_file: Path) -> None: + """ + Function loads variables from env file, existing environment variables take precedence + Args: + env_file (Path): path to env file + """ + + for line in env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip().strip("'\"") + if key and key not in os.environ: + os.environ[key] = value + + @staticmethod + def get(key: str) -> str: + """ + Function gets environment variable value + Args: + key (str): name of environment variable + Returns: + str: value of environment variable + Raises: + ValueError: if environment variable is not set + """ + + value = os.getenv(key) + if value: + return value + raise ValueError(f"No such env: {key}") + + @staticmethod + def set(key: str, value: str) -> None: + """ + Function sets value for environment variable + Args: + key (str): name of environment variable + value (str): new value for environment variable + """ + + os.environ[key] = value diff --git a/app/common/modules/data_restorator.py b/app/common/modules/data_restorator.py index 62a9196..719cf23 100644 --- a/app/common/modules/data_restorator.py +++ b/app/common/modules/data_restorator.py @@ -3,7 +3,6 @@ import geopandas as gpd import numpy as np import pandas as pd -from objectnat import get_balanced_buildings from app.common.exceptions.http_exception_wrapper import http_exception @@ -50,6 +49,28 @@ def _restore_target_population( buildings = buildings.to_crs(local_crs) return int(sum(buildings.area * buildings["storeys_count"]) * 0.8 / 33) + @staticmethod + def _balance_population( + buildings: gpd.GeoDataFrame, + population: int, + ) -> gpd.GeoDataFrame: + """ + Function distributes population between buildings proportionally to their living area + Args: + buildings (gpd.GeoDataFrame): living buildings data with "living_area" attribute + population (int): total population to distribute + Returns: + gpd.GeoDataFrame: buildings data with restored "population" attribute + """ + + shares = buildings["living_area"] / buildings["living_area"].sum() + buildings["population"] = np.floor(shares * population).astype(int) + remainder = int(population - buildings["population"].sum()) + if remainder > 0: + top = (shares * population).mod(1).nlargest(remainder).index + buildings.loc[top, "population"] += 1 + return buildings + # ToDo delete crs transformation def _restore_population( self, @@ -75,12 +96,11 @@ def _restore_population( ) buildings["living_area"] = buildings.area * buildings["storeys_count"] * 0.8 buildings["living_area"] = buildings["living_area"].astype(int) - balanced_buildings = get_balanced_buildings( - living_buildings=buildings, + buildings = self._balance_population( + buildings=buildings, population=int(target_population), ) - balanced_buildings["population"] = balanced_buildings["population"].astype(int) - return balanced_buildings.to_crs(4326) + return buildings.to_crs(4326) @staticmethod def _generate_demand_per_building( diff --git a/app/common/modules/name_mappings.py b/app/common/modules/name_mappings.py index 1a185e4..7511959 100644 --- a/app/common/modules/name_mappings.py +++ b/app/common/modules/name_mappings.py @@ -12,18 +12,18 @@ "min_dist": "Минмиальное расстояне до сервиса (м)", "building_index": "ID здания", "service_index": "ID сервиса", - "supplyed_demands_within": "Удовлетворённый спрос в нормативной доступности (чел)", - "supplyed_demands_without": "Удовлетворённый спрос вне нормативной доступности (чел)", + "supplied_demands_within": "Удовлетворённый спрос в нормативной доступности (чел)", + "supplied_demands_without": "Удовлетворённый спрос вне нормативной доступности (чел)", "carried_capacity_within": "Обеспечено в радиусе нормативной доступности (чел)", "carried_capacity_without": "Обеспечено вне радиуса нормативной доступности (чел)", - "provison_value": "Оценка обеспеченности", - "supplyed_demands_within_before": "Удовлетворённый спрос в нормативной доступности (до) (чел)", + "provision_value": "Оценка обеспеченности", + "supplied_demands_within_before": "Удовлетворённый спрос в нормативной доступности (до) (чел)", "us_demands_within_before": "Неудовлетворённый спрос в нормативной доступности (до) (чел)", - "supplyed_demands_without_before": "Удовлетворённый спрос вне нормативной доступности (до) (чел)", + "supplied_demands_without_before": "Удовлетворённый спрос вне нормативной доступности (до) (чел)", "us_demands_without_before": "Неудовлетворённый спрос вне нормативной доступности (до) (чел)", - "supplyed_demands_within_after": "Удовлетворённый спрос в нормативной доступности (после) (чел)", + "supplied_demands_within_after": "Удовлетворённый спрос в нормативной доступности (после) (чел)", "us_demands_within_after": "Неудовлетворённый спрос в нормативной доступности (после) (чел)", - "supplyed_demands_without_after": "Удовлетворённый спрос вне нормативной доступности (после) (чел)", + "supplied_demands_without_after": "Удовлетворённый спрос вне нормативной доступности (после) (чел)", "us_demands_without_after": "Неудовлетворённый спрос вне нормативной доступности (после) (чел)", "is_scenario_object": "Сценарный объект", } diff --git a/app/common/modules/objectnat_calculator.py b/app/common/modules/objectnat_calculator.py index c9540f4..95779e8 100644 --- a/app/common/modules/objectnat_calculator.py +++ b/app/common/modules/objectnat_calculator.py @@ -99,16 +99,16 @@ def _calculate_effects( # ToDo fix calculation without/before effects = effects.copy() supplied_demand_within_before = effects[ - "supplyed_demands_within_before" + "supplied_demands_within_before" ].fillna(0) supplied_demand_without_before = effects[ - "supplyed_demands_without_before" + "supplied_demands_without_before" ].fillna(0) - supplied_demand_within_after = effects["supplyed_demands_within_after"].fillna( + supplied_demand_within_after = effects["supplied_demands_within_after"].fillna( 0 ) supplied_demand_without_after = effects[ - "supplyed_demands_without_after" + "supplied_demands_without_after" ].fillna(0) unsupplied_demand_within_before = effects["us_demands_within_before"].fillna(0) unsupplied_demand_within_after = effects["us_demands_within_after"].fillna(0) @@ -121,11 +121,11 @@ def _calculate_effects( effects.dropna(subset="is_project", inplace=True) project_total_supplied_demands_before = effects[effects["is_project"]][ - "supplyed_demands_without_before" + "supplied_demands_without_before" ].fillna(0) project_total_supplied_demands_after = effects[effects["is_project"]][ - "supplyed_demands_without_after" + "supplied_demands_without_after" ].fillna(0) project_total_us_demands_before = effects[effects["is_project"]][ @@ -193,41 +193,41 @@ def estimate_effects( gpd.GeoDataFrame: layer with effects, provision before and after attributes """ - provision_before["supplyed_demands_within_before"] = provision_before[ - "supplyed_demands_within" + provision_before["supplied_demands_within_before"] = provision_before[ + "supplied_demands_within" ].copy() provision_before["us_demands_within_before"] = ( provision_before["demand"] - - provision_before["supplyed_demands_within_before"] + - provision_before["supplied_demands_within_before"] ) - provision_before["supplyed_demands_without_before"] = ( - provision_before["supplyed_demands_without"] - + provision_before["supplyed_demands_within_before"] + provision_before["supplied_demands_without_before"] = ( + provision_before["supplied_demands_without"] + + provision_before["supplied_demands_within_before"] ) provision_before["us_demands_without_before"] = ( provision_before["demand"] - - provision_before["supplyed_demands_within_before"] + - provision_before["supplied_demands_within_before"] ) - provision_after["supplyed_demands_within_after"] = provision_after[ - "supplyed_demands_within" + provision_after["supplied_demands_within_after"] = provision_after[ + "supplied_demands_within" ].copy() provision_after["us_demands_within_after"] = ( - provision_after["demand"] - provision_after["supplyed_demands_within_after"] + provision_after["demand"] - provision_after["supplied_demands_within_after"] ) - provision_after["supplyed_demands_without_after"] = ( - provision_after["supplyed_demands_within_after"] - + provision_after["supplyed_demands_without"].copy() + provision_after["supplied_demands_without_after"] = ( + provision_after["supplied_demands_within_after"] + + provision_after["supplied_demands_without"].copy() ) provision_after["us_demands_without_after"] = ( provision_after["demand"] - - provision_after["supplyed_demands_without_after"] + - provision_after["supplied_demands_without_after"] ) effects = provision_after.merge( diff --git a/app/dependencies.py b/app/dependencies.py index 5200480..813e2f1 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -1,9 +1,9 @@ import sys -from iduconfig import Config from loguru import logger from app.common.api_handler.api_handler import APIHandler +from app.common.config.config import Config from app.common.exceptions.http_exception_wrapper import http_exception from app.common.modules.effects_api_gateway import EffectsAPIGateway from app.effects.effects_service import EffectsService @@ -33,3 +33,4 @@ effects_mcp_service = EffectsService(effects_api_mcp_gateway) provision_service = ProvisionService(effects_api_gateway) +provision_mcp_service = ProvisionService(effects_api_mcp_gateway) diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index a5b88fd..af7aa49 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -29,7 +29,7 @@ "before_prove_data": { "buildings": FeatureCollection, "services": FeatureCollection, - "": FeatureCollection + "links": FeatureCollection }, "after_prove_data": { "buildings": FeatureCollection, @@ -47,7 +47,7 @@ "average_absolute_within": float, "median_absolute_within": int, }, - "": str + "text_pivot": str } """, ) diff --git a/app/effects/effects_service.py b/app/effects/effects_service.py index 6cc43e2..7214bc2 100644 --- a/app/effects/effects_service.py +++ b/app/effects/effects_service.py @@ -154,6 +154,10 @@ async def calculate_effects( target_scenario_population = await self.gateway.get_scenario_population_data( scenario_id=effects_params.scenario_id, token=token ) + # User-provided population overrides the scenario population restored + # from Urban API (see the CalculateObjectEffects tool contract). + if effects_params.target_population: + target_scenario_population = effects_params.target_population target_scenario_buildings = await self.gateway.get_scenario_buildings( scenario_id=effects_params.scenario_id, token=token ) diff --git a/app/main.py b/app/main.py index 9611348..a0ace85 100644 --- a/app/main.py +++ b/app/main.py @@ -11,7 +11,7 @@ from .common.middlewares.prometheus_handler import ObservabilityMiddleware from .dependencies import config, http_exception from .effects.effects_controller import effects_router -from .mcp import effects_mcp_app +from .mcp import effects_mcp_app, provision_mcp_app from .observability import OpenTelemetryAgent, PrometheusConfig from .observability.metrics import setup_metrics from .provision.provision_controller import provision_router @@ -46,13 +46,16 @@ async def lifespan(app: FastAPI): title="ObjectNat effects API", description="API for calculating effects for territory by ObjectNat library", version=APP_VERSION, - lifespan=combine_lifespans(lifespan, effects_mcp_app.lifespan), + lifespan=combine_lifespans( + lifespan, effects_mcp_app.lifespan, provision_mcp_app.lifespan + ), ) app.include_router(effects_router) app.include_router(provision_router) app.mount("/effects/mcp", effects_mcp_app) +app.mount("/provision/mcp", provision_mcp_app) # Add CORS middleware app.add_middleware( diff --git a/app/mcp.py b/app/mcp.py index 4313aba..6c10dd7 100644 --- a/app/mcp.py +++ b/app/mcp.py @@ -1,3 +1,10 @@ from app.effects.effects_mcp import effects_mcp +from app.provision.provision_mcp import provision_mcp + +# Provision tools are also exposed on the effects MCP endpoint so that +# consumers (gMART agents, ChatStorage replay) reach every tool via the +# single OBJECTS_EFFECTS_MCP_SERVER URL. +effects_mcp.mount(provision_mcp) effects_mcp_app = effects_mcp.http_app(path="/") +provision_mcp_app = provision_mcp.http_app(path="/") diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py index 7c8c081..03826a6 100644 --- a/app/provision/provision_controller.py +++ b/app/provision/provision_controller.py @@ -5,7 +5,11 @@ from app.common.auth.bearer import verify_bearer_token from app.dependencies import provision_service from app.dto.provision_dto import ProvisionDTO -from app.schemas.provision_base_schema import ProvisionSchema +from app.schemas.provision_base_schema import ( + MultiProvisionRequestSchema, + MultiProvisionSchema, + ProvisionSchema, +) provision_router = APIRouter(prefix="/provision", tags=["provision"]) @@ -17,3 +21,14 @@ async def calculate_provision( ) -> ProvisionSchema: return await provision_service.calculate_provision(provision_dto, token) + + +@provision_router.post("/calc_provisions", response_model=MultiProvisionSchema) +async def calculate_multi_provision( + multi_provision_params: MultiProvisionRequestSchema, + token: str = Depends(verify_bearer_token), +) -> MultiProvisionSchema: + + return await provision_service.calculate_multi_provision( + multi_provision_params, token + ) diff --git a/app/provision/provision_mcp.py b/app/provision/provision_mcp.py new file mode 100644 index 0000000..19dabd2 --- /dev/null +++ b/app/provision/provision_mcp.py @@ -0,0 +1,133 @@ +import traceback + +from fastmcp import FastMCP +from fastmcp.server.dependencies import get_access_token +from loguru import logger + +from app.dependencies import provision_mcp_service +from app.dto.provision_dto import ProvisionDTO +from app.schemas.provision_base_schema import ( + MultiProvisionRequestSchema, + ServiceInfoSchema, +) + +provision_mcp = FastMCP("Object Provision MCP server") + + +@provision_mcp.tool( + name="CalculateServiceProvision", + title="Get service provision for scenario", + description=""" + Calculate service provision by service type id for scenario id. + Population and demand are restored from Urban API data, then provision is evaluated + with a gravity-based model within the service normative accessibility. + + Args to select: + - scenario_id (int): Scenario ID from Urban API to calculate provision for. + - service_type_id (int): Service type ID to calculate provision for. + - target_population (int, optional): Total population of the scenario territory for demand + calculation. If not provided, population is restored from Urban API data. + + Returns provision layers as GeoJSON FeatureCollections in WGS84 (EPSG:4326). + Response format: + { + "buildings": FeatureCollection, + "services": FeatureCollection, + "links": FeatureCollection + } + """, +) +async def calc_service_provision( + scenario_id: int, service_type_id: int, target_population: int | None = None +): + + try: + token = get_access_token() + project_id = await provision_mcp_service.gateway.get_project_id_by_scenario( + scenario_id, token + ) + provision_dto = ProvisionDTO( + project_id=project_id, + scenario_id=scenario_id, + service_type_id=service_type_id, + target_population=target_population, + ) + result = await provision_mcp_service.calculate_provision(provision_dto, token) + return result.model_dump() + except Exception as e: + tb = traceback.format_exc() + logger.opt(exception=True).error( + f"Error in MCP tool 'CalculateServiceProvision': {type(e).__name__}: {e}" + ) + raise Exception(f"{type(e).__name__}: {e}\n\nTraceback:\n{tb}") from e + + +@provision_mcp.tool( + name="CalculateServicesProvision", + title="Get provision for multiple services", + description=""" + Calculate service provision for several service types at once for scenario id. + Population and demand are restored from Urban API data, then provision is evaluated + per service type with a gravity-based model within the service normative accessibility. + + Args to select: + - scenario_id (int): Scenario ID from Urban API to calculate provision for. + - services (dict): Service type IDs to calculate, each with display name and layer flag: + {"22": {"name": "Школа", "as_layer": true}, "21": {"name": "Детский сад", "as_layer": false}} + For as_layer=true the response includes GeoJSON layers, otherwise only summary statistics. + - target_population (int, optional): Total population of the scenario territory for demand + calculation, shared by all services. If not provided, population is restored from Urban API data. + + Returns per-service results keyed by service type id. + Response format: + { + "services": { + "22": { + "name": str, + "summary": { + "services_count": int, + "total_capacity": int, + "total_demand": int, + "satisfied_demand_within": int, + "satisfied_demand_without": int, + "unsatisfied_demand": int, + "balance": int, + "deficit": int, + "surplus": int, + "average_provision_value": float, + "median_provision_value": float + }, + "layers": { + "buildings": FeatureCollection, + "services": FeatureCollection, + "links": FeatureCollection + } | null, + "error": str | null + } + } + } + """, +) +async def calc_services_provision( + scenario_id: int, + services: dict[int, ServiceInfoSchema], + target_population: int | None = None, +): + + try: + token = get_access_token() + multi_provision_params = MultiProvisionRequestSchema( + scenario_id=scenario_id, + services=services, + target_population=target_population, + ) + result = await provision_mcp_service.calculate_multi_provision( + multi_provision_params, token + ) + return result.model_dump() + except Exception as e: + tb = traceback.format_exc() + logger.opt(exception=True).error( + f"Error in MCP tool 'CalculateServicesProvision': {type(e).__name__}: {e}" + ) + raise Exception(f"{type(e).__name__}: {e}\n\nTraceback:\n{tb}") from e diff --git a/app/provision/provision_service.py b/app/provision/provision_service.py index 8d3d8ff..8e6d1fc 100644 --- a/app/provision/provision_service.py +++ b/app/provision/provision_service.py @@ -1,6 +1,7 @@ import asyncio import json +import geopandas as gpd import pandas as pd from loguru import logger @@ -13,7 +14,13 @@ objectnat_calculator, ) from app.dto.provision_dto import ProvisionDTO -from app.schemas.provision_base_schema import ProvisionSchema +from app.schemas.provision_base_schema import ( + MultiProvisionRequestSchema, + MultiProvisionSchema, + ProvisionSchema, + ProvisionSummarySchema, + ServiceProvisionResultSchema, +) LIVING_BUILDINGS_ID = 4 @@ -23,36 +30,24 @@ class ProvisionService: def __init__(self, gateway: EffectsAPIGateway) -> None: self.gateway = gateway - async def calculate_provision( - self, provision_params: ProvisionDTO, token: str - ) -> ProvisionSchema: + async def _fetch_shared_data( + self, + project_id: int, + scenario_id: int, + token: str, + ) -> dict: """ - Calculate provision effects by project data and target scenario + Fetch scenario data which does not depend on service type Args: - provision_params (ProvisionDTO): Project data + project_id (int): Project ID + scenario_id (int): Target scenario ID token (str): Authorization token Returns: - gpd.GeoDataFrame: Provision for scenario. + dict: project data, context and target scenario buildings with populations """ - logger.info( - f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" - ) - project_data = await self.gateway.get_project_data( - provision_params.project_id, token - ) - project_territory = await self.gateway.get_project_territory( - provision_params.project_id, token - ) - service_default_capacity = await self.gateway.get_default_capacity( - service_type_id=provision_params.service_type_id - ) - normative_data = await self.gateway.get_service_normative( - territory_id=project_data["territory"]["id"], - context_ids=project_data["properties"]["context"], - service_type_id=provision_params.service_type_id, - token=token, - ) + project_data = await self.gateway.get_project_data(project_id, token) + project_territory = await self.gateway.get_project_territory(project_id, token) context_population = await self.gateway.get_context_population( territory_ids_list=project_data["properties"]["context"], token=token ) @@ -65,17 +60,62 @@ async def calculate_provision( context_buildings = await attribute_parser.parse_all_from_buildings( living_buildings=context_buildings, ) + target_scenario_population = await self.gateway.get_scenario_population_data( + scenario_id=scenario_id, token=token + ) + target_scenario_buildings = await self.gateway.get_scenario_buildings( + scenario_id=scenario_id, token=token + ) + target_scenario_buildings = await attribute_parser.parse_all_from_buildings( + living_buildings=target_scenario_buildings, + ) + return { + "project_data": project_data, + "context_population": context_population, + "context_buildings": context_buildings, + "target_scenario_population": target_scenario_population, + "target_scenario_buildings": target_scenario_buildings, + } + + async def _calculate_for_service( + self, + shared_data: dict, + scenario_id: int, + service_type_id: int, + token: str, + ) -> dict[str, gpd.GeoDataFrame]: + """ + Calculate provision for one service type over prefetched scenario data + Args: + shared_data (dict): data prefetched by _fetch_shared_data + scenario_id (int): Target scenario ID + service_type_id (int): Service type ID + token (str): Authorization token + Returns: + dict[str, gpd.GeoDataFrame]: dict with fields "buildings", "services" and "links" + """ + + project_data = shared_data["project_data"] + service_default_capacity = await self.gateway.get_default_capacity( + service_type_id=service_type_id + ) + normative_data = await self.gateway.get_service_normative( + territory_id=project_data["territory"]["id"], + context_ids=project_data["properties"]["context"], + service_type_id=service_type_id, + token=token, + ) context_buildings = await asyncio.to_thread( data_restorator.restore_demands, - buildings=context_buildings, + buildings=shared_data["context_buildings"].copy(), service_normative=normative_data["services_capacity_per_1000_normative"], service_normative_type=normative_data["capacity_type"], - target_population=context_population, + target_population=shared_data["context_population"], ) context_buildings["is_project"] = False context_services = await self.gateway.get_project_context_services( scenario_id=project_data["base_scenario"]["id"], - service_type_id=provision_params.service_type_id, + service_type_id=service_type_id, token=token, ) if context_services.empty: @@ -83,32 +123,23 @@ async def calculate_provision( raise http_exception( status_code=404, msg="No services of {service_type_id} type found in context", - _input={"service_type_id": provision_params.service_type_id}, + _input={"service_type_id": service_type_id}, _detail={}, ) context_services = await attribute_parser.parse_all_from_services( services=context_services, service_default_capacity=service_default_capacity ) - target_scenario_population = await self.gateway.get_scenario_population_data( - scenario_id=provision_params.scenario_id, token=token - ) - target_scenario_buildings = await self.gateway.get_scenario_buildings( - scenario_id=provision_params.scenario_id, token=token - ) - target_scenario_buildings = await attribute_parser.parse_all_from_buildings( - living_buildings=target_scenario_buildings, - ) target_scenario_buildings = await asyncio.to_thread( data_restorator.restore_demands, - buildings=target_scenario_buildings, + buildings=shared_data["target_scenario_buildings"].copy(), service_normative=normative_data["services_capacity_per_1000_normative"], service_normative_type=normative_data["capacity_type"], - target_population=target_scenario_population, + target_population=shared_data["target_scenario_population"], ) target_scenario_buildings["is_project"] = True target_scenario_services = await self.gateway.get_scenario_services( - scenario_id=provision_params.scenario_id, - service_type_id=provision_params.service_type_id, + scenario_id=scenario_id, + service_type_id=service_type_id, token=token, ) target_scenario_services = await attribute_parser.parse_all_from_services( @@ -153,8 +184,143 @@ async def calculate_provision( matrix=before_matrix, service_normative=normative_data["normative_value"], ) + return before_prove_data + + @staticmethod + def _build_summary( + buildings: gpd.GeoDataFrame, + services: gpd.GeoDataFrame, + ) -> ProvisionSummarySchema: + """ + Aggregate provision results into summary statistics + Args: + buildings (gpd.GeoDataFrame): buildings layer with provision attributes + services (gpd.GeoDataFrame): services layer with load attributes + Returns: + ProvisionSummarySchema: aggregated provision statistics + """ + + provision_values = buildings["provision_value"].dropna() + total_capacity = int(services["capacity"].sum()) + total_demand = int(buildings["demand"].sum()) + balance = total_capacity - total_demand + return ProvisionSummarySchema( + services_count=int(len(services)), + total_capacity=total_capacity, + total_demand=total_demand, + satisfied_demand_within=int(buildings["supplied_demands_within"].sum()), + satisfied_demand_without=int(buildings["supplied_demands_without"].sum()), + unsatisfied_demand=int(buildings["demand_left"].sum()), + balance=balance, + deficit=max(0, -balance), + surplus=max(0, balance), + average_provision_value=( + round(float(provision_values.mean()), 3) + if not provision_values.empty + else None + ), + median_provision_value=( + round(float(provision_values.median()), 3) + if not provision_values.empty + else None + ), + ) + + async def calculate_provision( + self, provision_params: ProvisionDTO, token: str + ) -> ProvisionSchema: + """ + Calculate provision effects by project data and target scenario + Args: + provision_params (ProvisionDTO): Project data + token (str): Authorization token + Returns: + gpd.GeoDataFrame: Provision for scenario. + """ + + logger.info( + f"Started calculating effects for {provision_params.scenario_id} and service{provision_params.service_type_id}" + ) + shared_data = await self._fetch_shared_data( + project_id=provision_params.project_id, + scenario_id=provision_params.scenario_id, + token=token, + ) + if provision_params.target_population: + shared_data["target_scenario_population"] = ( + provision_params.target_population + ) + before_prove_data = await self._calculate_for_service( + shared_data=shared_data, + scenario_id=provision_params.scenario_id, + service_type_id=provision_params.service_type_id, + token=token, + ) result = {k: json.loads(v.to_json()) for k, v in before_prove_data.items()} logger.info( f"Calculated PROVISION for {provision_params.scenario_id} and {provision_params.service_type_id}" ) return ProvisionSchema(**result) + + async def calculate_multi_provision( + self, multi_params: MultiProvisionRequestSchema, token: str + ) -> MultiProvisionSchema: + """ + Calculate provision for several service types over one scenario + Args: + multi_params (MultiProvisionRequestSchema): project, scenario and services to calculate + token (str): Authorization token + Returns: + MultiProvisionSchema: per-service summaries with optional GeoJSON layers + """ + + logger.info( + f"Started calculating multi provision for {multi_params.scenario_id} " + f"and services {list(multi_params.services)}" + ) + project_id = await self.gateway.get_project_id_by_scenario( + multi_params.scenario_id, token + ) + shared_data = await self._fetch_shared_data( + project_id=project_id, + scenario_id=multi_params.scenario_id, + token=token, + ) + if multi_params.target_population: + shared_data["target_scenario_population"] = multi_params.target_population + results = {} + for service_type_id, service_info in multi_params.services.items(): + try: + before_prove_data = await self._calculate_for_service( + shared_data=shared_data, + scenario_id=multi_params.scenario_id, + service_type_id=service_type_id, + token=token, + ) + except Exception as e: + logger.opt(exception=True).error( + f"Provision calculation failed for service type {service_type_id}: {e}" + ) + results[service_type_id] = ServiceProvisionResultSchema( + name=service_info.name, + error=f"{type(e).__name__}: {e}", + ) + continue + layers = None + if service_info.as_layer: + layers = ProvisionSchema( + **{ + k: json.loads(v.to_crs(4326).to_json()) + for k, v in before_prove_data.items() + } + ) + results[service_type_id] = ServiceProvisionResultSchema( + name=service_info.name, + summary=self._build_summary( + buildings=before_prove_data["buildings"], + services=before_prove_data["services"], + ), + layers=layers, + ) + logger.info(f"Calculated MULTI PROVISION for {multi_params.scenario_id}") + return MultiProvisionSchema(services=results) diff --git a/app/schemas/provision_base_schema.py b/app/schemas/provision_base_schema.py index db2f88c..b75ba0a 100644 --- a/app/schemas/provision_base_schema.py +++ b/app/schemas/provision_base_schema.py @@ -1,6 +1,6 @@ from typing import Any, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field class GeometrySchema(BaseModel): @@ -35,3 +35,64 @@ class ProvisionSchema(BaseModel): buildings: FeatureCollectionSchema services: FeatureCollectionSchema links: FeatureCollectionSchema + + +class ServiceInfoSchema(BaseModel): + + name: str = Field(..., examples=["Школа"], description="Service display name") + as_layer: bool = Field( + default=True, + description="If true, response includes GeoJSON layers for the service", + ) + + +class MultiProvisionRequestSchema(BaseModel): + + scenario_id: int = Field(..., examples=[192], description="Scenario ID") + services: dict[int, ServiceInfoSchema] = Field( + ..., + examples=[{22: {"name": "Школа", "as_layer": True}}], + description="Service type IDs to calculate provision for", + ) + target_population: int | None = Field( + default=None, + examples=[25000], + description=( + "Total population of the scenario territory for demand calculation. " + "If not provided, population is restored from Urban API data." + ), + ) + + +class ProvisionSummarySchema(BaseModel): + + services_count: int + total_capacity: int + total_demand: int + satisfied_demand_within: int + satisfied_demand_without: int + unsatisfied_demand: int + balance: int = Field( + description="Capacity minus demand; negative means shortage of places" + ) + deficit: int = Field( + ge=0, description="Places short of demand, 0 when capacity covers demand" + ) + surplus: int = Field( + ge=0, description="Places above demand, 0 when demand exceeds capacity" + ) + average_provision_value: float | None + median_provision_value: float | None + + +class ServiceProvisionResultSchema(BaseModel): + + name: str + summary: ProvisionSummarySchema | None = None + layers: ProvisionSchema | None = None + error: str | None = None + + +class MultiProvisionSchema(BaseModel): + + services: dict[int, ServiceProvisionResultSchema] diff --git a/requirements.txt b/requirements.txt index 2f5a88685119fbb8a019a0b2554beb3ee2373510..bc7ce14ba3f2912ef4c62fe8c9e83bc56d51e1af 100644 GIT binary patch delta 61 zcmaFBK8Jlm9j76K9)kf88%|uWDOkWz$xz2&%U}c&0Al0GoQ%?7B_<$|&AN=6nOJxk GxEKJEBMSro delta 126 zcmbQk{(yZ#ouL7P9)lr+36L~l;AP-q@MLgd2xZV^NM^`q$YV%j$Ye-osAI4Ns?lQr zsWqC|u4$OWkO&q837CO}4Zxc77)lv(844IG;aZG<@<5{}9+c(;>jr7q_;fQ909@A= AFaQ7m From eda8fe72db6a76cf23e2d3a055f0f8f3fe723178 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:15:51 +0300 Subject: [PATCH 52/61] chore(deps): update dependencies within minor/patch (#41) --- requirements.txt | Bin 924 -> 924 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc7ce14ba3f2912ef4c62fe8c9e83bc56d51e1af..eb2fd96daff86fd3ca02b89851e693bc470cef5c 100644 GIT binary patch delta 289 zcmYk2y-veG5QOK>vMfbPkAhAXa-aC$m7f44I$9JGV^I(%HfbYK(Z<>`;bC|P-h_rK z*;^8nZcg*<&d%+s7TqF)`Tl#Fw&BHC#~56;zC7%9+lffZj+NzLw* K&Oaf^u`ov{>orXP delta 293 zcmYk1y^6w65QWdZ5(2{Z{t$ws&nCHMWecA`!Om7(b%TY*-*&;u)-bJ$&*DS)CN@?& zcTljK!+dAvoTGbm4>iXiM6>X{j=>j?5k64HFNRoSgBeCx;16wBEU~EUGfa^4W{5e) z!dfO4e~7r)r;Zh-IJZ?&+&z_&r~IJF_ScIHTQ*PmL5>wq_CNwd1J!a3zL|^ZUwyvK z4Z5nxiSQ$vfV%pncDm=-%Q0QF&jkOVn+{Y$Ua2m+MJHXQeyI`)CnZ)~V#z5cQg!`I Kz^SEZD$E1tAT^)> From 2378dc2c71506f1eac45daa6092e9e108c20386b Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:08:55 +0300 Subject: [PATCH 53/61] feat(auth): require service tokens (#42) --- .env.example | 4 + .github/workflows/build_and_deploy.yml | 5 ++ Dockerfile | 3 +- app/common/api_handler/api_handler.py | 12 +++ app/common/auth/service_auth.py | 98 +++++++++++++++++++++++ app/common/modules/effects_api_gateway.py | 27 ++++--- app/dependencies.py | 10 ++- app/effects/effects_controller.py | 6 +- app/effects/effects_mcp.py | 8 +- app/main.py | 11 ++- app/provision/provision_controller.py | 10 +-- app/provision/provision_mcp.py | 10 +-- docker-compose.actions.yml | 6 +- requirements-auth.txt | 1 + tests/test_service_auth_transport.py | 22 +++++ 15 files changed, 195 insertions(+), 38 deletions(-) create mode 100644 app/common/auth/service_auth.py create mode 100644 requirements-auth.txt create mode 100644 tests/test_service_auth_transport.py diff --git a/.env.example b/.env.example index 98456e2..51a7ade 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,7 @@ LOGS_FILE="object_effects" URBAN_API="https://urban-api.testing" MCP_URBAN_API="https://urban-api.testing" PROMETHEUS_PORT=9464 +SERVICE_AUTH_SERVER_URL=https://keycloak.example.com +SERVICE_AUTH_REALM=IDU +SERVICE_AUTH_CLIENT_ID=object-effects +SERVICE_AUTH_CLIENT_SECRET=change-me diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index b2815e3..2a1781a 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -48,5 +48,10 @@ jobs: ENV_PATH: ${{secrets.ENV_PATH}} run: cp "$ENV_PATH"/.env.development ./ - name: run + env: + SERVICE_AUTH_SERVER_URL: ${{ vars.SERVICE_AUTH_SERVER_URL }} + SERVICE_AUTH_REALM: ${{ vars.SERVICE_AUTH_REALM }} + SERVICE_AUTH_CLIENT_ID: ${{ vars.SERVICE_AUTH_CLIENT_ID }} + SERVICE_AUTH_CLIENT_SECRET: ${{ secrets.SERVICE_AUTH_CLIENT_SECRET }} # run: docker run -d --name "$CONTAINER_NAME" --env-file ./.env.development -p 8210:8000 "$IMAGE_NAME":"$NOW" run: docker compose -f docker-compose.actions.yml up -d diff --git a/Dockerfile b/Dockerfile index 2a8fb14..1a574f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,8 @@ ENV APP_ENV=development COPY pip.conf /etc/xdg/pip/pip.conf # Install pip requirements COPY requirements.txt . -RUN python -m pip install -r requirements.txt +COPY requirements-auth.txt . +RUN python -m pip install -r requirements.txt -r requirements-auth.txt WORKDIR /app COPY . /app diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index b83ef1f..8f0c0be 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -1,4 +1,5 @@ import aiohttp +from idu_service_auth import KeycloakTokenClient from app.common.exceptions.http_exception_wrapper import http_exception @@ -8,6 +9,7 @@ class APIHandler: def __init__( self, base_url: str, + service_auth: KeycloakTokenClient, ) -> None: """Initialisation function @@ -18,6 +20,12 @@ def __init__( """ self.base_url = base_url + self.service_auth = service_auth + + async def _service_headers(self, headers: dict | None) -> dict[str, str]: + outgoing = dict(headers or {}) + outgoing.update(await self.service_auth.get_authorization_headers()) + return outgoing @staticmethod async def _check_response_status( @@ -89,6 +97,7 @@ async def get( params=params, session=session, ) + headers = await self._service_headers(headers) url = self.base_url + endpoint_url async with session.get(url=url, headers=headers, params=params) as response: result = await self._check_response_status(response) @@ -138,6 +147,7 @@ async def post( data=data, session=session, ) + headers = await self._service_headers(headers) url = self.base_url + endpoint_url async with session.post( url=url, @@ -184,6 +194,7 @@ async def put( data=data, session=session, ) + headers = await self._service_headers(headers) url = self.base_url + endpoint_url async with session.put( url=url, @@ -230,6 +241,7 @@ async def delete( data=data, session=session, ) + headers = await self._service_headers(headers) url = self.base_url + endpoint_url async with session.delete( url=url, diff --git a/app/common/auth/service_auth.py b/app/common/auth/service_auth.py new file mode 100644 index 0000000..9ec342b --- /dev/null +++ b/app/common/auth/service_auth.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from fastapi import Header, HTTPException, Security, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from fastmcp.exceptions import AuthorizationError, ToolError +from fastmcp.server.auth import AccessToken +from fastmcp.server.auth.providers.jwt import JWTVerifier +from fastmcp.server.dependencies import get_http_headers +from idu_service_auth import KeycloakTokenClient, KeycloakTokenConfig + +from app.common.config.config import Config + +USER_ID_HEADER = "X-User-Id" +SERVICE_ACCOUNT_PREFIX = "service-account-" +bearer_scheme = HTTPBearer(auto_error=True) + + +def build_service_auth(config: Config) -> KeycloakTokenClient: + return KeycloakTokenClient( + KeycloakTokenConfig( + auth_server_url=config.get("SERVICE_AUTH_SERVER_URL"), + realm=config.get("SERVICE_AUTH_REALM"), + client_id=config.get("SERVICE_AUTH_CLIENT_ID"), + client_secret=config.get("SERVICE_AUTH_CLIENT_SECRET"), + background_refresh=True, + ) + ) + + +def build_service_token_verifier(config: Config) -> "ServiceTokenVerifier": + return ServiceTokenVerifier(config) + + +async def get_current_user_id( + _credentials: HTTPAuthorizationCredentials = Security(bearer_scheme), + x_user_id: str | None = Header(default=None, alias=USER_ID_HEADER), +) -> str: + if not x_user_id or not x_user_id.strip(): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"{USER_ID_HEADER} header is required", + ) + return x_user_id.strip() + + +async def require_service_token( + credentials: HTTPAuthorizationCredentials = Security(bearer_scheme), +) -> None: + """Require a verified Keycloak service-account token.""" + + from app.dependencies import service_token_verifier + + try: + access_token = await service_token_verifier.verify_token( + credentials.credentials + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid service token", + ) from exc + if access_token is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid service token", + ) + + +def get_mcp_user_id() -> str: + user_id = get_http_headers(include_all=True).get("x-user-id", "").strip() + if not user_id: + raise ToolError(f"{USER_ID_HEADER} header is required") + return user_id + + +class ServiceTokenVerifier(JWTVerifier): + """Verify Keycloak JWTs and accept only client-credentials accounts.""" + + def __init__(self, config: Config) -> None: + server_url = config.get("SERVICE_AUTH_SERVER_URL").rstrip("/") + realm = config.get("SERVICE_AUTH_REALM") + issuer = f"{server_url}/realms/{realm}" + super().__init__( + jwks_uri=f"{issuer}/protocol/openid-connect/certs", + issuer=issuer, + algorithm="RS256", + ) + + async def verify_token(self, token: str) -> AccessToken | None: + access_token = await super().verify_token(token) + if access_token is None: + return None + username = access_token.claims.get("preferred_username", "") + if not isinstance(username, str) or not username.startswith( + SERVICE_ACCOUNT_PREFIX + ): + raise AuthorizationError("A service-account token is required") + return access_token diff --git a/app/common/modules/effects_api_gateway.py b/app/common/modules/effects_api_gateway.py index fa9204a..52b7f34 100644 --- a/app/common/modules/effects_api_gateway.py +++ b/app/common/modules/effects_api_gateway.py @@ -5,6 +5,7 @@ from shapely.geometry import shape from app.common.api_handler.api_handler import APIHandler +from app.common.auth.service_auth import USER_ID_HEADER from app.common.exceptions.http_exception_wrapper import http_exception @@ -27,7 +28,7 @@ async def get_project_id_by_scenario(self, scenario_id: int, token: str) -> int: proj_resp = await self.api_handler.get( f"/api/v1/scenarios/{scenario_id}", - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) return proj_resp["project"]["project_id"] @@ -54,13 +55,13 @@ async def get_service_normative( if len(context_ids) == 1: response = await self.api_handler.get( f"/api/v1/territory/{context_ids[0]}/normatives", - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) request_ter_id = context_ids[0] else: response = await self.api_handler.get( f"/api/v1/territory/{territory_id}/normatives", - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) request_ter_id = territory_id response_df = pd.DataFrame.from_records(response) @@ -153,7 +154,7 @@ async def get_project_data( response = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}", - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) return response @@ -173,7 +174,7 @@ async def get_scenario_buildings( buildings = await self.api_handler.get( endpoint_url=f"/api/v1/scenarios/{scenario_id}/geometries_with_all_objects", params={"physical_object_type_id": 4}, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) buildings_gdf = gpd.GeoDataFrame.from_features(buildings) if buildings_gdf.empty: @@ -200,7 +201,7 @@ async def get_project_context_buildings( params={ "physical_object_type_id": 4, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) context_buildings_gdf = gpd.GeoDataFrame.from_features(context_buildings) if context_buildings_gdf.empty: @@ -226,7 +227,7 @@ async def get_scenario_services( params={ "service_type_id": service_type_id, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) services_gdf = gpd.GeoDataFrame.from_features(services) if services_gdf.empty: @@ -255,7 +256,7 @@ async def get_project_context_services( params={ "service_type_id": service_type_id, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) context_services_gdf = gpd.GeoDataFrame.from_features(context_services) if context_services_gdf.empty: @@ -280,7 +281,7 @@ async def get_scenario_population_data( params={ "indicator_ids": 1, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) if len(population) < 1 or (value := population[0]["value"]) < 1: @@ -303,7 +304,7 @@ async def get_context_population( self.api_handler.get( endpoint_url=f"/api/v1/territory/{territory_id}/indicator_values", params={"indicator_ids": 1}, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) for territory_id in territory_ids_list ] @@ -325,7 +326,7 @@ async def get_project_territory( territory = await self.api_handler.get( endpoint_url=f"/api/v1/projects/{project_id}/territory", - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) territory_gdf = gpd.GeoDataFrame( geometry=[shape(territory["geometry"])], crs=4326 @@ -366,7 +367,7 @@ async def get_services_with_context( "service_type_id": service_type_id, "include_scenario_objects": True, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) return gpd.GeoDataFrame.from_features(services, crs=4326) @@ -389,6 +390,6 @@ async def get_physical_objects_with_context( "physical_object_type_id": physical_object_type_id, "include_scenario_objects": True, }, - headers={"Authorization": f"Bearer {token}"} if token else None, + headers={USER_ID_HEADER: token} if token else None, ) return gpd.GeoDataFrame.from_features(physical_objects, crs=4326) diff --git a/app/dependencies.py b/app/dependencies.py index 813e2f1..876acd2 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -3,6 +3,10 @@ from loguru import logger from app.common.api_handler.api_handler import APIHandler +from app.common.auth.service_auth import ( + build_service_auth, + build_service_token_verifier, +) from app.common.config.config import Config from app.common.exceptions.http_exception_wrapper import http_exception from app.common.modules.effects_api_gateway import EffectsAPIGateway @@ -16,6 +20,8 @@ logger.add(sys.stderr, format=log_format, level=log_level, colorize=True) config = Config() +service_auth = build_service_auth(config) +service_token_verifier = build_service_token_verifier(config) logger.add( ".log", @@ -23,8 +29,8 @@ level="INFO", ) -urban_api_handler = APIHandler(config.get("URBAN_API")) -urban_api_mcp_handler = APIHandler(config.get("MCP_URBAN_API")) +urban_api_handler = APIHandler(config.get("URBAN_API"), service_auth) +urban_api_mcp_handler = APIHandler(config.get("MCP_URBAN_API"), service_auth) effects_api_gateway = EffectsAPIGateway(urban_api_handler) effects_api_mcp_gateway = EffectsAPIGateway(urban_api_mcp_handler) diff --git a/app/effects/effects_controller.py b/app/effects/effects_controller.py index fdfc51e..bdc7d90 100644 --- a/app/effects/effects_controller.py +++ b/app/effects/effects_controller.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends -from app.common.auth.bearer import verify_bearer_token +from app.common.auth.service_auth import get_current_user_id from app.dependencies import effects_service from app.dto.provision_dto import ProvisionDTO @@ -14,7 +14,7 @@ @effects_router.get("/evaluate_provision", response_model=EffectsSchema) async def calculate_effects( params: Annotated[ProvisionDTO, Depends(ProvisionDTO)], - token: str = Depends(verify_bearer_token), + user_id: str = Depends(get_current_user_id), ) -> EffectsSchema: - return await effects_service.calculate_effects(params, token) + return await effects_service.calculate_effects(params, user_id) diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index af7aa49..eb57a0d 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -1,13 +1,13 @@ import traceback from fastmcp import FastMCP -from fastmcp.server.dependencies import get_access_token from loguru import logger -from app.dependencies import effects_mcp_service +from app.common.auth.service_auth import get_mcp_user_id +from app.dependencies import effects_mcp_service, service_token_verifier from app.dto.provision_dto import ProvisionDTO -effects_mcp = FastMCP("Object Effects MCP server") +effects_mcp = FastMCP("Object Effects MCP server", auth=service_token_verifier) @effects_mcp.tool( @@ -56,7 +56,7 @@ async def calc_provision_effects( ): try: - token = get_access_token() + token = get_mcp_user_id() project_id = await effects_mcp_service.gateway.get_project_id_by_scenario( scenario_id, token ) diff --git a/app/main.py b/app/main.py index a0ace85..d519550 100644 --- a/app/main.py +++ b/app/main.py @@ -1,15 +1,16 @@ from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, RedirectResponse from fastmcp.utilities.lifespan import combine_lifespans from loguru import logger from .__version__ import APP_VERSION +from .common.auth.service_auth import require_service_token from .common.middlewares.exception_handler import ExceptionHandlerMiddleware from .common.middlewares.prometheus_handler import ObservabilityMiddleware -from .dependencies import config, http_exception +from .dependencies import config, http_exception, service_auth from .effects.effects_controller import effects_router from .mcp import effects_mcp_app, provision_mcp_app from .observability import OpenTelemetryAgent, PrometheusConfig @@ -37,7 +38,9 @@ async def lifespan(app: FastAPI): ) setup_metrics() logger.info(f"Prometheus server started on {config.get('PROMETHEUS_PORT')}") - yield + async with service_auth: + await service_auth.get_access_token() + yield otel_agent.shutdown() logger.info("Prometheus server was shut down") @@ -79,7 +82,7 @@ async def read_root(): return {"status": "OK"} -@app.get("/logs") +@app.get("/logs", dependencies=[Depends(require_service_token)]) async def get_logs(): """ Get logs file from app diff --git a/app/provision/provision_controller.py b/app/provision/provision_controller.py index 03826a6..bc7bb47 100644 --- a/app/provision/provision_controller.py +++ b/app/provision/provision_controller.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, Depends -from app.common.auth.bearer import verify_bearer_token +from app.common.auth.service_auth import get_current_user_id from app.dependencies import provision_service from app.dto.provision_dto import ProvisionDTO from app.schemas.provision_base_schema import ( @@ -17,18 +17,18 @@ @provision_router.get("/calc_provision", response_model=ProvisionSchema) async def calculate_provision( provision_dto: Annotated[ProvisionDTO, Depends(ProvisionDTO)], - token: str = Depends(verify_bearer_token), + user_id: str = Depends(get_current_user_id), ) -> ProvisionSchema: - return await provision_service.calculate_provision(provision_dto, token) + return await provision_service.calculate_provision(provision_dto, user_id) @provision_router.post("/calc_provisions", response_model=MultiProvisionSchema) async def calculate_multi_provision( multi_provision_params: MultiProvisionRequestSchema, - token: str = Depends(verify_bearer_token), + user_id: str = Depends(get_current_user_id), ) -> MultiProvisionSchema: return await provision_service.calculate_multi_provision( - multi_provision_params, token + multi_provision_params, user_id ) diff --git a/app/provision/provision_mcp.py b/app/provision/provision_mcp.py index 19dabd2..246c98a 100644 --- a/app/provision/provision_mcp.py +++ b/app/provision/provision_mcp.py @@ -1,17 +1,17 @@ import traceback from fastmcp import FastMCP -from fastmcp.server.dependencies import get_access_token from loguru import logger -from app.dependencies import provision_mcp_service +from app.common.auth.service_auth import get_mcp_user_id +from app.dependencies import provision_mcp_service, service_token_verifier from app.dto.provision_dto import ProvisionDTO from app.schemas.provision_base_schema import ( MultiProvisionRequestSchema, ServiceInfoSchema, ) -provision_mcp = FastMCP("Object Provision MCP server") +provision_mcp = FastMCP("Object Provision MCP server", auth=service_token_verifier) @provision_mcp.tool( @@ -42,7 +42,7 @@ async def calc_service_provision( ): try: - token = get_access_token() + token = get_mcp_user_id() project_id = await provision_mcp_service.gateway.get_project_id_by_scenario( scenario_id, token ) @@ -115,7 +115,7 @@ async def calc_services_provision( ): try: - token = get_access_token() + token = get_mcp_user_id() multi_provision_params = MultiProvisionRequestSchema( scenario_id=scenario_id, services=services, diff --git a/docker-compose.actions.yml b/docker-compose.actions.yml index f11bb9a..eb786bd 100644 --- a/docker-compose.actions.yml +++ b/docker-compose.actions.yml @@ -7,5 +7,9 @@ services: - "9464:9464" env_file: - .env.development + environment: + SERVICE_AUTH_SERVER_URL: ${SERVICE_AUTH_SERVER_URL:?SERVICE_AUTH_SERVER_URL is required} + SERVICE_AUTH_REALM: ${SERVICE_AUTH_REALM:?SERVICE_AUTH_REALM is required} + SERVICE_AUTH_CLIENT_ID: ${SERVICE_AUTH_CLIENT_ID:?SERVICE_AUTH_CLIENT_ID is required} + SERVICE_AUTH_CLIENT_SECRET: ${SERVICE_AUTH_CLIENT_SECRET:?SERVICE_AUTH_CLIENT_SECRET is required} restart: always - diff --git a/requirements-auth.txt b/requirements-auth.txt new file mode 100644 index 0000000..cd5e3cb --- /dev/null +++ b/requirements-auth.txt @@ -0,0 +1 @@ +idu-service-auth @ https://github.com/IDUclub/idu-service-auth/archive/1b8a418d9b1ab702860eb7289a3b75244666a8d8.tar.gz diff --git a/tests/test_service_auth_transport.py b/tests/test_service_auth_transport.py new file mode 100644 index 0000000..9fb0f0e --- /dev/null +++ b/tests/test_service_auth_transport.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import pytest + +from app.common.api_handler.api_handler import APIHandler + + +class FakeServiceAuth: + async def get_authorization_headers(self): + return {"Authorization": "Bearer service-token"} + + +@pytest.mark.asyncio +async def test_service_token_cannot_be_overridden_by_request_headers(): + handler = APIHandler("http://urban", FakeServiceAuth()) + + headers = await handler._service_headers( + {"Authorization": "Bearer caller-token", "X-User-Id": "u1"} + ) + + assert headers["Authorization"] == "Bearer service-token" + assert headers["X-User-Id"] == "u1" From 2c76c508d7812e9283910dce464770109747f2ef Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:46:20 +0300 Subject: [PATCH 54/61] Merge pull request #44 from IDUclub/fix/deploy-env-scope fix(ci): scope the deploy job to the production environment --- .github/workflows/build_and_deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 2a1781a..cf791a1 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -36,6 +36,9 @@ jobs: run_container: runs-on: 65_runner needs: [build, stop_container] + # Required so the SERVICE_AUTH_* vars/secrets of the "production" environment resolve: + # without it they come out empty and docker-compose.actions.yml aborts on ${VAR:?...}. + environment: production env: NOW: ${{needs.build.outputs.now}} steps: From c1d3a5f716725ff996d4b508f7675fcf333afc25 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:11:50 +0300 Subject: [PATCH 55/61] Merge pull request #46 from IDUclub/codex/fix-m2m-user-context fix: separate MCP user context from service auth --- app/common/api_handler/api_handler.py | 1 + app/effects/effects_mcp.py | 6 +++--- app/provision/provision_mcp.py | 12 +++++++----- tests/test_service_auth_transport.py | 24 ++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index 8f0c0be..125c75c 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -23,6 +23,7 @@ def __init__( self.service_auth = service_auth async def _service_headers(self, headers: dict | None) -> dict[str, str]: + """Preserve request context while always replacing caller auth with M2M.""" outgoing = dict(headers or {}) outgoing.update(await self.service_auth.get_authorization_headers()) return outgoing diff --git a/app/effects/effects_mcp.py b/app/effects/effects_mcp.py index eb57a0d..5eab780 100644 --- a/app/effects/effects_mcp.py +++ b/app/effects/effects_mcp.py @@ -56,9 +56,9 @@ async def calc_provision_effects( ): try: - token = get_mcp_user_id() + user_id = get_mcp_user_id() project_id = await effects_mcp_service.gateway.get_project_id_by_scenario( - scenario_id, token + scenario_id, user_id ) effects_dto = ProvisionDTO( project_id=project_id, @@ -67,7 +67,7 @@ async def calc_provision_effects( target_population=target_population, ) result = await effects_mcp_service.calculate_effects( - effects_dto, token, for_mcp=True + effects_dto, user_id, for_mcp=True ) return result except Exception as e: diff --git a/app/provision/provision_mcp.py b/app/provision/provision_mcp.py index 246c98a..43ff8c4 100644 --- a/app/provision/provision_mcp.py +++ b/app/provision/provision_mcp.py @@ -42,9 +42,9 @@ async def calc_service_provision( ): try: - token = get_mcp_user_id() + user_id = get_mcp_user_id() project_id = await provision_mcp_service.gateway.get_project_id_by_scenario( - scenario_id, token + scenario_id, user_id ) provision_dto = ProvisionDTO( project_id=project_id, @@ -52,7 +52,9 @@ async def calc_service_provision( service_type_id=service_type_id, target_population=target_population, ) - result = await provision_mcp_service.calculate_provision(provision_dto, token) + result = await provision_mcp_service.calculate_provision( + provision_dto, user_id + ) return result.model_dump() except Exception as e: tb = traceback.format_exc() @@ -115,14 +117,14 @@ async def calc_services_provision( ): try: - token = get_mcp_user_id() + user_id = get_mcp_user_id() multi_provision_params = MultiProvisionRequestSchema( scenario_id=scenario_id, services=services, target_population=target_population, ) result = await provision_mcp_service.calculate_multi_provision( - multi_provision_params, token + multi_provision_params, user_id ) return result.model_dump() except Exception as e: diff --git a/tests/test_service_auth_transport.py b/tests/test_service_auth_transport.py index 9fb0f0e..564d174 100644 --- a/tests/test_service_auth_transport.py +++ b/tests/test_service_auth_transport.py @@ -1,8 +1,12 @@ from __future__ import annotations +from unittest.mock import patch + import pytest +from fastmcp.exceptions import ToolError from app.common.api_handler.api_handler import APIHandler +from app.common.auth.service_auth import get_mcp_user_id class FakeServiceAuth: @@ -20,3 +24,23 @@ async def test_service_token_cannot_be_overridden_by_request_headers(): assert headers["Authorization"] == "Bearer service-token" assert headers["X-User-Id"] == "u1" + + +def test_mcp_user_context_comes_from_x_user_id_not_authorization(): + with patch( + "app.common.auth.service_auth.get_http_headers", + return_value={ + "authorization": "Bearer service-token", + "x-user-id": " user-42 ", + }, + ): + assert get_mcp_user_id() == "user-42" + + +def test_mcp_user_context_is_required(): + with patch( + "app.common.auth.service_auth.get_http_headers", + return_value={"authorization": "Bearer service-token"}, + ): + with pytest.raises(ToolError, match="X-User-Id header is required"): + get_mcp_user_id() From ea53b8254fff0bbbf184556ad3dea6165fb00deb Mon Sep 17 00:00:00 2001 From: ruslan Date: Mon, 7 Sep 2026 17:27:01 +0300 Subject: [PATCH 56/61] - feat: ci dev pipeline --- .github/workflows/ci-dev.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/ci-dev.yml diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml new file mode 100644 index 0000000..fd1b9d1 --- /dev/null +++ b/.github/workflows/ci-dev.yml @@ -0,0 +1,26 @@ +name: Kubernetes dev release + +on: + push: + branches: + - dev + +permissions: + contents: read + +jobs: + release: + uses: IDUclub/urban-assistant-deploy/.github/workflows/reusable-application-release.yaml@main + with: + service: object-effects + test-command: | + python -m pip install --disable-pip-version-check uv==0.12.10 + uv python install 3.11 + uv venv --python 3.11 + uv pip install --python .venv/bin/python \ + --requirement requirements.txt \ + --requirement requirements-auth.txt \ + pytest \ + pytest-asyncio + .venv/bin/python -m pytest --verbose tests + secrets: inherit From bb697ec84bd0556a67583dfa4c3c2ea4df85238f Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:33:05 +0300 Subject: [PATCH 57/61] fix: allow unauthenticated log downloads (#50) --- README.md | 4 ++++ app/main.py | 5 ++--- tests/test_public_diagnostics.py | 12 ++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 tests/test_public_diagnostics.py diff --git a/README.md b/README.md index ab0a4ee..8dd256b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,6 @@ # ObjectEffectsAPI Repository for evaluation effects by ObjectNat library + +### Logs + +`GET /logs` downloads the application log file without authorization. diff --git a/app/main.py b/app/main.py index d519550..1fe9e09 100644 --- a/app/main.py +++ b/app/main.py @@ -1,13 +1,12 @@ from contextlib import asynccontextmanager -from fastapi import Depends, FastAPI +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, RedirectResponse from fastmcp.utilities.lifespan import combine_lifespans from loguru import logger from .__version__ import APP_VERSION -from .common.auth.service_auth import require_service_token from .common.middlewares.exception_handler import ExceptionHandlerMiddleware from .common.middlewares.prometheus_handler import ObservabilityMiddleware from .dependencies import config, http_exception, service_auth @@ -82,7 +81,7 @@ async def read_root(): return {"status": "OK"} -@app.get("/logs", dependencies=[Depends(require_service_token)]) +@app.get("/logs") async def get_logs(): """ Get logs file from app diff --git a/tests/test_public_diagnostics.py b/tests/test_public_diagnostics.py new file mode 100644 index 0000000..7bb78e9 --- /dev/null +++ b/tests/test_public_diagnostics.py @@ -0,0 +1,12 @@ +from fastapi.testclient import TestClient + +from app.main import app + + +def test_logs_are_public(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / ".log").write_text("diagnostic log\n", encoding="utf-8") + client = TestClient(app) + response = client.get("/logs") + assert response.status_code == 200 + assert response.text == "diagnostic log\n" From 9254b42854b658068aaaa8097bfcad113cb1477d Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:18:32 +0300 Subject: [PATCH 58/61] fix: configure environment for CI tests (#51) --- .github/workflows/ci-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index fd1b9d1..96472c4 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -22,5 +22,6 @@ jobs: --requirement requirements-auth.txt \ pytest \ pytest-asyncio - .venv/bin/python -m pytest --verbose tests + cp .env.example .env.test + APP_ENV=test .venv/bin/python -m pytest --verbose tests secrets: inherit From 70dbb08a89da65d0486bdfa9dbf7c83e28534d96 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:06:23 +0300 Subject: [PATCH 59/61] fix: respect configured Urban API roots behind load balancers (#52) * fix: normalize Urban API URLs for all request methods * fix: preserve configured Urban API proxy paths --- .env.example | 1 + app/common/api_handler/api_handler.py | 15 +++-- app/common/api_handler/urban_api_url.py | 19 ++++++ app/provision/provision_mcp.py | 4 +- tests/test_urban_api_url.py | 83 +++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 app/common/api_handler/urban_api_url.py create mode 100644 tests/test_urban_api_url.py diff --git a/.env.example b/.env.example index 51a7ade..8d4185b 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ LOGS_FILE="object_effects" +# Explicit API roots (e.g. https://host/urban_api) are preserved; bare origins use /api. URBAN_API="https://urban-api.testing" MCP_URBAN_API="https://urban-api.testing" PROMETHEUS_PORT=9464 diff --git a/app/common/api_handler/api_handler.py b/app/common/api_handler/api_handler.py index 125c75c..2ebcb6d 100644 --- a/app/common/api_handler/api_handler.py +++ b/app/common/api_handler/api_handler.py @@ -1,6 +1,7 @@ import aiohttp from idu_service_auth import KeycloakTokenClient +from app.common.api_handler.urban_api_url import normalize_urban_api_url from app.common.exceptions.http_exception_wrapper import http_exception @@ -19,7 +20,7 @@ def __init__( None """ - self.base_url = base_url + self.base_url = normalize_urban_api_url(base_url) self.service_auth = service_auth async def _service_headers(self, headers: dict | None) -> dict[str, str]: @@ -99,7 +100,8 @@ async def get( session=session, ) headers = await self._service_headers(headers) - url = self.base_url + endpoint_url + endpoint = endpoint_url.lstrip("/").removeprefix("api/") + url = f"{self.base_url}/{endpoint}" async with session.get(url=url, headers=headers, params=params) as response: result = await self._check_response_status(response) if isinstance(result, list): @@ -149,7 +151,8 @@ async def post( session=session, ) headers = await self._service_headers(headers) - url = self.base_url + endpoint_url + endpoint = endpoint_url.lstrip("/").removeprefix("api/") + url = f"{self.base_url}/{endpoint}" async with session.post( url=url, headers=headers, @@ -196,7 +199,8 @@ async def put( session=session, ) headers = await self._service_headers(headers) - url = self.base_url + endpoint_url + endpoint = endpoint_url.lstrip("/").removeprefix("api/") + url = f"{self.base_url}/{endpoint}" async with session.put( url=url, headers=headers, @@ -243,7 +247,8 @@ async def delete( session=session, ) headers = await self._service_headers(headers) - url = self.base_url + endpoint_url + endpoint = endpoint_url.lstrip("/").removeprefix("api/") + url = f"{self.base_url}/{endpoint}" async with session.delete( url=url, headers=headers, diff --git a/app/common/api_handler/urban_api_url.py b/app/common/api_handler/urban_api_url.py new file mode 100644 index 0000000..cda0900 --- /dev/null +++ b/app/common/api_handler/urban_api_url.py @@ -0,0 +1,19 @@ +"""Urban API roots for direct connections and load-balancer mounts.""" + +from urllib.parse import urlsplit, urlunsplit + + +def normalize_urban_api_url(base_url: str) -> str: + """Preserve explicit API roots; use /api only for an origin without a path.""" + url = urlsplit(base_url.strip()) + if ( + url.scheme not in {"http", "https"} + or not url.netloc + or url.query + or url.fragment + ): + raise ValueError( + "Urban API URL must be an HTTP(S) base URL without query or fragment" + ) + path = url.path.rstrip("/") or "/api" + return urlunsplit((url.scheme, url.netloc, path, "", "")) diff --git a/app/provision/provision_mcp.py b/app/provision/provision_mcp.py index 43ff8c4..d63e9d7 100644 --- a/app/provision/provision_mcp.py +++ b/app/provision/provision_mcp.py @@ -52,9 +52,7 @@ async def calc_service_provision( service_type_id=service_type_id, target_population=target_population, ) - result = await provision_mcp_service.calculate_provision( - provision_dto, user_id - ) + result = await provision_mcp_service.calculate_provision(provision_dto, user_id) return result.model_dump() except Exception as e: tb = traceback.format_exc() diff --git a/tests/test_urban_api_url.py b/tests/test_urban_api_url.py new file mode 100644 index 0000000..e5aa1de --- /dev/null +++ b/tests/test_urban_api_url.py @@ -0,0 +1,83 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.common.api_handler.api_handler import APIHandler +from app.common.api_handler.urban_api_url import normalize_urban_api_url + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "base, api_root", + [ + ("https://urban.test:8443", "https://urban.test:8443/api"), + ("https://urban.test:8443/", "https://urban.test:8443/api"), + ("https://urban.test:8443/api", "https://urban.test:8443/api"), + (" https://urban.test:8443/api/// ", "https://urban.test:8443/api"), + ( + "https://prostor-api.idu.actocgnitive.org/urban_api", + "https://prostor-api.idu.actocgnitive.org/urban_api", + ), + ( + "https://prostor-api.idu.actocgnitive.org/urban_api/", + "https://prostor-api.idu.actocgnitive.org/urban_api", + ), + ( + "https://urban.test/gateway/urban_api/", + "https://urban.test/gateway/urban_api", + ), + ], +) +@pytest.mark.parametrize("method", ["get", "post", "put", "delete"]) +@pytest.mark.parametrize( + "endpoint", + ["/api/v1/scenarios/7", "api/v1/scenarios/7", "/v1/scenarios/7", "v1/scenarios/7"], +) +async def test_requests_use_configured_api_root(base, api_root, method, endpoint): + auth = MagicMock() + auth.get_authorization_headers = AsyncMock( + return_value={"Authorization": "Bearer service"} + ) + handler = APIHandler(base, auth) + session = MagicMock() + response = MagicMock(status=200) + response.json = AsyncMock(return_value={"ok": True}) + request = getattr(session, method) + request.return_value.__aenter__.return_value = response + + assert await getattr(handler, method)( + endpoint, session=session, params={"page": 2} + ) == {"ok": True} + + assert request.call_args.kwargs["url"] == f"{api_root}/v1/scenarios/7" + assert request.call_args.kwargs["params"] == {"page": 2} + assert request.call_args.kwargs["headers"] == {"Authorization": "Bearer service"} + + +@pytest.mark.parametrize( + "base, expected", + [ + ("http://api", "http://api/api"), + ("https://urban.test/gateway/api/", "https://urban.test/gateway/api"), + ("https://urban.test/gateway/", "https://urban.test/gateway"), + ("https://urban.test/api/api/", "https://urban.test/api/api"), + ], +) +def test_normalization_preserves_authority_and_proxy_path(base, expected): + assert normalize_urban_api_url(base) == expected + assert normalize_urban_api_url(expected) == expected + + +@pytest.mark.parametrize( + "base", + [ + "", + "urban.test", + "ftp://urban.test", + "https://urban.test?x=1", + "https://urban.test#fragment", + ], +) +def test_invalid_base_url_is_rejected(base): + with pytest.raises(ValueError, match="Urban API URL"): + normalize_urban_api_url(base) From 9a20a45a3933343dbe213dfed0e6c79487b93337 Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:56:56 +0300 Subject: [PATCH 60/61] ci: (#54) - added URBAN_API to prod variables from github --- .github/workflows/build_and_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index cf791a1..6c9da6a 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -56,5 +56,6 @@ jobs: SERVICE_AUTH_REALM: ${{ vars.SERVICE_AUTH_REALM }} SERVICE_AUTH_CLIENT_ID: ${{ vars.SERVICE_AUTH_CLIENT_ID }} SERVICE_AUTH_CLIENT_SECRET: ${{ secrets.SERVICE_AUTH_CLIENT_SECRET }} + URBAN_API: ${{ vars.URBAN_API }} # run: docker run -d --name "$CONTAINER_NAME" --env-file ./.env.development -p 8210:8000 "$IMAGE_NAME":"$NOW" run: docker compose -f docker-compose.actions.yml up -d From 0ecf8340ecbd6c7c002058f4b2c98ec15c168c6c Mon Sep 17 00:00:00 2001 From: Turkov Leonid <100690100+LeonDeTur@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:19:19 +0300 Subject: [PATCH 61/61] ci: (#56) - added URBAN_API to prod variables from github