From 4cef666f1f534874b172ad8e26d3ca0a0042293b Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:29:42 +0200 Subject: [PATCH 1/6] test cronjob dbt run --- backend/dbt_trawlwatch/package-lock.yml | 8 ++++---- clevercloud/cron.json | 2 +- cron.sh | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backend/dbt_trawlwatch/package-lock.yml b/backend/dbt_trawlwatch/package-lock.yml index 41aef413..e397fd72 100644 --- a/backend/dbt_trawlwatch/package-lock.yml +++ b/backend/dbt_trawlwatch/package-lock.yml @@ -1,5 +1,5 @@ packages: -- package: dbt-labs/dbt_utils - name: dbt_utils - version: 1.3.0 -sha1_hash: 8067dd74fb58d3d05a437ab1975d5eeeaf3d8bea + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/clevercloud/cron.json b/clevercloud/cron.json index 96dc4ca5..e730d646 100644 --- a/clevercloud/cron.json +++ b/clevercloud/cron.json @@ -1,3 +1,3 @@ [ - "*/15 * * * * $ROOT/cron.sh" + "* * * * * $ROOT/cron.sh" ] \ No newline at end of file diff --git a/cron.sh b/cron.sh index 0d191397..d0b8f63b 100755 --- a/cron.sh +++ b/cron.sh @@ -1,5 +1,10 @@ #! /bin/bash -l source ${APP_HOME}/backend/.venv/bin/activate -python -m bloom.usecase.create_kpler_ais_messages +# python ${APP_HOME}/backend/load_spire_data_from_api.py +# python -m bloom.usecase.create_kpler_ais_messages &&\ +cd ${APP_HOME}/backend/dbt_trawlwatch &&\ +dbt run --select observ_spire_ais_data_retrievals &&\ +dbt run --select itm_vessel_last_raw_position &&\ +dbt run --select mart_dim_vessels__last_positions deactivate \ No newline at end of file From e2888137f52ee09402401dae290582bb9a6d07c3 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Tue, 7 Apr 2026 10:35:44 +0200 Subject: [PATCH 2/6] test: add dbt run for real-time positions --- cron.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron.sh b/cron.sh index d0b8f63b..26c6a546 100755 --- a/cron.sh +++ b/cron.sh @@ -1,9 +1,9 @@ #! /bin/bash -l source ${APP_HOME}/backend/.venv/bin/activate -# python ${APP_HOME}/backend/load_spire_data_from_api.py -# python -m bloom.usecase.create_kpler_ais_messages &&\ +python -m bloom.usecase.create_kpler_ais_messages &&\ cd ${APP_HOME}/backend/dbt_trawlwatch &&\ +dbt deps &&\ dbt run --select observ_spire_ais_data_retrievals &&\ dbt run --select itm_vessel_last_raw_position &&\ dbt run --select mart_dim_vessels__last_positions From 1b7da4c7ec143b19776c7018d5a2ebfad7fc6f67 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Tue, 7 Apr 2026 10:42:53 +0200 Subject: [PATCH 3/6] update time in cronjob: every 15 minutes --- clevercloud/cron.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clevercloud/cron.json b/clevercloud/cron.json index e730d646..96dc4ca5 100644 --- a/clevercloud/cron.json +++ b/clevercloud/cron.json @@ -1,3 +1,3 @@ [ - "* * * * * $ROOT/cron.sh" + "*/15 * * * * $ROOT/cron.sh" ] \ No newline at end of file From 3573024a0fa5efd59807e517035df06a6bf153f8 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Tue, 7 Apr 2026 10:52:33 +0200 Subject: [PATCH 4/6] test: double cronjob - spire endpoint and kpler endpoint --- clevercloud/cron.json | 3 ++- cron_kpler_endpoint.sh | 5 +++++ cron.sh => cron_spire_endpoint.sh | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 cron_kpler_endpoint.sh rename cron.sh => cron_spire_endpoint.sh (82%) diff --git a/clevercloud/cron.json b/clevercloud/cron.json index 96dc4ca5..cca435b2 100644 --- a/clevercloud/cron.json +++ b/clevercloud/cron.json @@ -1,3 +1,4 @@ [ - "*/15 * * * * $ROOT/cron.sh" + "*/15 * * * * $ROOT/cron_spire_endpoint.sh", + "*/15 * * * * $ROOT/cron_kpler_endpoint.sh" ] \ No newline at end of file diff --git a/cron_kpler_endpoint.sh b/cron_kpler_endpoint.sh new file mode 100755 index 00000000..0d191397 --- /dev/null +++ b/cron_kpler_endpoint.sh @@ -0,0 +1,5 @@ +#! /bin/bash -l + +source ${APP_HOME}/backend/.venv/bin/activate +python -m bloom.usecase.create_kpler_ais_messages +deactivate \ No newline at end of file diff --git a/cron.sh b/cron_spire_endpoint.sh similarity index 82% rename from cron.sh rename to cron_spire_endpoint.sh index 26c6a546..45bb7e28 100755 --- a/cron.sh +++ b/cron_spire_endpoint.sh @@ -1,7 +1,7 @@ #! /bin/bash -l source ${APP_HOME}/backend/.venv/bin/activate -python -m bloom.usecase.create_kpler_ais_messages &&\ +python "${APP_HOME}/backend/load_spire_data_from_api.py" &&\ cd ${APP_HOME}/backend/dbt_trawlwatch &&\ dbt deps &&\ dbt run --select observ_spire_ais_data_retrievals &&\ From 74f8106fd042b5a1b96fcdb1f5b10327e72f38f4 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Tue, 7 Apr 2026 11:05:28 +0200 Subject: [PATCH 5/6] update path to load_spire_data_from_api.py --- cron_spire_endpoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron_spire_endpoint.sh b/cron_spire_endpoint.sh index 45bb7e28..0564d577 100755 --- a/cron_spire_endpoint.sh +++ b/cron_spire_endpoint.sh @@ -1,7 +1,7 @@ #! /bin/bash -l source ${APP_HOME}/backend/.venv/bin/activate -python "${APP_HOME}/backend/load_spire_data_from_api.py" &&\ +python ${APP_HOME}/backend/load_spire_data_from_api.py &&\ cd ${APP_HOME}/backend/dbt_trawlwatch &&\ dbt deps &&\ dbt run --select observ_spire_ais_data_retrievals &&\ From afba6345d364997a5b9a584d6f1577d94d619617 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Tue, 7 Apr 2026 11:08:10 +0200 Subject: [PATCH 6/6] fix: remove set_duration task in load_spire_data_from_api and clean both AIS data collecting processes --- backend/bloom/usecase/create_kpler_ais_messages.py | 1 - backend/load_spire_data_from_api.py | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/backend/bloom/usecase/create_kpler_ais_messages.py b/backend/bloom/usecase/create_kpler_ais_messages.py index 3b454f90..9148fd87 100644 --- a/backend/bloom/usecase/create_kpler_ais_messages.py +++ b/backend/bloom/usecase/create_kpler_ais_messages.py @@ -28,7 +28,6 @@ async def run(self, dump_path: str): orm_data = [] try: - process_start=datetime.now(timezone.utc) current_datetime=None position_count= None diff --git a/backend/load_spire_data_from_api.py b/backend/load_spire_data_from_api.py index e40526b0..f8a64a33 100644 --- a/backend/load_spire_data_from_api.py +++ b/backend/load_spire_data_from_api.py @@ -1,6 +1,6 @@ import argparse import json -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from pathlib import Path from time import perf_counter @@ -22,7 +22,6 @@ def run(dump_path: str) -> None: orm_data = [] try: - process_start=datetime.now(timezone.utc) current_datetime=None position_count= None with db.session() as session: @@ -48,7 +47,7 @@ def run(dump_path: str) -> None: position_count=len(raw_vessels) if dump_path is not None: try: - now =current_datetime.strftime("%Y-%m-%dT%H:%M:%S") + now = current_datetime.strftime("%Y-%m-%dT%H:%M:%S") dump_file = Path(args.dump_path, f"spire_{now}").with_suffix(".json") with dump_file.open("wt") as handle: json.dump(raw_vessels, handle) @@ -71,11 +70,6 @@ def run(dump_path: str) -> None: session.commit() raise(e) session.commit() - if current_datetime != None: - TaskExecutionRepository.set_duration(session, - "load_spire_data_from_api", - current_datetime, - datetime.now(timezone.utc)-process_start) if position_count != None: TaskExecutionRepository.set_position_count(session, "load_spire_data_from_api",