From a475f7db099f754ecdbd4b54e0d3d60c490ccaac Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 10 Dec 2025 19:57:04 +0100 Subject: [PATCH 01/11] fix: getVesselTimeByZone not returned when excursion.arrival_at is undefined --- frontend/app/map/page.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/frontend/app/map/page.tsx b/frontend/app/map/page.tsx index 6969d5ea..86ec9a05 100644 --- a/frontend/app/map/page.tsx +++ b/frontend/app/map/page.tsx @@ -158,8 +158,18 @@ export default function MapPage() { const timeByMPAZone = await getVesselTimeByZone({ vesselId: vesselID, category: ZoneCategory.AMP, - startAt: startDate ? startDate > new Date(excursion.departure_at) ? new Date(startDate) : new Date(excursion.departure_at) : undefined, - endAt: endDate ? endDate < new Date(excursion.arrival_at!) ? new Date(endDate) : new Date(excursion.arrival_at!) : undefined, + startAt: startDate + ? startDate > new Date(excursion.departure_at) + ? new Date(startDate) + : new Date(excursion.departure_at) + : undefined, + endAt: endDate + ? excursion.arrival_at != undefined + ? endDate < new Date(excursion.arrival_at!) + ? new Date(endDate!) + : new Date(excursion.arrival_at!) + : new Date(endDate!) + : undefined, }) excursion.timeByMPAZone = timeByMPAZone console.log("timeByMPAZone", timeByMPAZone) From 55c9330718468b41a03664a999d3a5d3ce5c58f5 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Mon, 5 Jan 2026 17:43:04 +0100 Subject: [PATCH 02/11] map: revalidate vessels data on mount and reconnect --- frontend/app/map/page.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/app/map/page.tsx b/frontend/app/map/page.tsx index 86ec9a05..99400ece 100644 --- a/frontend/app/map/page.tsx +++ b/frontend/app/map/page.tsx @@ -61,7 +61,8 @@ export default function MapPage() { fetcher, { revalidateOnFocus: false, - revalidateOnReconnect: false, + revalidateOnReconnect: true, + revalidateOnMount: true, keepPreviousData: true, } ) @@ -151,10 +152,13 @@ export default function MapPage() { for (const excursion of vesselExcursions) { const segments = await getVesselSegments( vesselID, - excursion.excursion_id + excursion.excursion_id, + startDate, + endDate ) excursion.segments = segments.data + console.log(excursion.arrival_at == undefined) const timeByMPAZone = await getVesselTimeByZone({ vesselId: vesselID, category: ZoneCategory.AMP, From bd745336ecac28a9172f19a987d7fd0429b7375d Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Mon, 5 Jan 2026 18:04:08 +0100 Subject: [PATCH 03/11] add daterange parameters in getVesselSegments --- frontend/services/backend-rest-client.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frontend/services/backend-rest-client.ts b/frontend/services/backend-rest-client.ts index 32047bea..dab6e41c 100644 --- a/frontend/services/backend-rest-client.ts +++ b/frontend/services/backend-rest-client.ts @@ -111,8 +111,17 @@ export function getVesselExcursionsExtracts( return axios.get(url) } -export function getVesselSegments(vesselId: string, excursionId: string) { - const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments` +export function getVesselSegments(vesselId: string, excursionId: string, startDate?: Date, + endDate?: Date) { + let queryParams: string[] = [] + if (startDate) { + queryParams.push(`start_at=${startDate.toISOString()}`) + } + if (endDate) { + // queryParams.push(`end_at=${endDate.toISOString()}`) + queryParams.push(`end_at=${endDate.toISOString()}`) + } + const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments${queryParams.length > 0 ? `?${queryParams.join("&")}` : ""}` console.log(`GET ${url}`) return axios.get(url) } From e149e2b270b7ab5936f89e258b636cd70977f6e0 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:24:56 +0100 Subject: [PATCH 04/11] vessel: add FRANK BONEFAAS --- backend/dbt_trawlwatch/seeds/static_vessels_table.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/dbt_trawlwatch/seeds/static_vessels_table.csv b/backend/dbt_trawlwatch/seeds/static_vessels_table.csv index 0724ab50..7190e703 100644 --- a/backend/dbt_trawlwatch/seeds/static_vessels_table.csv +++ b/backend/dbt_trawlwatch/seeds/static_vessels_table.csv @@ -1701,4 +1701,5 @@ PS046;617095000;EGALABUR;91;> 80 m;CPV;Purse seiner;9710995;NA;D4GX;NA;FALSE;exp PS046;312078000;EGALABUR;91;> 80 m;BLZ;Purse seiner;9710995;NA;V3OI5;NA;TRUE;active;gfw, marine traffic and spire;NA; ESP000021905;312590000;TXORI BERRI;81;> 80 m;BLZ;Purse seiner;9006033;NA;V3UO9;NA;FALSE;exported;gfw, marine traffic and spire;NA; ESP000021905;461000089;TXORI BERRI;81;> 80 m;OMN;Purse seiner;9006033;NA;A4BB5;NA;TRUE;active;gfw, marine traffic and spire;NA; -DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA; \ No newline at end of file +DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA; +NLD199301863;232024007;FRANK BONEFAAS;119.65;> 80 m;GBR;Pelagic trawler;9074951;NLD199301863;MFYP7;H72;TRUE;active;fleet register;NA; \ No newline at end of file From 09a0c16a568b18de055ecf0644d59b3535e707f9 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:26:34 +0100 Subject: [PATCH 05/11] fix: dates format on historial_dim_vessels --- backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv b/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv index 9e382bbd..8f63f630 100644 --- a/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv +++ b/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv @@ -356,7 +356,7 @@ FRA000899950;GLENAN;84.1;FRA;PS;;9322669;FMHD;FRA000899950;4000;0;2500;0;;;FMHD; ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;515.2;0;184;115;;;00MV00343M;20/03/2012;;ACTIVE;EU REGISTER ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;305.44;0;184;115;;;00MV00343M;01/02/2002;19/03/2012;ACTIVE;EU REGISTER ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;;ITA000025490;305.44;0;184;115;;;00MV00343M;28/12/2001;31/01/2002;ACTIVE;EU REGISTER -NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;2019-09-03;;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2003-01-01;2019-09-02;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2001-02-01;2002-12-31;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;2000-06-30;2001-01-31;ACTIVE;EU REGISTER \ No newline at end of file +NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;03/09/2019;;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/01/2003;02/09/2019;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/02/2001;31/12/2002;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;30/06/2000;31/01/2001;ACTIVE;EU REGISTER \ No newline at end of file From 1e5b2af916baa087209f0eb40189376b525b4374 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:27:10 +0100 Subject: [PATCH 06/11] fix: remove filters on position characteristics (heading, course and rot) --- .../dbt_trawlwatch/models/staging/stg_vessel_positions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql b/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql index 9e275c54..2c86cd36 100644 --- a/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql +++ b/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql @@ -119,9 +119,9 @@ join_spire_ais_and_vessels as ( -- On ne conserve que la dernière remontée AIS and ais.position_latitude is not NULL and ais.position_longitude is not NULL and ais.position_speed is not NULL - and ais.position_heading is not NULL - and ais.position_course is not NULL - and ais.position_rot is not NULL + -- and ais.position_heading is not NULL + -- and ais.position_course is not NULL + -- and ais.position_rot is not NULL and ais.position_timestamp is not NULL ) From 13a840077250f04700ccac85b159eae0b5af5f12 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:28:46 +0100 Subject: [PATCH 07/11] fix: date joins to safe_between --- .../models/itm/positions/itm_vessel_positions_x_zones.sql | 4 ++-- .../models/itm/segments/itm_vessel_segments.sql | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql index a56f7351..d78dee67 100644 --- a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql +++ b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql @@ -64,9 +64,9 @@ positions_with_zones_candidates as ( -- Jointure des positions des navires avec from positions as pos inner join zec_list as zec on pos.vessel_id = zec.vessel_id - and pos.position_timestamp between zec.excursion_start_position_timestamp and zec.excursion_end_position_timestamp + and utils.safe_between(pos.position_timestamp, zec.excursion_start_position_timestamp, zec.excursion_end_position_timestamp) and pos.position_id = any(zec.excursion_position_ids) -) , +), positions_x_zones as ( -- Positions des navires dans les zones maritimes (croisement spatial) select distinct diff --git a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql index e668c994..643a4ef6 100644 --- a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql +++ b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql @@ -118,8 +118,7 @@ vessel_positions as ( -- On conserve uniquement les positions correspondant à des excursions inner join ( select * from {{ ref('itm_vessel_excursions') }} ) as exc on pos.vessel_id = exc.vessel_id - and pos.position_timestamp between exc.excursion_start_position_timestamp and exc.excursion_end_position_timestamp - + and utils.safe_between(pos.position_timestamp, exc.excursion_start_position_timestamp, exc.excursion_end_position_timestamp) {{ MMSI_filter }} ), From 14728f6e5c8ad3b5fa010c3ad1b90144a0d0583a Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:29:56 +0100 Subject: [PATCH 08/11] fix: optional filter not working when missing --- .../models/itm/positions/itm_vessel_positions.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql index f54ba731..16e9e444 100644 --- a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql +++ b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql @@ -83,7 +83,7 @@ raw_vessel_positions_load as ( -- Données remontées par le microbatch sur stg_ from {{ ref('stg_vessel_positions') }} where TRUE and vessel_id not like 'UNKNOWN_MMSI=%' - {{ MMSI_filter }} + {{ MMSI_filter }} ), @@ -137,8 +137,7 @@ raw_vessel_positions as ( -- Données remontées par le microbatch sur stg_vesse position_point, cast(NULL as boolean) as rn from raw_vessel_positions_load - - {{ MMSI_filter }} + where TRUE {{ MMSI_filter }} union all -- on ajoute la « dernière » ligne de chaque navire select * from previous_positions From b14fdf2ed6aa2b77d4d2b80b987273469265eec6 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:30:36 +0100 Subject: [PATCH 09/11] fix: wrong indent --- .../models/itm/segments/itm_vessel_segments.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql index 643a4ef6..51ce8d53 100644 --- a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql +++ b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql @@ -337,8 +337,8 @@ segment_metrics as ( round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev), 0)::numeric,0) as segment_duration_s, round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev)/3600, 0)::numeric,4) as segment_duration_h, - round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m, - round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm, + round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m, + round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm, position_course as segment_course_at_end, position_course_prev as segment_course_at_start, From 620970694e8993eb42904a504fb3f14e7e2fc118 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:31:25 +0100 Subject: [PATCH 10/11] add local orchestration script --- etl_demo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl_demo.sh b/etl_demo.sh index b8007d9e..f1cf2a44 100755 --- a/etl_demo.sh +++ b/etl_demo.sh @@ -5,7 +5,7 @@ source /Users/marthevienne/12_bloom/backend/.venv/bin/activate [ ! -f /Users/marthevienne/12_bloom/backend/.env ] || export $(grep -v '^#' /Users/marthevienne/12_bloom/backend/.env | xargs) cd ./backend/dbt_trawlwatch -sleep 60 +sleep 30 dbt run --select observ_spire_ais_data_retrievals dbt run --select itm_vessel_last_raw_position From 43dc21959c821e59dcdb08a91a342b30deb8b3c6 Mon Sep 17 00:00:00 2001 From: marthevienne <123016211+marthevienne@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:31:54 +0100 Subject: [PATCH 11/11] remove orchestration script --- etl.sh | 9 --------- 1 file changed, 9 deletions(-) delete mode 100755 etl.sh diff --git a/etl.sh b/etl.sh deleted file mode 100755 index 4a27fc26..00000000 --- a/etl.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/sh - -source /home/bas/venv/bin/activate -cd $APP_FOLDER - -python bloom/tasks/load_spire_data_from_api.py && \ -flock bloom/tasks/clean_positions.py --command " - python bloom/tasks/clean_positions.py && - python bloom/tasks/create_update_excursions_segments.py" \ No newline at end of file