Minor fixes

backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql

@@ -83,7 +83,7 @@ raw_vessel_positions_load as ( -- Données remontées par le microbatch sur stg_
         from {{ ref('stg_vessel_positions') }} 
         where TRUE
             and vessel_id not like 'UNKNOWN_MMSI=%'
-        {{ MMSI_filter }}
+            {{ MMSI_filter }}
 
 ),
 
@@ -137,8 +137,7 @@ raw_vessel_positions as ( -- Données remontées par le microbatch sur stg_vesse
             position_point,
             cast(NULL as boolean) as rn
         from raw_vessel_positions_load
-
-        {{ MMSI_filter }}
+        where TRUE {{ MMSI_filter }}
 
         union all          -- on ajoute la « dernière » ligne de chaque navire
         select * from previous_positions

backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql

@@ -64,9 +64,9 @@ positions_with_zones_candidates as ( -- Jointure des positions des navires avec
   from positions as pos
 	inner join zec_list as zec
 		on pos.vessel_id = zec.vessel_id
-    and pos.position_timestamp between zec.excursion_start_position_timestamp and zec.excursion_end_position_timestamp
+    and utils.safe_between(pos.position_timestamp, zec.excursion_start_position_timestamp, zec.excursion_end_position_timestamp)
     and pos.position_id = any(zec.excursion_position_ids)
-) ,
+),
 
 positions_x_zones as ( -- Positions des navires dans les zones maritimes (croisement spatial)
   select distinct

backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql

@@ -118,8 +118,7 @@ vessel_positions as (
     -- On conserve uniquement les positions correspondant à des excursions
     inner join ( select * from {{ ref('itm_vessel_excursions') }} ) as exc
         on pos.vessel_id = exc.vessel_id
-        and pos.position_timestamp between exc.excursion_start_position_timestamp and exc.excursion_end_position_timestamp
-
+        and utils.safe_between(pos.position_timestamp, exc.excursion_start_position_timestamp, exc.excursion_end_position_timestamp)
     {{ MMSI_filter }}
 ),
 
@@ -338,8 +337,8 @@ segment_metrics as (
         round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev), 0)::numeric,0)  as segment_duration_s,
         round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev)/3600, 0)::numeric,4) as segment_duration_h,
 
-         round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m,
-         round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm,
+        round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m,
+        round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm,
 
         position_course as segment_course_at_end,
         position_course_prev as segment_course_at_start,

backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql

@@ -119,9 +119,9 @@ join_spire_ais_and_vessels as ( -- On ne conserve que la dernière remontée AIS
         and ais.position_latitude is not NULL
         and ais.position_longitude is not NULL
         and ais.position_speed is not NULL
-        and ais.position_heading is not NULL
-        and ais.position_course is not NULL
-        and ais.position_rot is not NULL
+        -- and ais.position_heading is not NULL
+        -- and ais.position_course is not NULL
+        -- and ais.position_rot is not NULL
         and ais.position_timestamp is not NULL
 )
 

backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv

@@ -356,7 +356,7 @@ FRA000899950;GLENAN;84.1;FRA;PS;;9322669;FMHD;FRA000899950;4000;0;2500;0;;;FMHD;
 ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;515.2;0;184;115;;;00MV00343M;20/03/2012;;ACTIVE;EU REGISTER
 ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;305.44;0;184;115;;;00MV00343M;01/02/2002;19/03/2012;ACTIVE;EU REGISTER
 ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;;ITA000025490;305.44;0;184;115;;;00MV00343M;28/12/2001;31/01/2002;ACTIVE;EU REGISTER
-NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;2019-09-03;;ACTIVE;EU REGISTER
-NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2003-01-01;2019-09-02;ACTIVE;EU REGISTER
-NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2001-02-01;2002-12-31;ACTIVE;EU REGISTER
-NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;2000-06-30;2001-01-31;ACTIVE;EU REGISTER
+NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;03/09/2019;;ACTIVE;EU REGISTER
+NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/01/2003;02/09/2019;ACTIVE;EU REGISTER
+NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/02/2001;31/12/2002;ACTIVE;EU REGISTER
+NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;30/06/2000;31/01/2001;ACTIVE;EU REGISTER

backend/dbt_trawlwatch/seeds/static_vessels_table.csv

@@ -1701,4 +1701,5 @@ PS046;617095000;EGALABUR;91;> 80 m;CPV;Purse seiner;9710995;NA;D4GX;NA;FALSE;exp
 PS046;312078000;EGALABUR;91;> 80 m;BLZ;Purse seiner;9710995;NA;V3OI5;NA;TRUE;active;gfw, marine traffic and spire;NA;
 ESP000021905;312590000;TXORI BERRI;81;> 80 m;BLZ;Purse seiner;9006033;NA;V3UO9;NA;FALSE;exported;gfw, marine traffic and spire;NA;
 ESP000021905;461000089;TXORI BERRI;81;> 80 m;OMN;Purse seiner;9006033;NA;A4BB5;NA;TRUE;active;gfw, marine traffic and spire;NA;
-DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA;
+DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA;
+NLD199301863;232024007;FRANK BONEFAAS;119.65;> 80 m;GBR;Pelagic trawler;9074951;NLD199301863;MFYP7;H72;TRUE;active;fleet register;NA;

etl_demo.sh

@@ -5,7 +5,7 @@ source /Users/marthevienne/12_bloom/backend/.venv/bin/activate
 [ ! -f /Users/marthevienne/12_bloom/backend/.env ] || export $(grep -v '^#' /Users/marthevienne/12_bloom/backend/.env | xargs)
 cd ./backend/dbt_trawlwatch
 
-sleep 60
+sleep 30
 
 dbt run --select observ_spire_ais_data_retrievals
 dbt run --select itm_vessel_last_raw_position

frontend/app/map/page.tsx

@@ -61,7 +61,8 @@ export default function MapPage() {
     fetcher,
     {
       revalidateOnFocus: false,
-      revalidateOnReconnect: false,
+      revalidateOnReconnect: true,
+      revalidateOnMount: true,
       keepPreviousData: true,
     }
   )
@@ -151,15 +152,28 @@ export default function MapPage() {
         for (const excursion of vesselExcursions) {
           const segments = await getVesselSegments(
             vesselID,
-            excursion.excursion_id
+            excursion.excursion_id,
+            startDate,
+            endDate
           )
           excursion.segments = segments.data
 
+          console.log(excursion.arrival_at == undefined)
           const timeByMPAZone = await getVesselTimeByZone({
             vesselId: vesselID,
             category: ZoneCategory.AMP,
-            startAt: startDate ? startDate > new Date(excursion.departure_at) ? new Date(startDate) : new Date(excursion.departure_at) : undefined,
-            endAt: endDate ? endDate < new Date(excursion.arrival_at!) ? new Date(endDate) : new Date(excursion.arrival_at!) : undefined,
+            startAt: startDate
+              ? startDate > new Date(excursion.departure_at)
+                ? new Date(startDate)
+                : new Date(excursion.departure_at)
+              : undefined,
+            endAt: endDate
+              ? excursion.arrival_at != undefined
+                ? endDate < new Date(excursion.arrival_at!)
+                  ? new Date(endDate!)
+                  : new Date(excursion.arrival_at!)
+                : new Date(endDate!)
+              : undefined,
           })
           excursion.timeByMPAZone = timeByMPAZone
           console.log("timeByMPAZone", timeByMPAZone)

frontend/services/backend-rest-client.ts

@@ -111,8 +111,17 @@ export function getVesselExcursionsExtracts(
   return axios.get<VesselExcursionSummary>(url)
 }
 
-export function getVesselSegments(vesselId: string, excursionId: string) {
-  const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments`
+export function getVesselSegments(vesselId: string, excursionId: string, startDate?: Date,
+  endDate?: Date) {
+  let queryParams: string[] = []
+  if (startDate) {
+    queryParams.push(`start_at=${startDate.toISOString()}`)
+  }
+  if (endDate) {
+    // queryParams.push(`end_at=${endDate.toISOString()}`)
+    queryParams.push(`end_at=${endDate.toISOString()}`)
+  }
+  const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments${queryParams.length > 0 ? `?${queryParams.join("&")}` : ""}`
   console.log(`GET ${url}`)
   return axios.get<VesselExcursionSegment[]>(url)
 }