diff --git a/tests/api/test_fixture_flattening.py b/tests/api/test_fixture_flattening.py new file mode 100644 index 00000000..ca2435bb --- /dev/null +++ b/tests/api/test_fixture_flattening.py @@ -0,0 +1,86 @@ +from unittest import TestCase + +from vortexasdk.api.entity_flattening import ( + convert_to_flat_dict, + convert_fixture_to_flat_dict, +) + + +class TestFixtureFlattening(TestCase): + """Test that convert_fixture_to_flat_dict groups corporate_entities by layer.""" + + FIXTURE = { + "id": "abc123", + "vessel": { + "id": "v1", + "name": "ALPINE EAGLE", + "imo": 9480980, + "mmsi": 255804460, + "dwt": 300000, + "cubic_capacity": 330000, + "vessel_class": "vlcc", + "classes": [], + "corporate_entities": [ + { + "id": "ec1", + "layer": "effective_controller", + "label": "Frontline Ltd", + "probability": 1, + "source": "external", + }, + { + "id": "tc1", + "layer": "time_charterer", + "label": "Shell", + "probability": 0.8, + "source": "model", + "start_timestamp": "2023-06-01", + "end_timestamp": "2024-06-01", + }, + ], + "tags": [], + "flag": [], + "scrubber": [], + }, + "tonnes": 270000, + "laycan_from": "2024-01-01T00:00:00+0000", + "laycan_to": "2024-01-03T00:00:00+0000", + "fixing_timestamp": "2023-12-28T14:30:00+0000", + "vtx_fulfilled": True, + "origin": {"id": "org1", "label": "Arabian Gulf"}, + "destination": {"id": "dst1", "label": "South Korea"}, + "product": {"id": "prd1", "label": "Crude"}, + "charterer": {"id": "c1", "label": "Trafigura"}, + } + + def test_generic_flattener_uses_numeric_indices(self): + flat = convert_to_flat_dict(self.FIXTURE, columns="all") + assert "vessel.corporate_entities.0.label" in flat + assert ( + "vessel.corporate_entities.effective_controller.label" not in flat + ) + + def test_fixture_flattener_groups_by_layer(self): + flat = convert_fixture_to_flat_dict(self.FIXTURE, columns="all") + assert ( + flat["vessel.corporate_entities.effective_controller.label"] + == "Frontline Ltd" + ) + assert ( + flat["vessel.corporate_entities.time_charterer.label"] == "Shell" + ) + + def test_fixture_flattener_preserves_flat_fields(self): + flat = convert_fixture_to_flat_dict(self.FIXTURE, columns="all") + assert flat["vessel.name"] == "ALPINE EAGLE" + assert flat["vessel.imo"] == 9480980 + assert flat["tonnes"] == 270000 + assert flat["charterer.label"] == "Trafigura" + + def test_fixture_flattener_column_filter(self): + cols = [ + "vessel.name", + "vessel.corporate_entities.effective_controller.label", + ] + flat = convert_fixture_to_flat_dict(self.FIXTURE, columns=cols) + assert set(flat.keys()) == set(cols) diff --git a/tests/endpoints/test_fixtures_real.py b/tests/endpoints/test_fixtures_real.py new file mode 100644 index 00000000..32547faf --- /dev/null +++ b/tests/endpoints/test_fixtures_real.py @@ -0,0 +1,241 @@ +from datetime import datetime + +from tests.testcases import TestCaseUsingRealAPI +from vortexasdk.endpoints.fixtures import Fixtures +from vortexasdk.api.entity_flattening import ( + convert_to_flat_dict, + convert_fixture_to_flat_dict, +) + + +class TestFixturesReal(TestCaseUsingRealAPI): + """Integration tests for Fixtures endpoint (RND-21448).""" + + def test_rnd_21448_old_flattener_broken_behavior(self): + """ + RND-21448: INITIAL BROKEN BEHAVIOR + + Before fix: When users tried to request layer-based columns like + "vessel.corporate_entities.effective_controller.label" using the old + convert_to_flat_dict, those columns would be EMPTY/MISSING because + the old flattener uses numeric indices. + + This reproduces the exact bug from the ticket. + """ + result = ( + Fixtures() + .search( + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_list() + ) + + assert len(result) > 0 + first_fixture = result[0].model_dump() + + # Simulate old to_df() behavior: request layer-based columns + old_flat = convert_to_flat_dict( + first_fixture, + columns=[ + "vessel.name", + "tonnes", + "vessel.corporate_entities.effective_controller.label", + "vessel.corporate_entities.time_charterer.label", + ], + ) + + # BROKEN: Requested columns would be missing or None + assert ( + "vessel.corporate_entities.effective_controller.label" + not in old_flat + or old_flat["vessel.corporate_entities.effective_controller.label"] + is None + ), "RND-21448 (BROKEN): Requesting layer-based corporate_entities returns empty/missing" + + assert ( + "vessel.corporate_entities.time_charterer.label" not in old_flat + or old_flat["vessel.corporate_entities.time_charterer.label"] + is None + ), "RND-21448 (BROKEN): Requesting layer-based corporate_entities returns empty/missing" + + # OLD behavior: has numeric indices instead + old_flat_all = convert_to_flat_dict(first_fixture, columns="all") + numeric_keys = [ + k + for k in old_flat_all.keys() + if "corporate_entities.0." in k or "corporate_entities.1." in k + ] + assert ( + len(numeric_keys) > 0 + ), "RND-21448 (BROKEN): Old flattener produces numeric indices like corporate_entities.0.label" + + def test_rnd_21448_new_fixture_flattener_works(self): + """ + RND-21448: After fix, the new fixture flattener correctly groups corporate_entities + by layer name, making layer-based columns accessible and populated. + + This test verifies the FIX works: convert_fixture_to_flat_dict produces layer-based keys + with actual values matching what's in to_list(). + """ + result = ( + Fixtures() + .search( + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_list() + ) + + assert len(result) > 0 + first_fixture = result[0] + first_fixture_dict = first_fixture.model_dump() + + new_flat = convert_fixture_to_flat_dict( + first_fixture_dict, columns="all" + ) + + layer_based_keys = [ + k + for k in new_flat.keys() + if "corporate_entities.effective_controller." in k + or "corporate_entities.time_charterer." in k + or "corporate_entities.charterer." in k + ] + assert ( + len(layer_based_keys) > 0 + ), "RND-21448 (fixed): new flattener should produce layer-based keys" + + for key in layer_based_keys: + value = new_flat.get(key) + if value is not None: + # At least some layer-based values should be populated + assert True + break + else: + assert ( + len(layer_based_keys) > 0 + ), "RND-21448 (fixed): layer-based keys should be present" + + assert "tonnes" in new_flat + assert new_flat.get("tonnes") is not None + + def test_rnd_21448_to_df_integration(self): + """ + RND-21448: Verify that to_df() with the new fixture flattener returns + populated nested vessel fields (matching to_list() data). + """ + + list_result = ( + Fixtures() + .search( + filter_time_field="fixing_timestamp", + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_list() + ) + + assert len(list_result) > 0 + + df = ( + Fixtures() + .search( + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_df( + columns=[ + "id", + "vessel.id", + "vessel.name", + "vessel.imo", + "tonnes", + "vessel.corporate_entities.effective_controller.label", + "vessel.corporate_entities.time_charterer.label", + "charterer.label", + ] + ) + ) + + assert len(df) > 0 + + assert df["vessel.name"].notna().any() + assert df["vessel.imo"].notna().any() + assert df["tonnes"].notna().any() + + has_effective_controller = ( + df["vessel.corporate_entities.effective_controller.label"] + .notna() + .any() + ) + has_time_charterer = ( + df["vessel.corporate_entities.time_charterer.label"].notna().any() + ) + assert has_effective_controller or has_time_charterer + + def test_to_df_returns_data(self): + """Test that to_df() returns fixtures with default columns.""" + df = ( + Fixtures() + .search( + filter_time_field="fixing_timestamp", + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_df() + ) + + assert len(df) > 0 + assert "vessel.name" in df.columns + assert "tonnes" in df.columns + assert "charterer.label" in df.columns + + def test_to_df_nested_corporate_entities_populated(self): + """Test that nested vessel.corporate_entities fields are populated (RND-21448).""" + df = ( + Fixtures() + .search( + filter_time_field="fixing_timestamp", + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_df( + columns=[ + "id", + "vessel.name", + "vessel.imo", + "vessel.corporate_entities.effective_controller.label", + "vessel.corporate_entities.time_charterer.label", + "tonnes", + ] + ) + ) + + assert len(df) > 0 + assert "vessel.name" in df.columns + assert ( + "vessel.corporate_entities.effective_controller.label" + in df.columns + ) + assert ( + df["vessel.corporate_entities.effective_controller.label"] + .notna() + .any() + ) + + def test_to_df_tonnes_column_populated(self): + """Test that tonnes column (fixed typo from tones) is populated.""" + df = ( + Fixtures() + .search( + filter_time_field="fixing_timestamp", + filter_time_min=datetime(2020, 1, 1), + filter_time_max=datetime(2020, 1, 2), + ) + .to_df(columns=["id", "tonnes"]) + ) + + assert len(df) > 0 + assert "tonnes" in df.columns + assert df["tonnes"].notna().sum() > len(df) * 0.5 diff --git a/vortexasdk/api/entity_flattening.py b/vortexasdk/api/entity_flattening.py index aa46fedc..0cf9441d 100644 --- a/vortexasdk/api/entity_flattening.py +++ b/vortexasdk/api/entity_flattening.py @@ -84,5 +84,27 @@ def _flatten_attributes(dictionary: Dict, key: str) -> Dict: return copied_dict +def convert_fixture_to_flat_dict( + fixture: Dict, columns: Union[Literal["all"], List[str]] = "all" +) -> Dict: + """Convert nested `Fixture` object to flat dictionary, keeping *cols*.""" + as_dict = _group_fixture_attributes_by_layer(fixture) + + formatted = flatten_dictionary(as_dict) + + if columns == "all": + return formatted + else: + return {k: v for k, v in formatted.items() if k in columns} + + +def _group_fixture_attributes_by_layer(fixture: Dict) -> Dict: + """Group relevant `Fixture` attributes by `Entity.layer`.""" + copied = copy.deepcopy(fixture) + if "vessel" in copied and isinstance(copied["vessel"], dict): + copied["vessel"] = _flatten_vessel_entity(copied["vessel"]) + return copied + + def _group_by_layer(entity_list: List[Dict]) -> Dict: return {e["layer"]: e for e in entity_list} diff --git a/vortexasdk/api/fixture.py b/vortexasdk/api/fixture.py index b201d28c..10946d0e 100644 --- a/vortexasdk/api/fixture.py +++ b/vortexasdk/api/fixture.py @@ -16,7 +16,7 @@ class Fixture(BaseModel): vessel: Optional[VesselEntity] = None laycan_from: Optional[str] = None laycan_to: Optional[str] = None - tones: Optional[int] = None + tonnes: Optional[int] = None fixing_timestamp: Optional[str] = None vtx_fulfilled: Optional[bool] = None origin: Optional[Entity] = None diff --git a/vortexasdk/endpoints/fixtures.py b/vortexasdk/endpoints/fixtures.py index e2d6f33f..98a87ecf 100644 --- a/vortexasdk/endpoints/fixtures.py +++ b/vortexasdk/endpoints/fixtures.py @@ -161,7 +161,7 @@ def search( returns - | vessel.name | tones | origin.label | product.label | + | vessel.name | tonnes | origin.label | product.label | | ------------------------ | ---------------- | ------------ | ------------- | | ALPINE EAGLE | 454.96048964485 | UK | Crude | diff --git a/vortexasdk/endpoints/fixtures_result.py b/vortexasdk/endpoints/fixtures_result.py index ce8f14bc..f99486e5 100644 --- a/vortexasdk/endpoints/fixtures_result.py +++ b/vortexasdk/endpoints/fixtures_result.py @@ -7,7 +7,7 @@ import pandas as pd from vortexasdk.api.fixture import Fixture -from vortexasdk.api.entity_flattening import convert_to_flat_dict +from vortexasdk.api.entity_flattening import convert_fixture_to_flat_dict from vortexasdk.api.search_result import Result from vortexasdk.logger import get_logger from vortexasdk.result_conversions import create_dataframe, create_list @@ -20,7 +20,7 @@ "vessel.name", "laycan_from", "laycan_to", - "tones", + "tonnes", "fixing_timestamp", "vtx_fulfilled", "destination.label", @@ -49,40 +49,40 @@ def to_df( columns: The Fixtures columns we want in the dataframe. Defaults to `columns = [ "id", - 'vessels.corporate_entities.charterer.id', - 'vessels.corporate_entities.charterer.label', - 'vessels.corporate_entities.charterer.layer', - 'vessels.corporate_entities.charterer.probability', - 'vessels.corporate_entities.charterer.source', - 'vessels.corporate_entities.effective_controller.id', - 'vessels.corporate_entities.effective_controller.label', - 'vessels.corporate_entities.effective_controller.layer', - 'vessels.corporate_entities.effective_controller.probability', - 'vessels.corporate_entities.effective_controller.source', - 'vessels.corporate_entities.time_charterer.end_timestamp', - 'vessels.corporate_entities.time_charterer.id', - 'vessels.corporate_entities.time_charterer.label', - 'vessels.corporate_entities.time_charterer.layer', - 'vessels.corporate_entities.time_charterer.probability', - 'vessels.corporate_entities.time_charterer.source', - 'vessels.corporate_entities.time_charterer.start_timestamp', - 'vessels.cubic_capacity', - 'vessels.dwt', - 'vessels.end_timestamp', - 'vessels.id', - 'vessels.imo', - 'vessels.mmsi', - 'vessels.name', - 'vessels.start_timestamp', - 'vessels.status', - 'vessels.tags.end_timestamp', - 'vessels.tags.start_timestamp', - 'vessels.tags.tag', - 'vessels.vessel_class', - 'vessels.voyage_id', + 'vessel.corporate_entities.charterer.id', + 'vessel.corporate_entities.charterer.label', + 'vessel.corporate_entities.charterer.layer', + 'vessel.corporate_entities.charterer.probability', + 'vessel.corporate_entities.charterer.source', + 'vessel.corporate_entities.effective_controller.id', + 'vessel.corporate_entities.effective_controller.label', + 'vessel.corporate_entities.effective_controller.layer', + 'vessel.corporate_entities.effective_controller.probability', + 'vessel.corporate_entities.effective_controller.source', + 'vessel.corporate_entities.time_charterer.end_timestamp', + 'vessel.corporate_entities.time_charterer.id', + 'vessel.corporate_entities.time_charterer.label', + 'vessel.corporate_entities.time_charterer.layer', + 'vessel.corporate_entities.time_charterer.probability', + 'vessel.corporate_entities.time_charterer.source', + 'vessel.corporate_entities.time_charterer.start_timestamp', + 'vessel.cubic_capacity', + 'vessel.dwt', + 'vessel.end_timestamp', + 'vessel.id', + 'vessel.imo', + 'vessel.mmsi', + 'vessel.name', + 'vessel.start_timestamp', + 'vessel.status', + 'vessel.tags.end_timestamp', + 'vessel.tags.start_timestamp', + 'vessel.tags.tag', + 'vessel.vessel_class', + 'vessel.voyage_id', "laycan_from", "laycan_to", - "tones", + "tonnes", "fixing_timestamp", "vtx_fulfilled", "destination.label", @@ -103,7 +103,7 @@ def to_df( "vessel.name", "laycan_from", "laycan_to", - "tones", + "tonnes", "fixing_timestamp", "vtx_fulfilled", "destination.label", @@ -117,7 +117,9 @@ def to_df( `pd.DataFrame` of Fixtures. """ - flatten = functools.partial(convert_to_flat_dict, columns=columns) + flatten = functools.partial( + convert_fixture_to_flat_dict, columns=columns + ) with Pool(os.cpu_count()) as pool: records = pool.map(flatten, super().to_list()) diff --git a/vortexasdk/version.py b/vortexasdk/version.py index 7b64613a..8873ad42 100644 --- a/vortexasdk/version.py +++ b/vortexasdk/version.py @@ -1 +1 @@ -__version__ = "1.0.29" +__version__ = "1.0.30a1"