diff --git a/tests/conftest.py b/tests/conftest.py index 71a9cbfa4..574a3b593 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,10 +32,8 @@ test_data_path = Path(__file__).parent.absolute() / "test_data" -# A list to register functions that return general spools or patches -# These are to be used for running many patches/spools through -# Generic tests. -SPOOL_FIXTURES = [] +# A list to register functions that return patches, for running many of +# them through generic tests (the `patch` meta-fixture below). PATCH_FIXTURES = [] # By default DASCore only issues a warning once per line. This ensures @@ -511,7 +509,7 @@ def patch(request): return request.getfixturevalue(request.param) -@pytest.fixture(scope="class") +@pytest.fixture(scope="session") def one_file_dir(tmp_path_factory, random_patch): """Create a directory with a single DAS file.""" out = Path(tmp_path_factory.mktemp("one_file_file_spool")) @@ -526,7 +524,7 @@ def random_directory_spool(tmp_path_factory): return dc.examples.random_directory_spool(path=path) -@pytest.fixture(scope="class") +@pytest.fixture(scope="session") def two_patch_directory(tmp_path_factory, terra15_das_example_path, random_patch): """Create a directory of DAS files for testing.""" # first copy in a terra15 file @@ -537,7 +535,7 @@ def two_patch_directory(tmp_path_factory, terra15_das_example_path, random_patch return dir_path -@pytest.fixture(scope="class") +@pytest.fixture(scope="session") def diverse_spool_directory(diverse_spool, tmp_path_factory): """Save the diverse spool contents to a directory. @@ -549,29 +547,10 @@ def diverse_spool_directory(diverse_spool, tmp_path_factory): return ex.spool_to_directory(diverse_spool, path=out) -@pytest.fixture(scope="class") -def adjacent_spool_directory(tmp_path_factory, adjacent_spool_no_overlap): - """Create a directory of adjacent patches.""" - # create a directory with several patch files in it. - dir_path = Path(tmp_path_factory.mktemp("data")) - for num, patch in enumerate(adjacent_spool_no_overlap): - path = dir_path / f"{num}_patch.hdf5" - dc.write(patch, path, file_format="dasdae") - return dir_path - - # --- Spool fixtures -@pytest.fixture() -@register_func(SPOOL_FIXTURES) -def terra15_das_spool(terra15_das_example_path) -> SpoolType: - """Return the spool of Terra15 Das Array.""" - return read(terra15_das_example_path, file_format="terra15") - - @pytest.fixture(scope="session") -@register_func(SPOOL_FIXTURES) def terra15_das_unfinished_path() -> Path: """Return the spool of Terra15 Das Array.""" out = fetch("terra15_das_unfinished.hdf5") @@ -579,15 +558,13 @@ def terra15_das_unfinished_path() -> Path: return out -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def random_spool() -> SpoolType: """Init a random array.""" return get_example_spool("random_das") -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def adjacent_spool_no_overlap(random_patch) -> dc.BaseSpool: """ Create a spool with several patches within one time sample but not @@ -609,22 +586,19 @@ def adjacent_spool_no_overlap(random_patch) -> dc.BaseSpool: return dc.spool([pa2, pa1, pa3]) -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def one_file_directory_spool(one_file_dir): """Create a directory with a single DAS file.""" return Spool.from_directory(one_file_dir).update() -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def diverse_spool(): """Create a spool with a diverse set of patches for testing.""" return ex.diverse_spool() -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def diverse_directory_spool(diverse_spool_directory): """Save the diverse spool contents to a directory.""" out = dc.spool(diverse_spool_directory).update() @@ -633,8 +607,7 @@ def diverse_directory_spool(diverse_spool_directory): out.indexer.close() -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def basic_file_spool(two_patch_directory): """Return a DAS bank on basic_bank_directory.""" out = Spool.from_directory(two_patch_directory).update().update() @@ -643,14 +616,12 @@ def basic_file_spool(two_patch_directory): @pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) def terra15_file_spool(terra15_v5_path): """A file spool for terra15.""" return dc.spool(terra15_v5_path) -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def memory_spool_dim_1_patches(): """ Memory spool with patches that have length 1 in one dimension. @@ -666,8 +637,7 @@ def memory_spool_dim_1_patches(): return spool -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def all_examples_spool(tmp_path_factory, terra15_das_example_path): """Create a spool from all the example files.""" # Indexing the example files where they sit would write an index into the @@ -683,8 +653,7 @@ def all_examples_spool(tmp_path_factory, terra15_das_example_path): return dc.spool(directory).update() -@pytest.fixture(scope="class") -@register_func(SPOOL_FIXTURES) +@pytest.fixture(scope="session") def memory_spool_small_dt_differences(random_spool): """Create a memory spool with slightly different time_steps.""" out = [] @@ -698,7 +667,6 @@ def memory_spool_small_dt_differences(random_spool): @pytest.fixture(scope="session") -@register_func(SPOOL_FIXTURES) def spool_with_non_coords(): """Return a spool which has some non-coordinate patches inside.""" patches = list(dc.examples.get_example_spool(length=3)) @@ -706,12 +674,6 @@ def spool_with_non_coords(): return dc.spool(patches) -@pytest.fixture(scope="class", params=SPOOL_FIXTURES) -def spool(request): - """A meta-fixtures for collecting all spools used in testing.""" - return request.getfixturevalue(request.param) - - # --- Misc. test fixtures diff --git a/tests/test_core/test_annotation_loader.py b/tests/test_core/test_annotation_loader.py index c00df427c..703180e60 100644 --- a/tests/test_core/test_annotation_loader.py +++ b/tests/test_core/test_annotation_loader.py @@ -62,7 +62,7 @@ def _denies_access() -> bool: DENIES_ACCESS = _denies_access() -@pytest.fixture +@pytest.fixture(scope="module") def curve() -> Moveout: """A moveout a path may be drawn from.""" return Moveout( @@ -75,7 +75,7 @@ def curve() -> Moveout: ) -@pytest.fixture +@pytest.fixture(scope="module") def regions() -> dc.AnnotationSet: """A set of regions, which a bare table can hold.""" frame = pd.DataFrame( @@ -103,7 +103,7 @@ def regions() -> dc.AnnotationSet: ) -@pytest.fixture +@pytest.fixture(scope="module") def with_vertices(curve) -> dc.AnnotationSet: """A set holding a hand-drawn path and one drawn from a curve.""" drawn = curve.vertices(5) @@ -133,7 +133,7 @@ def with_vertices(curve) -> dc.AnnotationSet: return dc.AnnotationSet(frame, dims=DIMS, vertices=vertices) -@pytest.fixture +@pytest.fixture(scope="module") def picks() -> dc.AnnotationSet: """A set of time ranges made by a picker, on its own acquisition.""" frame = pd.DataFrame( @@ -277,7 +277,7 @@ def test_an_unstated_bound(self, tmp_path): class TestDeclaredDtypes: """A CSV states no types; the declaration beside it gives them back.""" - @pytest.fixture + @pytest.fixture(scope="class") def typed(self): """A set declaring a categorical and a nullable integer column.""" frame = pd.DataFrame( @@ -1689,7 +1689,7 @@ def _forge(frame: pd.DataFrame, path, documents: str) -> None: class TestParquet: """The same tables, with their types kept, for a set too big to want text.""" - @pytest.fixture + @pytest.fixture(scope="class") def mixed(self) -> dc.AnnotationSet: """A set whose columns hold what a CSV would have to spell as text.""" frame = pd.DataFrame( diff --git a/tests/test_core/test_coord_segmented.py b/tests/test_core/test_coord_segmented.py index ccca3b114..8c92d7999 100644 --- a/tests/test_core/test_coord_segmented.py +++ b/tests/test_core/test_coord_segmented.py @@ -133,13 +133,6 @@ def test_fusing_inputs_rejected_by_class(self): with pytest.raises(ValidationError, match="fuse"): CoordSegmented(segments=(c1, c2)) - def test_overlap_raises(self): - """Overlapping segments are rejected.""" - c1 = get_coord(start=0.0, stop=10.0, step=1.0) - c2 = get_coord(start=5.0, stop=15.0, step=1.0) - with pytest.raises(CoordError, match="overlap"): - concat_coords(c1, c2) - def test_shared_value_raises(self): """Segments sharing a boundary value are rejected (not strict).""" c1 = get_coord(start=0.0, stop=10.0, step=1.0) # max 9 @@ -539,11 +532,6 @@ def test_the_guard_asks_every_segment(self, float_gap_coord): class TestSimplifyAndSnap: """Tests for tolerance-bounded simplification and snapping.""" - def test_simplify_zero_keeps_structure(self, float_gap_coord): - """Zero tolerance cannot absorb a real gap.""" - out = float_gap_coord.simplify(0) - assert out == float_gap_coord - def test_simplify_absorbs_gap_within_tolerance(self, float_gap_coord): """A large enough tolerance collapses to a single range.""" out = float_gap_coord.simplify(3.0) @@ -1150,10 +1138,10 @@ def test_units(self): class TestPlannedSpoolWriteGuard: """The gap write guard covers plan-assembled spools (round-4 F3).""" - @pytest.fixture() - def gapped_planned_spool(self, tmp_path): + @pytest.fixture(scope="class") + def gapped_planned_spool(self, tmp_path_factory): """A file-backed planned spool whose output spans a real gap.""" - src = tmp_path / "src" + src = tmp_path_factory.mktemp("gapped_planned") / "src" src.mkdir() p1 = dc.get_example_patch() t = p1.get_coord("time") diff --git a/tests/test_core/test_coords.py b/tests/test_core/test_coords.py index 404590c09..284f58200 100644 --- a/tests/test_core/test_coords.py +++ b/tests/test_core/test_coords.py @@ -156,11 +156,15 @@ def coord(request) -> BaseCoord: return request.getfixturevalue(request.param) -@pytest.fixture(scope="session", params=COORDS) +@pytest.fixture(scope="session") def long_coord(coord) -> BaseCoord: - """Meta-fixture for returning all coords with len > 7.""" - if len(coord) < 7: - pytest.skip("Only coords with len 3 or more used.") + """The coord meta-fixture, for tests which need one longer than 7. + + Every coord in COORDS is at least 100 long. Do not add `params=COORDS` + here: `coord` is already parametrized over them, so a second pass runs + each test once per pair of coords to see the same twelve. + """ + assert len(coord) > 7 return coord @@ -195,10 +199,6 @@ def assert_value_in_one_step(coord, index, value, greater=True): class TestBasics: """A suite of basic tests for coordinates.""" - def test_coord_init(self, coord): - """Simply run to insure all coords initialize.""" - assert isinstance(coord, BaseCoord) - def test_bad_init(self): """Ensure no parameters raises error.""" with pytest.raises(CoordError): @@ -1041,13 +1041,6 @@ def test_duplicate_array_samples(self, long_coord): assert len(coord) == len(inds) assert np.all(coord.values == coord.values[0]) - def test_non_integer_array_with_samples_raises(self, evenly_sampled_coord): - """Samples argument should require integer arrays.""" - vals = np.array([1.01, 2.0, 3.0]) - msg = "requires integer dtype" - with pytest.raises(CoordError, match=msg): - evenly_sampled_coord.select(vals, samples=True) - def test_duplicate_array_values(self, long_coord): """Ensure duplicate values cause duplicates in array.""" second_value = long_coord.values[1] @@ -2020,7 +2013,7 @@ def test_timedelta(self, evenly_sampled_time_delta_coord): out = evenly_sampled_time_delta_coord.get_sample_count(12 * dt) assert out == 12 - @pytest.mark.parametrize("sample", (0, 10, 100, 42, 13)) + @pytest.mark.parametrize("sample", (0, 42)) def test_samples(self, evenly_sampled_coord, sample): """Ensure value is returned when samples==True.""" assert len(evenly_sampled_coord) >= sample @@ -2351,7 +2344,9 @@ def test_negative_length_raises( with pytest.raises(ParameterError, match="non-negative"): coord.change_length(length) - @pytest.mark.parametrize("length", [2.5, 3.0, "3", None, True, False]) + # A float, a string and a bool: the three kinds of thing which are not + # an integer length (bool is the one the check has a clause for). + @pytest.mark.parametrize("length", [2.5, "3", True]) def test_non_integer_length_raises( self, evenly_sampled_coord, basic_non_coord, length ): diff --git a/tests/test_core/test_directory_spool.py b/tests/test_core/test_directory_spool.py index 909110aa7..b7527b3e4 100644 --- a/tests/test_core/test_directory_spool.py +++ b/tests/test_core/test_directory_spool.py @@ -15,13 +15,10 @@ from dascore.constants import ONE_SECOND from dascore.core.spool import Spool from dascore.exceptions import InvalidSpoolError, MissingPatchError, ParameterError -from dascore.utils.misc import register_func, suppress_warnings +from dascore.utils.misc import suppress_warnings -DIRECTORY_SPOOLS = [] - -@pytest.fixture(scope="class") -@register_func(DIRECTORY_SPOOLS) +@pytest.fixture(scope="module") def dir_spool_index_out_of_order(random_spool, tmp_path_factory): """Create an index that isn't order chronologically.""" path = tmp_path_factory.mktemp("out_of_order_index") @@ -38,16 +35,14 @@ def dir_spool_index_out_of_order(random_spool, tmp_path_factory): return spool -@pytest.fixture(scope="class") -@register_func(DIRECTORY_SPOOLS) +@pytest.fixture(scope="module") def one_directory_spool(one_file_dir): """Create a directory with a single DAS file.""" spool = Spool.from_directory(one_file_dir) return spool.update() -@pytest.fixture(scope="class") -@register_func(DIRECTORY_SPOOLS) +@pytest.fixture(scope="module") def non_distance_dir_spool(tmp_path_factory): """Create a directory with a single DAS file.""" # Simulate a patch that has time but no canonical distance coordinate. @@ -59,8 +54,7 @@ def non_distance_dir_spool(tmp_path_factory): return dc.spool(path).update() -@pytest.fixture(scope="class") -@register_func(DIRECTORY_SPOOLS) +@pytest.fixture(scope="module") def multi_patch_file_spool(tmp_path_factory): """Create a directory whose single file contains multiple patches.""" path = tmp_path_factory.mktemp("multi_patch_file_spool") @@ -76,32 +70,20 @@ def multi_patch_file_spool(tmp_path_factory): @pytest.fixture def directory_spool_redundant_index(random_spool, tmp_path_factory): - """Force a spool to be indexed many times with same files.""" + """A spool re-indexed over files whose contents did not change.""" path = Path(tmp_path_factory.mktemp("redundant_index_spool")) dascore.examples.spool_to_directory(random_spool, path, "dasdae") spool = dc.spool(path).update() - - # Touch each file, re-index to saturate index with duplicates. - for _ in range(12): - for file_path in path.glob("*"): - file_path.touch() - spool = spool.update() - return spool - - -@pytest.fixture(scope="class", params=DIRECTORY_SPOOLS) -def directory_spool(request): - """Meta fixture for getting all file spools.""" - return request.getfixturevalue(request.param) + # Touch, then re-index: one round is what puts an already-indexed file + # through the indexer again, which is the state under test. + for file_path in path.glob("*"): + file_path.touch() + return spool.update() class TestDirectorySpoolBasics: """Basic tests for the directory spool.""" - def test_isinstance(self, directory_spool): - """Simply ensure expected type was returned.""" - assert isinstance(directory_spool, Spool) - def test_selected_str(self, diverse_directory_spool): """Ensure select kwargs show up in str.""" new = diverse_directory_spool.select(tag="big_gaps") @@ -246,18 +228,6 @@ def first_patch_range(self, random_spool): time = patch.get_coord("time") return (time.min(), time.max()) - def test_contents_restricted(self, spool_dir, random_spool, first_patch_range): - """Rows outside the requested range must not appear (regression).""" - spool = Spool.from_directory(spool_dir).update().select(time=first_patch_range) - assert 1 <= len(spool) < len(random_spool) - contents = spool.get_contents() - assert (contents["time_min"] <= first_patch_range[1]).all() - assert (contents["time_max"] >= first_patch_range[0]).all() - for patch in spool: - time = patch.get_coord("time") - assert time.min() >= first_patch_range[0] - assert time.max() <= first_patch_range[1] - def test_selected_spool_refuses_update( self, spool_dir, random_spool, first_patch_range ): @@ -266,11 +236,6 @@ def test_selected_spool_refuses_update( with pytest.raises(InvalidSpoolError, match="root spool"): spool.update() - def test_select_kwargs_parameter_removed(self, spool_dir): - """The constructor no longer accepts select_kwargs.""" - with pytest.raises(TypeError, match="select_kwargs"): - Spool.from_directory(spool_dir, select_kwargs={"tag": "x"}) - class TestDirectoryIndex: """Tests for returning summaries of all files in managed directory.""" @@ -361,13 +326,14 @@ def test_specify_index_path(self, random_patch, tmp_path_factory): assert isinstance(patch, dc.Patch) assert not default_index_path.exists() - def test_nested_directories(self, diverse_spool, tmp_path_factory): + def test_nested_directories(self, random_spool, tmp_path_factory): """Ensure files in nested directories work up to 3 levels.""" - # split the spool into 3 - sp_len = len(diverse_spool) + # One patch per level: what is under test is the walk, not how many + # files each level holds. + sp_len = len(random_spool) num = 3 spools = [ - diverse_spool[int((x / num) * sp_len) : int(((x + 1) / num) * sp_len)] + random_spool[int((x / num) * sp_len) : int(((x + 1) / num) * sp_len)] for x in range(num) ] # write each group to a different sub path @@ -422,33 +388,6 @@ def test_is_in_tag(self, basic_file_spool, spool_tag): out = basic_file_spool.select(tag=tag_collection).get_contents() assert out["tag"].isin(tag_collection).all() - def test_multiple_selects(self, diverse_directory_spool): - """Ensure selects can be stacked.""" - spool = diverse_directory_spool - contents = spool.get_contents() - duration = contents["time_max"] - contents["time_min"] - new_max = (contents["time_min"] + duration.mean() / 2).median() - out = ( - spool.select(acquisition_key="DAS2.*") - .select(tag="ran*") - .select(time=(None, new_max)) - ) - assert len(out) > 0 - # first check content dataframe - new_content = out.get_contents() - assert len(new_content) == len(out) - assert (new_content["acquisition_key"] == "DAS2.R2D1..RAW").all() - assert (new_content["tag"].str.startswith("ran")).all() - assert (new_content["time_max"] <= new_max).all() - # then check patches - for patch in out: - assert patch.attrs["acquisition_key"] == "DAS2.R2D1..RAW" - assert patch.attrs["tag"].startswith("ran") - assert patch.get_coord("time").max() <= new_max - # ensure raises when selecting off the end of the spool - with pytest.raises(IndexError): - out[len(new_content)] - def test_select_time_tuple_with_string(self, basic_file_spool): """Ensure time tuples with strings still work.""" time_str = "2017-09-18T00:00:04" @@ -493,26 +432,11 @@ def test_select_correct_history_str(self, diverse_directory_spool): class TestBasicChunk: """Tests for chunking filespool.""" - @pytest.fixture(scope="class") - def dir_spool_1_dim_patches(self, memory_spool_dim_1_patches, tmp_path_factory): - """Create a directory with patches that have 1 dim in time.""" - path = tmp_path_factory.mktemp("dir_spool_1_dim_patches") - out = dc.examples.spool_to_directory(memory_spool_dim_1_patches, path) - return dc.spool(out).update() - def test_directory_path_doesnt_change(self, one_file_directory_spool): """Chunking shouldn't change the path to the managed directory.""" out = one_file_directory_spool.chunk(time=1) assert out.spool_path == one_file_directory_spool.spool_path - def test_chunk_doesnt_modify_original(self, one_file_directory_spool): - """Chunking shouldn't modify original spool or its dfs.""" - spool = one_file_directory_spool - contents_before_chunk = spool.get_contents() - _ = spool.chunk(time=2) - contents_after_chunk = spool.get_contents() - assert contents_before_chunk.equals(contents_after_chunk) - def test_sub_chunk(self, one_file_directory_spool): """Ensure the patches can be subdivided.""" spool = one_file_directory_spool @@ -528,18 +452,6 @@ def test_sub_chunk(self, one_file_directory_spool): for patch in patch_list: assert isinstance(patch, dc.Patch) - def test_merge_1_dim_patches(self, dir_spool_1_dim_patches): - """Ensure patches with one sample in time can be merged.""" - spool = dir_spool_1_dim_patches - new = spool.chunk(time=None) - assert len(new) == 1 - patch = new[0] - content = spool.get_contents() - time_coord = patch.get_coord("time") - assert time_coord.min() == content["time_min"].min() - assert time_coord.max() == content["time_max"].max() - assert time_coord.step == spool[0].get_coord("time").step - def test_chunk_out_of_order_index(self, dir_spool_index_out_of_order): """Ensure when the index isn't ordered chunk can still work.""" spool = dir_spool_index_out_of_order @@ -554,11 +466,15 @@ def test_chunk_out_of_order_index(self, dir_spool_index_out_of_order): # samples shorter than what was asked for. Maybe revisit this? assert diff <= 2 * (time_coord.step / ONE_SECOND) - def test_chunk_redundant_index(self, directory_spool_redundant_index): + def test_chunk_redundant_index(self, directory_spool_redundant_index, random_spool): """Ensure redundant indices are handled effectively with chunking""" - spool = directory_spool_redundant_index.chunk(time=None) - patch = spool[0] - assert isinstance(patch, dc.Patch) + spool = directory_spool_redundant_index + # Re-indexing unchanged files adds no rows, so the contiguous + # patches still merge into one rather than into one per index pass. + assert len(spool.get_contents()) == len(random_spool) + merged = spool.chunk(time=None) + assert len(merged) == 1 + assert isinstance(merged[0], dc.Patch) class TestGetContents: diff --git a/tests/test_core/test_patch.py b/tests/test_core/test_patch.py index 757ce19d9..8a7f271ec 100644 --- a/tests/test_core/test_patch.py +++ b/tests/test_core/test_patch.py @@ -485,18 +485,6 @@ def test_summary_unknown_coord_field_raises(self, random_patch): _ = summary.time_missing assert not hasattr(summary, "time_missing") - def test_summary_flattened_lookup_is_removed(self, random_patch): - """Flattened summary item access should no longer work.""" - summary = random_patch.summary - with pytest.raises(TypeError): - _ = summary["time_min"] - - def test_summary_get_coord_is_removed(self, random_patch): - """PatchSummary should only expose get_coord_summary.""" - summary = random_patch.summary - with pytest.raises(AttributeError, match="get_coord"): - summary.get_coord("time") - def test_flat_dump_prefers_coord_values_over_attrs(self, random_patch): """flat_dump should overlay coord summaries on top of attrs.""" attrs = dc.PatchAttrs(**(random_patch.attrs.model_dump() | {"time_step": 10})) diff --git a/tests/test_core/test_patch_chunk.py b/tests/test_core/test_patch_chunk.py index f8f92ca26..9946b61ba 100644 --- a/tests/test_core/test_patch_chunk.py +++ b/tests/test_core/test_patch_chunk.py @@ -46,27 +46,6 @@ def random_spool_df(self, random_spool): df = random_spool.get_contents().sort_values("time_min").reset_index(drop=True) return df - def test_merge_eq(self, adjacent_spool_no_overlap): - """Ensure merged spools are equal.""" - sp1 = adjacent_spool_no_overlap.chunk(time=2) - sp2 = adjacent_spool_no_overlap.chunk(time=2) - assert sp1 == sp2 - - def test_merge_chunk_adjacent_no_overlap(self, adjacent_spool_no_overlap): - """Ensure chunking works on simple case of contiguous data w/ no overlap.""" - new = adjacent_spool_no_overlap.chunk(time=None) - out_list = list(new) - assert len(new) == len(out_list) == 1 - - def test_adjacent_merge_no_overlap(self, adjacent_spool_no_overlap): - """Test that the adjacent patches get merged.""" - spool = adjacent_spool_no_overlap - st_len = len(spool) - merged_st = spool.chunk(time=None) - merged_len = len(merged_st) - assert merged_len < st_len - assert merged_len == 1 - def test_chunk_doesnt_modify_original(self, random_spool): """Chunking shouldn't modify original spool.""" first = random_spool.get_contents().copy() @@ -377,11 +356,6 @@ def test_ellipsis(self, spool_slight_gap): pa2 = spool_slight_gap.chunk(time=None) assert pa1 == pa2 - def test_merge_transposed_patches(self, spool_complete_overlap): - """Ensure if one of the patches is transposed merge still works.""" - # TODO for now this won't work; its probably a silly edge case to complicate - # the code over, but maybe revisit. - def test_merge_monotonic_no_overlap(self, adjacent_spool_monotonic): """Ensure monotonic coords can merge.""" sp = adjacent_spool_monotonic.chunk(time=...) @@ -726,47 +700,6 @@ def test_chunk_merge_then_chunk_split(self, random_spool): assert not pd.isna(df_time_max), f"DF row {i} has NaN time_max" assert df_time_min <= df_time_max, f"DF row {i} has invalid time range" - def test_multiple_chained_chunks(self, random_spool): - """Test multiple chained chunk operations. See #533.""" - # Chain multiple chunk operations - result_spool = random_spool.chunk(time=...).chunk(time=5).chunk(time=2) - - # Should be able to access all patches - for i in range(len(result_spool)): - patch = result_spool[i] - assert isinstance(patch, dc.Patch) - - def test_chunk_split_then_merge(self, random_spool): - """Test chaining chunk split followed by merge. See #533.""" - # First chunk into smaller pieces, then merge back - result_spool = random_spool.chunk(time=1).chunk(time=...) - - # Should be able to access patches (this test the fix works) - first_patch = result_spool[0] - assert isinstance(first_patch, dc.Patch) - - # The merge operation should result in fewer patches than the chunked operation - chunked_spool = random_spool.chunk(time=1) - assert len(result_spool) <= len(chunked_spool) - - # Verify NO patches have NaN values and dataframe consistency - result_contents = result_spool.get_contents().reset_index(drop=True) - for i, patch in enumerate(result_spool): - # Assert no NaN values in patch attributes - time_coord = patch.get_coord("time") - assert not pd.isna(time_coord.min()), f"Patch {i} has NaN time_min" - assert not pd.isna(time_coord.max()), f"Patch {i} has NaN time_max" - - # Verify dataframe contains reasonable time values - df_row = result_contents.iloc[i] - df_time_min = dc.to_datetime64(df_row["time_min"]) - df_time_max = dc.to_datetime64(df_row["time_max"]) - - # Dataframe times should not be NaN or invalid - assert not pd.isna(df_time_min), f"DF row {i} has NaN time_min" - assert not pd.isna(df_time_max), f"DF row {i} has NaN time_max" - assert df_time_min <= df_time_max, f"DF row {i} has invalid time range" - def test_chunk_non_adjacent_within_tolerance_warns(self, random_patch): """ Non-adjacent patches can still merge, but the coordinate type may change. @@ -1259,18 +1192,6 @@ def test_compatible_units_convert(self): class TestUnitChunkValue: """Chunk lengths carrying the coordinate's own units.""" - def test_seconds_match_bare(self, random_spool): - """A duration in seconds equals the bare seconds value.""" - quant = random_spool.chunk(time=3 * dc.units.s) - bare = random_spool.chunk(time=3) - assert len(quant) == len(bare) - assert [x.shape for x in quant] == [x.shape for x in bare] - - def test_other_time_unit(self, random_spool): - """Milliseconds convert to the same chunk as seconds.""" - out = random_spool.chunk(time=3000 * dc.units.ms) - assert len(out) == len(random_spool.chunk(time=3)) - def test_distance_unit_converts(self, random_spool): """A distance in feet converts to the coordinate's metres.""" out = random_spool.chunk(distance=100 * dc.units.ft) @@ -1282,11 +1203,6 @@ def test_unitless_coord_raises(self, random_spool): with pytest.raises(UnitError, match="no units"): dc.spool(patches).chunk(distance=100 * dc.units.ft) - def test_wrong_dimensionality_raises(self, random_spool): - """A length cannot chunk time.""" - with pytest.raises(UnitError, match="time-like"): - random_spool.chunk(time=100 * dc.units.ft) - class TestSizeChunk: """Chunk lengths expressed as a data size.""" @@ -1302,15 +1218,12 @@ def _make(dtype, start, distance=50, samples=400): } return dc.Patch(data=data, coords=coords, dims=("distance", "time")) - @pytest.fixture(scope="class") - def mixed_dtype_spool(self): - """Two contiguous patches whose element types differ.""" - start = np.datetime64("2020-01-01T00:00:00") - second = start + np.timedelta64(1600, "ms") - return dc.spool([self._make("float64", start), self._make("float32", second)]) - @pytest.mark.parametrize( - "size", ("1 MB", "2 MB", "1 MiB", "500 kB"), ids=lambda x: x.replace(" ", "") + # One decimal unit and one binary one; the four sizes all split the + # spool, so what the other two added was the arithmetic. + "size", + ("1 MiB", "500 kB"), + ids=lambda x: x.replace(" ", ""), ) def test_never_exceeds_request(self, random_spool, size): """Every output patch fits inside the requested size.""" @@ -1338,15 +1251,6 @@ def test_smaller_dtype_gives_more_samples(self): narrow_samples = narrow[0].shape[narrow[0].get_axis("time")] assert narrow_samples == 2 * wide_samples - def test_mixed_dtype_partition_uses_upcast(self, mixed_dtype_spool): - """A mixed partition is sized against the dtype assembly upcasts to.""" - target = dc.get_quantity("100 kB") - plan = mixed_dtype_spool.chunk_plan(time=target) - (part,) = plan.params["size"]["partitions"] - assert part["dtype"] == "float64" - out = mixed_dtype_spool.chunk(time=target) - assert max(x.data.nbytes for x in out) <= target.to("byte").magnitude - def test_mixed_steps_in_one_partition_stay_bounded(self): """ Sizing must use the partition's smallest step, not its median. @@ -1404,10 +1308,14 @@ def make(dtype, start): out = spool.chunk(time=target) assert max(x.data.nbytes for x in out) <= target.to("byte").magnitude - def test_slab_larger_than_target_warns(self, random_spool): + def test_slab_larger_than_target_warns(self): """One sample is the floor; the request cannot be honored below it.""" + # 200 channels of float64 is a 1,600 byte slab, so one sample already + # exceeds the request. The default patch's 400 byte slab would not. + start = np.datetime64("2020-01-01T00:00:00") + spool = dc.spool([self._make("float64", start, distance=200, samples=10)]) with pytest.warns(UserWarning, match="larger than the requested size"): - out = random_spool.chunk(time=1 * dc.units.kB) + out = spool.chunk(time=1 * dc.units.kB) patch = out[0] assert patch.shape[patch.get_axis("time")] == 1 @@ -1536,13 +1444,6 @@ def make(dtype, start): assert claimed == [str(x.data.dtype) for x in out] assert out == dc.spool(list(out)) - def test_merge_mode_records_no_size(self, random_spool): - """A merge takes no length, so no size is ever resolved.""" - plan = random_spool.chunk_plan(time=None) - assert plan.merge_mode - assert "size" not in plan.params - assert len(random_spool.chunk(time=None)) == 1 - def test_merge_mode_rejects_size_overlap(self, random_spool): """A size overlap is still an overlap, which merging forbids.""" with pytest.raises(ParameterError, match="keep_partial and overlap"): diff --git a/tests/test_core/test_spool.py b/tests/test_core/test_spool.py index 007ea3043..bf3a7b2fc 100644 --- a/tests/test_core/test_spool.py +++ b/tests/test_core/test_spool.py @@ -28,7 +28,7 @@ from dascore.io.index.planned import PlanResolver from dascore.io.segy import SegyV1_0 from dascore.utils.downloader import fetch -from dascore.utils.misc import deep_equality_check, suppress_warnings +from dascore.utils.misc import suppress_warnings from dascore.utils.patch_assembly import _estimate_merge_samples, _get_varying_dim from dascore.utils.time import to_datetime64, to_timedelta64 @@ -77,12 +77,6 @@ def test_updated_spool_eq(self, random_spool): """Ensure updating the spool doesn't change equality.""" assert random_spool == random_spool.update() - def test_empty_spool_str(self): - """Ensure and empty spool has a string rep. See #295.""" - spool = dc.spool([]) - spool_str = str(spool) - assert "Spool" in spool_str - def test_spool_with_empty_patch_str(self): """A spool with an empty patch should have a str.""" spool = dc.spool(dc.Patch()) @@ -202,12 +196,6 @@ def test_single_patch_input_uses_lazy_storage(self, random_patch): # simple access never bootstrapped the index backend assert spool._catalog._backend is None - def test_empty_memory_spool(self): - """An empty Spool is a valid, iterable, zero-length spool.""" - spool = Spool() - assert len(spool) == 0 - assert list(spool) == [] - class TestSpoolHelpers: """Tests for helper functions used by spool implementations.""" @@ -461,14 +449,6 @@ def test_unsigned_out_of_bounds_raises(self, random_spool): class TestSpoolIterable: """Tests for iterating Spools.""" - def test_len(self, random_spool): - """Ensure the spool has a length.""" - assert len(random_spool) == len(list(random_spool)) - - def test_index(self, random_spool): - """Ensure the spool can be indexed.""" - assert isinstance(random_spool[0], dc.Patch) - def test_list_o_patches(self, random_spool): """Ensure random_string can be iterated.""" for pa in random_spool: @@ -477,12 +457,6 @@ def test_list_o_patches(self, random_spool): for pa in patch_list: assert isinstance(pa, dc.Patch) - def test_index_error(self, random_spool): - """Ensure an IndexError is raised when indexing beyond spool.""" - spool_len = len(random_spool) - with pytest.raises(IndexError, match="out of bounds"): - _ = random_spool[spool_len] - def test_index_returns_corresponding_patch(self, random_spool): """Ensure the index returns the correct patch.""" spool_list = list(random_spool) @@ -1064,45 +1038,6 @@ def test_dft_patch_access(self, random_dft_patch): assert isinstance(patch, dc.Patch) -class TestDeepEqualityCheck: - """Coverage for deep_equality_check branches (formerly via spool attrs).""" - - def test_non_dict_comparison(self): - """Plain value comparison inside dicts.""" - assert deep_equality_check({"a": "hello"}, {"a": "hello"}) - assert not deep_equality_check({"a": "hello"}, {"a": "world"}) - - def test_objects_with_dict(self): - """Objects compare via recursive __dict__ comparison.""" - - class TestObject: - def __init__(self, value): - self.value = value - - assert deep_equality_check({"o": TestObject(42)}, {"o": TestObject(42)}) - assert not deep_equality_check({"o": TestObject(1)}, {"o": TestObject(2)}) - - def test_mixed_types(self): - """Ints, lists, and numpy arrays compare by value.""" - d1 = {"i": 42, "l": [1, 2, 3], "a": np.array([1, 2, 3])} - d2 = {"i": 42, "l": [1, 2, 3], "a": np.array([1, 2, 3])} - assert deep_equality_check(d1, d2) - d2["a"] = np.array([1, 2, 4]) - assert not deep_equality_check(d1, d2) - - def test_dataframes(self): - """DataFrames compare via .equals.""" - df1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - df2 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - assert deep_equality_check({"df": df1}, {"df": df2}) - df3 = pd.DataFrame({"a": [1, 2, 4], "b": [4, 5, 6]}) - assert not deep_equality_check({"df": df1}, {"df": df3}) - - def test_unequal_sub_dicts(self): - """Nested dicts with different values are unequal.""" - assert not deep_equality_check({"d": {1: 2}}, {"d": {2: 3}}) - - class TestSpoolCoverageEdges: """Cover remaining spool-machinery branches with real operations.""" @@ -1110,11 +1045,16 @@ class TestSpoolCoverageEdges: def many_contiguous(self): """Twelve contiguous patches (for >10-row merge handling).""" t0 = np.datetime64("2020-01-01", "ns") - patch = dc.get_example_patch(time_min=t0) + # Twelve is the point (the de-dup branch needs more than ten rows); + # how much data is in each of them is not. + shape = (10, 50) + patch = dc.get_example_patch(time_min=t0, shape=shape) step = patch.get_coord("time").step out = [patch] for _ in range(11): - nxt = dc.get_example_patch(time_min=out[-1].get_coord("time").max() + step) + nxt = dc.get_example_patch( + time_min=out[-1].get_coord("time").max() + step, shape=shape + ) out.append(nxt) return out @@ -1205,7 +1145,7 @@ def test_merge_buffer_grows_when_estimate_short(self, many_contiguous, monkeypat ) def test_empty_memory_spool_len_iter_repr(self): - """A bare Spool() is a valid empty spool.""" + """A bare Spool() is a valid empty spool, string and all. See #295.""" empty = Spool() assert len(empty) == 0 assert list(empty) == [] diff --git a/tests/test_core/test_spool_contracts.py b/tests/test_core/test_spool_contracts.py index 5bdf9653d..c57942131 100644 --- a/tests/test_core/test_spool_contracts.py +++ b/tests/test_core/test_spool_contracts.py @@ -12,7 +12,6 @@ import pytest import dascore as dc -import dascore.core.spool as spool_module from dascore.exceptions import InvalidSpoolError from dascore.io.index.planned import PlanResolver @@ -156,13 +155,6 @@ def test_every_spool_is_spool(self, patches, tmp_path): for spool in (live, dir_spool, file_spool, live.chunk(time=2)): assert type(spool) is dc.Spool - def test_removed_names_gone(self): - """The old concrete class names are deleted outright.""" - for name in ("MemorySpool", "DirectorySpool", "FileSpool"): - assert not hasattr(spool_module, name) - with pytest.raises(ImportError): - from dascore.clients.dirspool import DirectorySpool # noqa - def test_live_patch_predicate(self, patches, tmp_path): """has_live_patches distinguishes memory content, not class.""" assert dc.spool(patches).has_live_patches diff --git a/tests/test_core/test_spool_gaps.py b/tests/test_core/test_spool_gaps.py index f9b7bd62e..d8f8f9c5c 100644 --- a/tests/test_core/test_spool_gaps.py +++ b/tests/test_core/test_spool_gaps.py @@ -120,11 +120,6 @@ def test_missing_dim_dropped(self, spool_with_non_coords): with pytest.raises(ChunkError, match="lack the dimension"): spool_with_non_coords.get_gaps(missing_dim="raise") - def test_bad_missing_dim_raises(self, gappy_spool): - """A typo in missing_dim raises rather than silently dropping.""" - with pytest.raises(ParameterError, match="missing_dim"): - gappy_spool.get_gaps(missing_dim="rasie") - def test_plan_backed_spool(self, gappy_spool): """A report describes the patches the spool holds, not their sources.""" merged = gappy_spool.concatenate(time=None) @@ -151,14 +146,6 @@ def test_samples_selection_is_measured(self, gappy_spool): # untrimmed spool's assert (out["gap_size"] > gappy_spool.get_gaps()["gap_size"]).all() - def test_group_id_ignores_construction_order(self): - """A group keeps its id however the spool was assembled.""" - early, late = random_spool(tag="a"), random_spool(tag="b") - expected = {"a": 0, "b": 1} - for patches in ([*early, *late], [*late, *early]): - out = dc.spool(patches).get_coverage() - assert dict(zip(out["tag"], out["group_id"])) == expected - def test_group_colliding_with_emitted_column(self, gappy_spool): """Grouping by a column the report emits is refused.""" with pytest.raises(ParameterError, match="collide"): diff --git a/tests/test_core/test_spool_inventory.py b/tests/test_core/test_spool_inventory.py index 36ece2830..6a0c30676 100644 --- a/tests/test_core/test_spool_inventory.py +++ b/tests/test_core/test_spool_inventory.py @@ -195,7 +195,7 @@ def write_inventory(root, files): return root -@pytest.fixture(scope="class") +@pytest.fixture(scope="module") def data_directory(tmp_path_factory, patch, inventory): """A directory of data which carries the inventory describing it.""" path = tmp_path_factory.mktemp("blessed") @@ -615,7 +615,7 @@ def test_equality_never_raises(self, tmp_path, patch, inventory): class TestOnUnresolved: """What a spool does with a patch its inventory does not describe.""" - @pytest.fixture + @pytest.fixture(scope="class") def mixed(self, patch): """A spool of one patch the example inventory knows and one it does not.""" return dc.spool([patch, dc.get_example_patch("random_das")]) diff --git a/tests/test_core/test_spool_select_spec.py b/tests/test_core/test_spool_select_spec.py index 83c5d2d1f..23bdc5b78 100644 --- a/tests/test_core/test_spool_select_spec.py +++ b/tests/test_core/test_spool_select_spec.py @@ -33,7 +33,14 @@ def spool(request, tmp_path_factory): parity net proving one selector engine serves identity and restructured spools alike. """ - base = dc.get_example_spool("random_das") + # 8,000 samples per patch rather than 600,000, at a step which keeps + # each of them 8 seconds long: the specs select windows in seconds, and + # a window narrower than one patch is what several of them are about. + # Distance stays wider than the 10-sample window TestSamples asks for, + # or that trim would be a no-op. + base = dc.get_example_spool( + "random_das", shape=(40, 200), time_step=dc.to_timedelta64(0.04) + ) if request.param.startswith("memory"): out = dc.spool(list(base)) else: @@ -158,9 +165,10 @@ class TestSamples: """samples=True never excludes patches; trims on load (#447).""" def test_length_preserved(self, spool): - """The spool keeps every patch.""" + """The spool keeps every patch, and the window is a real trim.""" out = spool.select(distance=(0, 10), samples=True) assert len(out) == len(spool) + assert len(spool[0].get_coord("distance")) > 10 def test_patch_trimmed_on_load(self, spool): """Loaded patches carry the sample trim.""" diff --git a/tests/test_imports.py b/tests/test_imports.py index 6a5f8d457..1a5665d29 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -6,6 +6,7 @@ import subprocess import sys +from textwrap import dedent import pytest @@ -28,26 +29,50 @@ class TestLazyImports: """Ensure expensive optional machinery is not imported eagerly.""" @pytest.mark.concurrency - def test_matplotlib_not_imported(self): - """Importing dascore should not import matplotlib (it is slow).""" - code = "import dascore, sys; assert 'matplotlib' not in sys.modules" + def test_nothing_expensive_is_imported_eagerly(self): + """Walk one clean interpreter from a bare import through to viz. + + Every check needs a process which has not yet imported what the + check before it pulls in, so they run in order inside one + subprocess rather than one process each. + """ + code = dedent(""" + import sys + import dascore + + for name in ("matplotlib", "scipy.signal", "numba"): + assert name not in sys.modules, name + " imported by dascore" + + from dascore.utils.imports import lazy_import + + hann = lazy_import("scipy.signal.windows", "hann") + assert "scipy.signal" not in sys.modules, "lazy import resolved early" + assert hann.__name__ == "hann", "lazy proxy resolved to the wrong thing" + assert "scipy.signal" in sys.modules, "use did not resolve the proxy" + + try: + dascore.not_a_real_attribute + except AttributeError: + pass + else: + raise AssertionError("AttributeError not raised") + + from dascore import viz + + assert callable(viz.waterfall), "from-import of viz did not work" + assert callable(dascore.viz.waterfall), "viz attribute hook did not work" + assert "matplotlib" in sys.modules, "viz left matplotlib unimported" + """) _run_snippet(code) @pytest.mark.concurrency - def test_scipy_signal_not_imported(self): - """Importing dascore should not import scipy.signal (it is slow).""" - code = "import dascore, sys; assert 'scipy.signal' not in sys.modules" - _run_snippet(code) - - @pytest.mark.concurrency - def test_numba_not_imported(self): - """Importing dascore should not import numba (it is slow).""" - code = "import dascore, sys; assert 'numba' not in sys.modules" - _run_snippet(code) + def test_jit_kernels_import_numba(self): + """The jit kernel modules pull numba in when they are imported. - @pytest.mark.concurrency - def test_numba_imported_with_jit_kernels(self): - """The jit kernel modules should pull numba in when they are imported.""" + Its own subprocess, and its own importorskip: folded into the test + above it would report as passed on a job without numba installed, + where what it says is nothing at all. + """ pytest.importorskip("numba") code = ( "import sys, dascore; " @@ -57,59 +82,16 @@ def test_numba_imported_with_jit_kernels(self): ) _run_snippet(code) - @pytest.mark.concurrency - def test_lazy_import_doesnt_import_scipy_signal_until_use(self): - """The lazy proxy should resolve scipy.signal only on first use.""" - code = ( - "import sys; " - "from dascore.utils.imports import lazy_import; " - "hann = lazy_import('scipy.signal.windows', 'hann'); " - "assert 'scipy.signal' not in sys.modules; " - "assert hann.__name__ == 'hann'; " - "assert 'scipy.signal' in sys.modules" - ) - _run_snippet(code) - def test_lazy_import_proxy_forwards_calls_and_attrs(self): """The lazy proxy should behave like the resolved target object.""" sqrt = lazy_import("math", "sqrt") assert sqrt(4) == 2 assert sqrt.__name__ == "sqrt" - @pytest.mark.concurrency - def test_viz_module_lazy_loads(self): - """Accessing dascore.viz should still work via lazy (PEP 562) import.""" - code = ( - "import dascore; " - "assert callable(dascore.viz.waterfall); " - "import sys; assert 'matplotlib' in sys.modules" - ) - _run_snippet(code) - def test_viz_module_lazy_loads_in_process(self): """Accessing dascore.viz should use the package attribute hook.""" assert callable(dascore.__getattr__("viz").waterfall) - @pytest.mark.concurrency - def test_viz_from_import_still_works(self): - """The package attribute hook should preserve from-import behavior.""" - code = "from dascore import viz; assert callable(viz.waterfall)" - _run_snippet(code) - - @pytest.mark.concurrency - def test_missing_attribute_raises(self): - """Unknown attributes on the package should still raise AttributeError.""" - code = ( - "import dascore\n" - "try:\n" - " dascore.not_a_real_attribute\n" - "except AttributeError:\n" - " pass\n" - "else:\n" - " raise AssertionError('AttributeError not raised')\n" - ) - _run_snippet(code) - def test_missing_attribute_raises_in_process(self): """Unknown package attributes should raise in the parent process too.""" with pytest.raises(AttributeError, match="not_a_real_attribute"): diff --git a/tests/test_io/test_common_io.py b/tests/test_io/test_common_io.py index 34451f41d..cf61f60a9 100644 --- a/tests/test_io/test_common_io.py +++ b/tests/test_io/test_common_io.py @@ -87,7 +87,9 @@ NeubrexDASV1(): ("neubrex_das_1.h5",), NeubrexRFSV1(): ("neubrex_dss_forge.h5", "neubrex_dts_forge.h5"), ODH4V1(): ("optasense_odh4_1.h5",), - OptoDASV8(): ("opto_das_1.hdf5",), + # decimated_optodas.hdf5 is the #419 regression file; it is in the matrix + # so every contract runs over it, not only a read. + OptoDASV8(): ("opto_das_1.hdf5", "decimated_optodas.hdf5"), SR4731V200(): ("ofl100_1.sor", "ofl100_2.sor", "ofl100_3.sor"), ProdMLV2_0(): ("prodml_2.0.h5", "opta_sense_quantx_v2.h5"), ProdMLV2_1(): ( diff --git a/tests/test_io/test_dasdae/test_dasdae.py b/tests/test_io/test_dasdae/test_dasdae.py index 1f32779d3..87ee244f5 100644 --- a/tests/test_io/test_dasdae/test_dasdae.py +++ b/tests/test_io/test_dasdae/test_dasdae.py @@ -4,7 +4,6 @@ import pickle import shutil -from pathlib import Path from typing import ClassVar import h5py @@ -52,15 +51,6 @@ def written_dascore_v1_random(random_patch, tmp_path_factory): return path -@pytest.fixture(scope="class") -@register_func(WRITTEN_FILES) -def written_dascore_v1_random_copy(written_dascore_v1_random, tmp_path_factory): - """Copy the previous DASDAE file for compatibility-oriented tests.""" - new_path = tmp_path_factory.mktemp("dasdae_test_path") / "copied_dasdae.h5" - shutil.copy(written_dascore_v1_random, new_path) - return new_path - - @pytest.fixture(scope="class") @register_func(WRITTEN_FILES) def written_dascore_v1_empty(tmp_path_factory): @@ -83,19 +73,9 @@ def written_dascore_correlate(tmp_path_factory, random_patch): return path -@pytest.fixture(params=WRITTEN_FILES, scope="class") -def dasdae_v1_file_path(request): - """Gatherer fixture to iterate through each written dasdae format.""" - return request.getfixturevalue(request.param) - - class TestWriteDASDAE: """Ensure the format can be written.""" - def test_file_exists(self, dasdae_v1_file_path): - """The file should *of course* exist.""" - assert Path(dasdae_v1_file_path).exists() - def test_append(self, written_dascore_v1_random, tmp_path_factory, random_patch): """Ensure files can be appended to unindexed dasdae file.""" # make a copy of the dasdae file. @@ -112,24 +92,6 @@ def test_append(self, written_dascore_v1_random, tmp_path_factory, random_patch) assert len(df) == len(df_pre) + 1 assert (df["time_min"] == to_datetime64("1990-01-01")).any() - def test_append_after_copy( - self, written_dascore_v1_random_copy, tmp_path_factory, random_patch - ): - """Ensure append still works on a copied DASDAE file.""" - # make a copy of the dasdae file. - new_path = tmp_path_factory.mktemp("dasdae_append") / "tmp.h5" - shutil.copy(written_dascore_v1_random_copy, new_path) - # ensure the patch exists in the copied spool. - df_pre = dc.spool(new_path).get_contents() - assert len(df_pre) == 1 - # append patch to dasdae file - new_patch = random_patch.update_coords(time_min="1990-01-01") - dc.write(new_patch, new_path, "DASDAE") - # ensure the file has grown in contents - df = dc.spool(new_path).get_contents() - assert len(df) == len(df_pre) + 1 - assert (df["time_min"] == to_datetime64("1990-01-01")).any() - def test_write_again(self, written_dascore_v1_random, random_patch): """Ensure a patch can be written again to file (should overwrite old).""" random_patch.io.write(written_dascore_v1_random, "dasdae") @@ -145,28 +107,12 @@ def test_write_cc_patch(self, written_dascore_correlate): class TestReadDASDAE: """Test for reading a dasdae format.""" - def test_round_trip_random_patch(self, random_patch, tmp_path_factory): - """Ensure the random patch can be round-tripped.""" - path = tmp_path_factory.mktemp("dasdae_round_trip") / "rt.h5" - dc.write(random_patch, path, "DASDAE") - out = dc.read(path) - assert len(out) == 1 - assert out[0].equals(random_patch) - def test_round_trip_empty_patch(self, written_dascore_v1_empty): """Ensure an empty patch can be deserialized.""" spool = dc.read(written_dascore_v1_empty) assert len(spool) == 1 spool[0].equals(dc.Patch()) - def test_reads_legacy_fixture(self): - """Legacy DASDAE fixtures still need to remain readable.""" - path = fetch("example_dasdae_event_1.h5") - with config_context(allow_dasdae_format_unpickle=True): - spool = dc.read(path, file_format="DASDAE") - assert len(spool) == 1 - assert spool[0].dims - def test_append_to_legacy_file_keeps_new_attrs( self, random_patch, tmp_path_factory ): @@ -286,22 +232,6 @@ def test_read_filters_patch_attrs_before_loading(self, tmp_path, random_patch): assert len(out) == 1 assert out[0].attrs.tag == "S120" - def test_get_format_false(self, generic_hdf5): - """A generic HDF5 file is not a DASDAE file.""" - parser = DASDAEV1() - assert not parser.get_format(generic_hdf5) - - def test_read_empty_selection_returns_no_patches( - self, tmp_path_factory, random_patch - ): - """Selections outside an empty patch should return no patches.""" - path = tmp_path_factory.mktemp("dasdae_read_empty_selection") / "out.h5" - time = random_patch.get_coord("time") - random_patch.io.write(path, "dasdae") - empty_range_start = time.max() + 3 * time.step - out = dc.read(path, time=(empty_range_start, ...)) - assert len(out) == 0 - class TestScanDASDAE: """Tests for scanning the dasdae format.""" @@ -317,23 +247,6 @@ def test_scan_returns_info(self, written_dascore_v1_random, random_patch): for key in common_keys: assert info1[key] == info2[key] - def test_scan_has_source_patch_key(self, written_dascore_v1_random): - """Scanned DASDAE patches should expose source patch ids.""" - patch = dc.scan(written_dascore_v1_random)[0] - assert patch.source_patch_key - - def test_copied_fixture_matches_original( - self, - written_dascore_v1_random, - written_dascore_v1_random_copy, - ): - """Copying a DASDAE file should not change scan output.""" - df1 = dc.scan_to_df(written_dascore_v1_random) - df2 = dc.scan_to_df(written_dascore_v1_random_copy) - # common fields should be equal (except path) - common = list((set(df1) & set(df2)) - {"source_path"}) - assert df1[common].equals(df2[common]) - def test_get_patch_summary_has_file_metadata(self, random_spool): """The summary helper should stamp DASDAE metadata on each row.""" out = DASDAEV1()._get_patch_summary(random_spool) @@ -871,7 +784,7 @@ def test_roundtrip_datetime_coord(self, tmp_path_factory, random_patch): new = random_patch.update_coords(dt=("distance", dt)) new.io.write(path, "dasdae") patch = dc.spool(path, file_format="DASDAE")[0] - assert isinstance(patch, dc.Patch) + assert patch == new def test_roundtrip_nullish_datetime_coord(self, tmp_path_factory, random_patch): """Ensure a patch with an attached datetime coord with nulls works.""" diff --git a/tests/test_io/test_febus/test_febusbsl.py b/tests/test_io/test_febus/test_febusbsl.py index 5a424aa78..d9ff9da9c 100644 --- a/tests/test_io/test_febus/test_febusbsl.py +++ b/tests/test_io/test_febus/test_febusbsl.py @@ -10,9 +10,8 @@ from numpy.testing import assert_allclose import dascore as dc -from dascore.constants import STORAGE_PROVENANCE_ATTRS from dascore.io.febus import FebusBSLH5V1 -from dascore.io.febus.g1utils import _get_bsl_attrs, _get_g1_h5_base_coords +from dascore.io.febus.g1utils import _get_g1_h5_base_coords from dascore.utils.downloader import fetch BSL_NAME = "febusg1_C2_2026-06-03T17.18.13+0200.bsl.h5" @@ -33,13 +32,6 @@ def bsl_patch(self, bsl_path): """Return the parsed G1 BSL patch.""" return self.parser.read(bsl_path)[0] - def test_get_format(self, bsl_path): - """Ensure the BSL HDF5 format can be auto-detected.""" - assert self.parser.get_format(bsl_path) == ( - self.parser.name, - self.parser.version, - ) - def test_future_format_version_not_claimed(self, bsl_path, tmp_path): """Future BSL format versions should not be claimed by the v1 reader.""" new_path = tmp_path / bsl_path.name @@ -63,14 +55,6 @@ def test_scan(self, bsl_path): assert attr.data_type == "strain" assert attr.data_units == dc.get_quantity("microstrain") - def test_private_attrs_without_io_provenance(self, bsl_path): - """The low-level attrs helper can still omit DASCore IO attrs.""" - with h5py.File(bsl_path) as h5: - attrs = _get_bsl_attrs(h5) - assert "file_format" not in attrs - assert "file_version" not in attrs - assert "path" not in attrs - def test_read(self, bsl_patch): """Ensure the BSL file is read into a patch with expected shape.""" assert isinstance(bsl_patch, dc.Patch) @@ -79,11 +63,6 @@ def test_read(self, bsl_patch): assert "temperature" in bsl_patch.coords.coord_map assert bsl_patch.coords.dim_map["temperature"] == ("time",) - def test_read_attrs_omit_storage_provenance(self, bsl_patch): - """Where the bytes live belongs to the spool, not to patch attrs.""" - names = set(dict(bsl_patch.attrs)) - assert not names & set(STORAGE_PROVENANCE_ATTRS) - def test_distance_range(self, bsl_patch): """Distance should span 50-149 m.""" dist = bsl_patch.get_coord("distance") @@ -206,12 +185,3 @@ def test_select(self, bsl_path, bsl_patch): assert out.get_coord("time").max() == time.values[20] assert_allclose(out.get_coord("distance").min(), 55.0) assert_allclose(out.get_coord("distance").max(), 60.0) - - def test_out_of_range_selects_empty_spool(self, bsl_path, bsl_patch): - """Out-of-range time and distance selections should return empty spools.""" - time = bsl_patch.get_coord("time") - dist = bsl_patch.get_coord("distance") - assert not len( - self.parser.read(bsl_path, time=(time.max() + np.timedelta64(1, "s"), ...)) - ) - assert not len(self.parser.read(bsl_path, distance=(dist.max() + 1, ...))) diff --git a/tests/test_io/test_febus/test_febusg1.py b/tests/test_io/test_febus/test_febusg1.py index afd41773e..8ffff5ea7 100644 --- a/tests/test_io/test_febus/test_febusg1.py +++ b/tests/test_io/test_febus/test_febusg1.py @@ -14,7 +14,6 @@ import pytest import dascore as dc -from dascore.constants import STORAGE_PROVENANCE_ATTRS from dascore.io.febus.core import FebusG1CSV1, FebusMTXH5V1 from dascore.io.febus.g1utils import _is_g1_file from dascore.utils.downloader import fetch @@ -130,26 +129,9 @@ def test_public_scan_adds_source_metadata(self, g1_path): assert attr.source_version == g1.version -class TestG1Read: - """Tests for reading G1 files into patches.""" - - def test_read(self, g1_path): - """Ensure a G1 file is read into a Patch with expected data.""" - spool = dc.read(g1_path) - assert len(spool) == 1 - patch = spool[0] - assert isinstance(patch, dc.Patch) - - class TestG1MTXH5: """Tests for Brillouin spectrum HDF5 files.""" - def test_get_format(self, mtx_h5_path): - """Ensure the MTX HDF5 format can be auto-detected.""" - fiber = FebusMTXH5V1() - assert fiber.get_format(mtx_h5_path) == (fiber.name, fiber.version) - assert dc.get_format(mtx_h5_path) == (fiber.name, fiber.version) - def test_read(self, mtx_h5_path): """Ensure MTX HDF5 data are read into a 3D patch.""" patch = dc.read(mtx_h5_path)[0] @@ -182,21 +164,6 @@ def test_read_preserves_mtx_array_order(self, tmp_path): np.testing.assert_array_equal(patch.data, stored) np.testing.assert_allclose(frequency.values, expected_frequency) - def test_read_attrs_omit_storage_provenance(self, mtx_h5_path): - """Where the bytes live belongs to the spool, not to patch attrs.""" - patch = dc.read(mtx_h5_path)[0] - names = set(dict(patch.attrs)) - assert not names & set(STORAGE_PROVENANCE_ATTRS) - - def test_scan_matches_read_attrs(self, mtx_h5_path): - """Scan and read should return matching coord summaries.""" - summary = dc.scan(mtx_h5_path)[0] - patch_summary = dc.read(mtx_h5_path)[0].summary - assert summary.dims == patch_summary.dims - assert summary.coords == patch_summary.coords - assert summary.source_format == FebusMTXH5V1.name - assert summary.source_version == FebusMTXH5V1.version - def test_selects(self, mtx_h5_path): """Read supports selecting along all three dimensions.""" fiber = FebusMTXH5V1() @@ -265,9 +232,3 @@ def test_mtx_scan_warns(self, g1_mtx_buffer): fiber = FebusG1CSV1() with pytest.warns(UserWarning, match=self.mtx_text): fiber.scan(g1_mtx_buffer) - - def test_directory_spool(self, two_patch_directory): - """Ensure a directory spool works and can read files.""" - spool = dc.spool(two_patch_directory).update() - patch = spool[0] - assert isinstance(patch, dc.Patch) diff --git a/tests/test_io/test_hdas/test_hdas.py b/tests/test_io/test_hdas/test_hdas.py index baa27d235..9ae65d24a 100644 --- a/tests/test_io/test_hdas/test_hdas.py +++ b/tests/test_io/test_hdas/test_hdas.py @@ -83,7 +83,12 @@ def test_orientation(self, hdas_v2_patch): assert not hdas_v2_patch.attrs.data_units def test_v1_does_not_claim(self, hdas_v2_path): - """The V1 reader must not claim a V2 file.""" + """The V1 reader must not claim a V2 file. + + The common contract cannot say this: HDASV2 subclasses HDASV1, so + test_all_other_files_arent_format skips the V2 files while it is + testing V1. + """ assert not HDASV1().get_format(hdas_v2_path) diff --git a/tests/test_io/test_index/test_catalog.py b/tests/test_io/test_index/test_catalog.py index 295be639c..416c01227 100644 --- a/tests/test_io/test_index/test_catalog.py +++ b/tests/test_io/test_index/test_catalog.py @@ -444,46 +444,30 @@ def test_membership_order_survives_a_predicate(self): class TestGlobTranslation: """The in-memory glob has to mean what the index's GLOB means.""" - @pytest.mark.parametrize( - "pattern", - [ - "a*", - "a?c", - "*", - "?", - "v[^x]*", - "v[!x]*", - "[abc]d", - "[]]x", - "a[b-d]e", - "[^]]a", - "x\\y*", - "no_meta", - "a[", - "[]", - "[z-a]", - "[^z-a]", - "[]-a]", - "[^]-a]", - "[]a]", - ], - ) - def test_agrees_with_sqlite(self, pattern): + def test_agrees_with_sqlite(self): """ SQLite decides what a glob means, since it is what answers one. Reaching for fnmatch instead made `[!x]` and `[^x]` each select the half of a spool the other did not. + + The patterns loop inside one test rather than parametrizing it: + each is a fifth of a millisecond, and the assertion already names + the pattern and value which disagreed. """ + patterns = ["a*", "a?c", "*", "?", "v[^x]*", "v[!x]*", "[abc]d", "[]]x"] + patterns += ["a[b-d]e", "[^]]a", "x\\y*", "no_meta", "a[", "[]", "[z-a]"] + patterns += ["[^z-a]", "[]-a]", "[^]-a]", "[]a]"] values = ["abc", "a1c", "vax", "vxx", "ad", "]x", "ace", "", "a", "a["] values += ["x\\yz", "!a", "^a", "]a", "no_meta", "[]", "-", "]", "_"] with sqlite3.connect(":memory:") as connection: - regex = glob_to_regex(pattern) - for value in values: - expected = connection.execute( - "SELECT ? GLOB ?", (value, pattern) - ).fetchone()[0] - assert bool(expected) == bool(regex.match(value)), (pattern, value) + for pattern in patterns: + regex = glob_to_regex(pattern) + for value in values: + expected = connection.execute( + "SELECT ? GLOB ?", (value, pattern) + ).fetchone()[0] + assert bool(expected) == bool(regex.match(value)), (pattern, value) def test_a_reversed_range_matches_its_low_endpoint(self): """ diff --git a/tests/test_io/test_index/test_heterogeneity_stress.py b/tests/test_io/test_index/test_heterogeneity_stress.py index a09834fec..2e50cfaf5 100644 --- a/tests/test_io/test_index/test_heterogeneity_stress.py +++ b/tests/test_io/test_index/test_heterogeneity_stress.py @@ -158,7 +158,7 @@ def summaries(): return make_random_summaries(N_PATCHES, seed=42) -@pytest.fixture() +@pytest.fixture(scope="module") def backend(tmp_path_factory, summaries): """A SQLite backend ingesting the random population.""" path = tmp_path_factory.mktemp("stress") / "index.sqlite3" diff --git a/tests/test_io/test_index/test_index_contract.py b/tests/test_io/test_index/test_index_contract.py index 4a3e95eb7..d488314bb 100644 --- a/tests/test_io/test_index/test_index_contract.py +++ b/tests/test_io/test_index/test_index_contract.py @@ -564,13 +564,14 @@ def test_pivot_respects_query(self, backend): class TestLineageIds: """A patch can be found by the id it carries, without loading it.""" - @pytest.fixture - def written_spool(self, tmp_path): + @pytest.fixture(scope="class") + def written_spool(self, tmp_path_factory): """Three patches on disk, each its own datum.""" + path = tmp_path_factory.mktemp("lineage_ids") with config_context(patch_provenance="disabled"): for index, patch in enumerate(dc.get_example_spool("random_das")): - patch.io.write(tmp_path / f"{index}.h5", "dasdae") - return dc.spool(tmp_path).update() + patch.io.write(path / f"{index}.h5", "dasdae") + return dc.spool(path).update() def test_the_two_ids_are_different_columns(self, written_spool): """The row's id is private; the patch's owns the public name.""" diff --git a/tests/test_io/test_index/test_plan.py b/tests/test_io/test_index/test_plan.py index f52381abf..3c22731e6 100644 --- a/tests/test_io/test_index/test_plan.py +++ b/tests/test_io/test_index/test_plan.py @@ -75,11 +75,6 @@ def test_merge_mode_forbids_overlap(self, random_flat): with pytest.raises(ParameterError, match="merging"): build_chunk_plan(random_flat, time=..., keep_partial=True) - def test_overlap_ge_length_raises(self, random_flat): - """D6: overlap >= length raises cleanly.""" - with pytest.raises(ParameterError, match="overlap"): - build_chunk_plan(random_flat, time=2, overlap=2) - def test_unknown_group_raises(self, random_flat): """Explicit group names must exist somewhere in the spool.""" with pytest.raises(InvalidSpoolQueryError, match="bob"): @@ -112,11 +107,6 @@ def test_contiguous_spool_single_output(self, random_flat): assert len(plan.members) == len(random_flat) assert set(plan.members["_patch_id"]) == set(random_flat["_patch_id"]) - def test_members_unmodified_when_contiguous(self, random_flat): - """Contiguous members load whole (no trims).""" - plan = build_chunk_plan(random_flat, time=None) - assert not plan.members["_modified"].any() - def test_diverse_partitions(self, diverse_flat): """The diverse spool partitions by identity attrs, never raising.""" plan = build_chunk_plan(diverse_flat, time=None) @@ -193,13 +183,6 @@ def test_too_short_partition_skipped(self): with pytest.raises(ChunkError, match="sufficient length"): build_chunk_plan(df, time=100) - def test_overlap(self, random_flat): - """Overlapping chunks step by length minus overlap.""" - plan = build_chunk_plan(random_flat, time=4, overlap=2) - starts = plan.outputs["time_min"].sort_values().values - strides = np.diff(starts) - assert (abs(strides - to_timedelta64(2)) <= to_timedelta64(0.01)).all() - def test_middle_value_step(self): """D7: the partition step is the middle value of member steps.""" t0 = np.datetime64("2020-01-01", "ns") @@ -210,28 +193,6 @@ def test_middle_value_step(self): assert plan.outputs["time_step"].iloc[0] == time.step -class TestMissingDim: - """Spec section 7 (D2): patches lacking the chunk dim.""" - - @pytest.fixture() - def flat_with_null(self, random_flat): - """A flat relation with one null time envelope.""" - df = random_flat.copy() - df.loc[df.index[0], ["time_min", "time_max"]] = (pd.NaT, pd.NaT) - return df - - def test_raise_by_default(self, flat_with_null): - """Null chunk-dim envelopes raise by default.""" - with pytest.raises(ChunkError, match="missing_dim"): - build_chunk_plan(flat_with_null, time=None) - - def test_drop_opt_in(self, flat_with_null): - """missing_dim='drop' excludes the offending rows.""" - plan = build_chunk_plan(flat_with_null, time=None, missing_dim="drop") - dropped = flat_with_null["_patch_id"].iloc[0] - assert dropped not in set(plan.members["_patch_id"]) - - class TestConflict: """Spec 2.5: attr policing within a partition.""" @@ -290,11 +251,6 @@ def test_drop(self, conflicted_patches): plan = build_chunk_plan(_flat(conflicted_patches), time=None, conflict="drop") assert "data_units" not in plan.outputs.columns - def test_unknown_policy_raises(self, conflicted_patches): - """A misspelled conflict policy cannot silently behave like drop.""" - with pytest.raises(ParameterError, match="conflict must be"): - build_chunk_plan(_flat(conflicted_patches), time=None, conflict="keep_fist") - class TestGroupParameter: """Group attrs partition instead of raising.""" @@ -552,17 +508,6 @@ def test_unknown_steps_share_one_group(self): assert labels.nunique() == 2 assert labels.iloc[1] == labels.iloc[2] - def test_descending_contiguous_merges(self): - """Contiguous descending patches produce a single merge output.""" - p = dc.get_example_patch() - flipped = p.flip("time") - t = p.get_coord("time") - span = t.max() - t.min() + t.step - shifted = flipped.update_coords(time=flipped.get_coord("time").data + span) - plan = dc.spool([shifted, flipped]).chunk_plan(time=None) - assert len(plan.outputs) == 1 - assert len(plan.members) == 2 - class TestSamplesAdjustedEnvelopes: """Samples residual envelope adjustment (2026-07-18 F4).""" diff --git a/tests/test_io/test_index/test_union.py b/tests/test_io/test_index/test_union.py index 4234bf245..a55b10806 100644 --- a/tests/test_io/test_index/test_union.py +++ b/tests/test_io/test_index/test_union.py @@ -26,12 +26,13 @@ def contiguous_patches(): return p1, p2 -@pytest.fixture() -def dir_spool(contiguous_patches, tmp_path): +@pytest.fixture(scope="module") +def dir_spool(contiguous_patches, tmp_path_factory): """A directory spool holding the first patch.""" p1, _ = contiguous_patches - dc.write(p1, tmp_path / "a.h5", "dasdae") - return dc.spool(tmp_path).update() + path = tmp_path_factory.mktemp("union_dir_spool") + dc.write(p1, path / "a.h5", "dasdae") + return dc.spool(path).update() class TestMemoryUnion: diff --git a/tests/test_io/test_indexer.py b/tests/test_io/test_indexer.py index a4e492b07..08fb56e64 100644 --- a/tests/test_io/test_indexer.py +++ b/tests/test_io/test_indexer.py @@ -400,13 +400,22 @@ def test_noop_update_rescans_nothing(self, basic_indexer): after = basic_indexer._backend.get_sources()["last_indexed_ns"].max() assert before == after - def test_update_with_specific_paths(self, basic_indexer): + def test_update_with_specific_paths(self, two_patch_directory, tmp_path_factory): """Updating with specific paths restricts the rescan.""" - files = sorted(basic_indexer.path.rglob("*.hdf5")) + # Its own copy: this test changes the files' modification times, and + # the directory fixture is shared with the rest of the session. + directory = tmp_path_factory.mktemp("specific_paths") / "data" + shutil.copytree(two_patch_directory, directory) + # Whatever indexed the shared directory before left its index in it, + # and the timestamps below would then be that run's, not this one's. + for index in Path(directory).glob(".dascore_index*"): + index.unlink() + indexer = DBDirectoryIndexer(directory).update(progress=None) + files = sorted(indexer.path.rglob("*.hdf5")) assert len(files) >= 2 def _indexed_times(): - sources = basic_indexer._backend.get_sources().set_index("source_path") + sources = indexer._backend.get_sources().set_index("source_path") return sources["last_indexed_ns"].to_dict() before = _indexed_times() @@ -414,16 +423,17 @@ def _indexed_times(): stat = path.stat() os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) - first, second = (basic_indexer._rel(path) for path in files[:2]) - basic_indexer.update(paths=[files[0].name], progress=None) + first, second = (indexer._rel(path) for path in files[:2]) + indexer.update(paths=[files[0].name], progress=None) after_relative = _indexed_times() assert after_relative[first] > before[first] assert after_relative[second] == before[second] - basic_indexer.update(paths=[str(files[1])], progress=None) + indexer.update(paths=[str(files[1])], progress=None) after_absolute = _indexed_times() assert after_absolute[first] == after_relative[first] assert after_absolute[second] > after_relative[second] + indexer.close() class TestNameResolution: diff --git a/tests/test_io/test_io_core.py b/tests/test_io/test_io_core.py index 35eb95444..e7b831539 100644 --- a/tests/test_io/test_io_core.py +++ b/tests/test_io/test_io_core.py @@ -989,13 +989,19 @@ def nested_directory_with_patches(self, tmpdir_factory, random_patch): random_patch.io.write(path_3, "dasdae") return out + @pytest.fixture(scope="class") + def two_files(self, tmp_path_factory, random_patch): + """Two patch files, for the tests which only scan them.""" + path = tmp_path_factory.mktemp("two_files") + paths = (path / "patch_1.h5", path / "patch_2.h5") + for each in paths: + random_patch.io.write(each, "dasdae") + return paths + @pytest.mark.parametrize("func", [dc.scan, dc.scan_to_df, dc.scan_payloads]) - def test_scan_accepts_a_collection(self, func, tmp_path, random_patch): + def test_scan_accepts_a_collection(self, func, two_files): """A collection of resources scans as the sum of its members.""" - path_1 = tmp_path / "patch_1.h5" - path_2 = tmp_path / "patch_2.h5" - random_patch.io.write(path_1, "dasdae") - random_patch.io.write(path_2, "dasdae") + path_1, path_2 = two_files expected = len(func(path_1)) + len(func(path_2)) assert len(func([path_1, path_2])) == expected # A set is a collection too, and the dispatcher does not index. @@ -1006,12 +1012,9 @@ def test_scan_accepts_a_collection_of_patches(self, random_patch): assert len(dc.scan([random_patch, random_patch])) == 2 @pytest.mark.parametrize("func", [dc.scan, dc.scan_to_df, dc.scan_payloads]) - def test_scan_accepts_a_one_shot_iterable(self, func, tmp_path, random_patch): + def test_scan_accepts_a_one_shot_iterable(self, func, two_files): """A generator input scans every element, not silently nothing (#818).""" - path_1 = tmp_path / "patch_1.h5" - path_2 = tmp_path / "patch_2.h5" - random_patch.io.write(path_1, "dasdae") - random_patch.io.write(path_2, "dasdae") + path_1, path_2 = two_files expected = len(func([path_1, path_2])) assert expected == 2 assert len(func(p for p in [path_1, path_2])) == expected diff --git a/tests/test_io/test_mseed/test_mseed.py b/tests/test_io/test_mseed/test_mseed.py index c2369fb5b..a67f44739 100644 --- a/tests/test_io/test_mseed/test_mseed.py +++ b/tests/test_io/test_mseed/test_mseed.py @@ -180,10 +180,6 @@ def test_get_format_v2(self, mseed_v2_path): """MiniSEED v2 files can be detected.""" assert MSeedV2().get_format(mseed_v2_path) == ("MSEED", "2") - def test_get_format_v3(self, mseed_v3_path): - """MiniSEED v3 files can be detected.""" - assert MSeedV2().get_format(mseed_v3_path) == ("MSEED", "3") - def test_get_format_from_dascore(self, mseed_v3_path): """DASCore can detect MiniSEED files through plugin discovery.""" assert dc.get_format(mseed_v3_path) == ("MSEED", "3") @@ -604,6 +600,8 @@ class Record: assert mseed_utils._record_dtype(Record()) == "float64" + # One encoding per distinct dtype the table maps to, plus one it does + # not know: repeating int32 for 3, 10 and 11 says the same thing thrice. @pytest.mark.parametrize( ("encoding", "dtype"), ( @@ -612,8 +610,6 @@ class Record: (3, "int32"), (4, "float32"), (5, "float64"), - (10, "int32"), - (11, "int32"), (999, ""), ), ) diff --git a/tests/test_io/test_netcdf/test_netcdf.py b/tests/test_io/test_netcdf/test_netcdf.py index dcac3bb71..e8e8f2086 100644 --- a/tests/test_io/test_netcdf/test_netcdf.py +++ b/tests/test_io/test_netcdf/test_netcdf.py @@ -19,7 +19,6 @@ get_cf_version, is_netcdf4_file, ) -from dascore.utils.downloader import fetch from dascore.utils.remote_io import ( clear_remote_file_cache, get_remote_cache_path, @@ -231,15 +230,6 @@ def test_is_netcdf4_file_from_dimension_list_attr(self, tmp_path): with h5py.File(path, "r") as h5file: assert not is_netcdf4_file(h5file) - def test_get_cf_version_decodes_bytes(self, tmp_path): - """CF version extraction should decode byte attrs.""" - path = tmp_path / "cf_version_bytes.nc" - with h5py.File(path, "w") as h5file: - h5file.attrs["Conventions"] = np.bytes_("CF-1.9") - - with h5py.File(path, "r") as h5file: - assert get_cf_version(h5file) == "1.9" - class TestNetCDFCoreHelpers: """Direct tests for lightweight NetCDF core helpers.""" @@ -561,10 +551,6 @@ def _cached_file_count() -> int: assert patch == example_patch assert _cached_file_count() == 0, "read should stream, not download" - def test_get_format(self, minimal_cf_netcdf_path): - """Test format detection.""" - assert dc.get_format(minimal_cf_netcdf_path) == ("NETCDF_CF", "1.8") - def test_get_format_without_xarray_import( self, minimal_cf_netcdf_path, monkeypatch ): @@ -580,55 +566,6 @@ def _import_module(name, package=None): assert dc.get_format(minimal_cf_netcdf_path) == ("NETCDF_CF", "1.8") - def test_get_format_rejects_silixa_carina_hdf5(self): - """ - NETCDF_CF must not claim Silixa Carina/iDAS HDF5 files. - - These files (e.g. the INGV Mt Etna deployment) are written through a - netCDF library, so they carry _NCProperties and dimension scales, but - their netCDF coordinate variables are empty or zeroed; the usable - metadata lives in Silixa attrs on the root. They have no Conventions - attr, so the version requirement in get_format must reject them. - """ - path = fetch("silixa_h5_ingv_1.h5") - formatter = netcdf_core.NetCDFCFV18() - with h5py.File(path, "r") as h5file: - assert is_netcdf4_file(h5file) - assert get_cf_version(h5file) is None - assert formatter.get_format(h5file) is False - - def test_round_trip(self, example_patch, tmp_path): - """Test round-trip: patch -> NetCDF -> patch.""" - _require_xarray_netcdf_engine() - path = tmp_path / "roundtrip.nc" - - # Write and read back - dc.write(example_patch, path, file_format="netcdf_cf") - spool = dc.read(path, file_format="netcdf_cf") - recovered_patch = spool[0] - - # Check data preservation - np.testing.assert_array_almost_equal( - example_patch.data, recovered_patch.data, decimal=6 - ) - - # Check coordinate preservation - for coord_name in example_patch.coords.coord_map: - orig_coord = example_patch.coords.get_array(coord_name) - recovered_coord = recovered_patch.coords.get_array(coord_name) - - if coord_name == "time": - # Time coordinates might have slight precision differences - # due to CF time conversion (float64 seconds -> datetime64[ns]) - time_diff = np.abs(orig_coord - recovered_coord) - assert np.all( - time_diff < np.timedelta64(200, "us") - ) # 200 microsecond tolerance - else: - np.testing.assert_array_almost_equal( - orig_coord, recovered_coord, decimal=6 - ) - class TestNetCDFXarrayCompatibility: """Tests for compatibility between DASCore NetCDF output and xarray.""" @@ -756,15 +693,6 @@ def multi_patch_spool(self): patch2 = dc.get_example_patch("random_das") return dc.spool([patch1, patch2]) - @pytest.fixture - def invalid_hdf5_file(self, tmp_path): - """Create an invalid HDF5 file (not NetCDF).""" - path = tmp_path / "invalid.nc" - with h5py.File(path, "w") as h5file: - rng = np.random.default_rng() - h5file.create_dataset("random_data", data=rng.standard_normal((100, 50))) - return path - @pytest.fixture def compressed_netcdf_file(self, tmp_path): """Create a compressed NetCDF file for testing.""" @@ -797,11 +725,6 @@ def test_multi_patch_write_error(self, multi_patch_spool, tmp_path): ): dc.write(multi_patch_spool, path, file_format="netcdf_cf") - def test_invalid_netcdf_file(self, invalid_hdf5_file): - """Test behavior with invalid NetCDF file.""" - with h5py.File(invalid_hdf5_file, "r") as h5file: - assert not is_netcdf4_file(h5file) - def test_compression_options(self, compressed_netcdf_file): """Test NetCDF file creation with compression options.""" path, original_patch = compressed_netcdf_file diff --git a/tests/test_io/test_optodas/test_optodas.py b/tests/test_io/test_optodas/test_optodas.py index 912b8dab8..d771ca5aa 100644 --- a/tests/test_io/test_optodas/test_optodas.py +++ b/tests/test_io/test_optodas/test_optodas.py @@ -10,19 +10,6 @@ class TestOptoDASIssues: """Test case related to issues in OptoDAS parser.""" - def test_read_decimated_patch(self): - """Tests for reading spatially decimated patch (#419)""" - path = fetch("decimated_optodas.hdf5") - fiber_io = OptoDASV8() - - fmt_str, version_str = fiber_io.get_format(path) - assert (fmt_str, version_str) == (fiber_io.name, fiber_io.version) - - spool = fiber_io.read(path) - patch = spool[0] - assert isinstance(patch, dc.Patch) - assert patch.data.shape - def test_scan_distance_units_preserved(self): """Snapped and exact scan coordinates should retain header units.""" path = fetch("decimated_optodas.hdf5") diff --git a/tests/test_io/test_pickle/test_pickle.py b/tests/test_io/test_pickle/test_pickle.py index b22778eac..cb8bdafcc 100644 --- a/tests/test_io/test_pickle/test_pickle.py +++ b/tests/test_io/test_pickle/test_pickle.py @@ -34,14 +34,6 @@ def test_not_pickle(self, generic_hdf5): parser = PickleIO() assert not parser.get_format(generic_hdf5) - def test_read_pickle(self, pickle_patch_path, random_patch): - """Ensure a pickle file can be read.""" - out = dc.read(pickle_patch_path) - assert isinstance(out, dc.BaseSpool) - assert len(out) == 1 - assert isinstance(out[0], dc.Patch) - assert random_patch == out[0] - def test_spool_from_pickle(self, pickle_patch_path, random_patch): """dc.spool on a scanless format wraps the read spool and serves it. diff --git a/tests/test_io/test_prodml/test_prod_ml.py b/tests/test_io/test_prodml/test_prod_ml.py index 3d8d01d03..f843cc2b4 100644 --- a/tests/test_io/test_prodml/test_prod_ml.py +++ b/tests/test_io/test_prodml/test_prod_ml.py @@ -73,11 +73,6 @@ def silixa_h5_patch(self, idas_h5_example_path): """Get the silixa file, return Patch.""" return dc.spool(idas_h5_example_path)[0] - def test_read_silixa(self, silixa_h5_patch): - """Ensure we can read Silixa file.""" - assert isinstance(silixa_h5_patch, dc.Patch) - assert silixa_h5_patch.shape - def test_has_gauge_length(self, silixa_h5_patch): """Ensure gauge-length is found in patch attrs.""" patch = silixa_h5_patch diff --git a/tests/test_io/test_remote_common_io.py b/tests/test_io/test_remote_common_io.py index 04ba09a07..afbf36f47 100644 --- a/tests/test_io/test_remote_common_io.py +++ b/tests/test_io/test_remote_common_io.py @@ -8,7 +8,7 @@ import dascore as dc from dascore.utils.downloader import fetch -from dascore.utils.misc import suppress_warnings +from dascore.utils.misc import iterate, suppress_warnings from tests.test_io._common_io_test_utils import ( get_flat_io_test, get_representative_io_test, @@ -33,16 +33,39 @@ ), ] -# Sintela protobuf walks its MTLV envelope with three small sequential reads -# per record (magic, header, payload), so a modest file issues hundreds of -# reads. That is fine locally and over memory://, but each read becomes a -# request on the localhost-HTTP range-streaming path, which blows the timeouts -# below. Remote coverage for this format stays at the memory:// level. +# What the remote matrix is for is the streaming path, not the readers: every +# reader is already read, scanned and format-detected against the same files +# by tests/test_io/test_common_io.py. These nine cover the ways a reader can +# reach the bytes -- whole-file HDF5, ranged HDF5, a plain binary walk, a +# SEG-Y trace scan, an obspy handoff -- plus NETCDF_CF, the only one which +# unwraps the handle through get_h5py_file into h5netcdf. +# +# Sintela_Protobuf is deliberately not among them: it walks its MTLV envelope +# with three small sequential reads per record, so a modest file becomes +# hundreds of range requests and blows the timeouts below. Its remote +# coverage stays at the memory:// level. +REMOTE_FORMATS = { + ("PRODML", "2.1"), + ("DASDAE", "1"), + ("TDMS", "4713"), + ("sentek", "5"), + ("Sintela_Binary", "3"), + ("SR4731", "200"), + ("segy", "1.0"), + ("MSEED", "2"), + ("NETCDF_CF", "1.8"), +} +# One file each: what is under test is the streaming path, and a second file +# of the same format goes down the same one. REMOTE_COMMON_IO_READ_TESTS = { - io: fetch_names + io: next(iter(iterate(fetch_names))) for io, fetch_names in COMMON_IO_READ_TESTS.items() - if io.name != "Sintela_Protobuf" + if (io.name, io.version) in REMOTE_FORMATS } +# A rename or a version bump would otherwise drop that format out of the +# matrix silently, leaving a shorter run and no failure. +_matched = {(io.name, io.version) for io in REMOTE_COMMON_IO_READ_TESTS} +assert _matched == REMOTE_FORMATS, f"no reader for {sorted(REMOTE_FORMATS - _matched)}" REMOTE_GET_FORMAT_CASES = get_flat_io_test(REMOTE_COMMON_IO_READ_TESTS) REMOTE_REPRESENTATIVE_CASES = get_representative_io_test(REMOTE_COMMON_IO_READ_TESTS) diff --git a/tests/test_io/test_remote_memory.py b/tests/test_io/test_remote_memory.py index b3d07cd1e..23da5a139 100644 --- a/tests/test_io/test_remote_memory.py +++ b/tests/test_io/test_remote_memory.py @@ -192,10 +192,10 @@ class TestMemoryRemoteMetadataAccess: @pytest.mark.parametrize( ("fetch_name", "expected"), [ + # One HDF5 reader and one which walks a plain binary file: what + # this is about is the cache, not the readers. ("h5_simple_2.h5", ("H5Simple", "1")), ("sample_tdms_file_v4713.tdms", ("TDMS", "4713")), - ("DASDMSShot00_20230328155653619.das", ("sentek", "5")), - ("sintela_binary_v3_test_1.raw", ("Sintela_Binary", "3")), ], ) def test_get_format_avoids_local_cache( @@ -211,8 +211,6 @@ def test_get_format_avoids_local_cache( [ ("h5_simple_2.h5", ("H5Simple", "1")), ("sample_tdms_file_v4713.tdms", ("TDMS", "4713")), - ("DASDMSShot00_20230328155653619.das", ("sentek", "5")), - ("sintela_binary_v3_test_1.raw", ("Sintela_Binary", "3")), ], ) def test_scan_avoids_local_cache(self, fetch_name, expected, memory_fetch_copy): diff --git a/tests/test_io/test_sintela/test_protobuf.py b/tests/test_io/test_sintela/test_protobuf.py index 9e455a73e..0413a83b8 100644 --- a/tests/test_io/test_sintela/test_protobuf.py +++ b/tests/test_io/test_sintela/test_protobuf.py @@ -880,7 +880,10 @@ def test_timeseries_scan_rejects_missing_time( ): fiber_io.scan(path) - @pytest.mark.parametrize("bad_sample_rate", [0.0, -1.0, np.nan, np.inf]) + # The check is `not isfinite(x) or x <= 0`. Zero and a negative are not + # the same case: `< 0` would let zero through, `== 0` would let the + # negative through, and neither would notice the non-finite half. + @pytest.mark.parametrize("bad_sample_rate", [0.0, -1.0, np.inf]) def test_timeseries_scan_rejects_invalid_sample_rate( self, fiber_io, write_sintela_file, ts_records, bad_sample_rate ): @@ -899,7 +902,7 @@ def test_timeseries_scan_rejects_invalid_sample_rate( ): fiber_io.scan(path) - @pytest.mark.parametrize("bad_spacing", [0.0, -1.0, np.nan, np.inf]) + @pytest.mark.parametrize("bad_spacing", [0.0, -1.0, np.inf]) def test_scan_rejects_invalid_channel_spacing( self, fiber_io, write_sintela_file, ts_records, bad_spacing ): @@ -1086,7 +1089,7 @@ def test_fft_read_rejects_bad_sizes( with pytest.raises(InvalidFiberFileError, match="FFT payload size"): fiber_io.read(path) - @pytest.mark.parametrize("bad_bin_res", [0.0, -1.0, np.nan, np.inf]) + @pytest.mark.parametrize("bad_bin_res", [0.0, -1.0, np.inf]) def test_fft_scan_rejects_invalid_bin_res( self, fiber_io, write_sintela_file, fft_records, bad_bin_res ): diff --git a/tests/test_io/test_sr4731/test_sr4731.py b/tests/test_io/test_sr4731/test_sr4731.py index 57d4a9262..74e2f9fcb 100644 --- a/tests/test_io/test_sr4731/test_sr4731.py +++ b/tests/test_io/test_sr4731/test_sr4731.py @@ -156,13 +156,6 @@ def sor_patch(self, sor_path): """Return the parsed SR-4731 patch.""" return self.parser.read(sor_path)[0] - def test_get_format(self, sor_path): - """Ensure the SOR file is identified.""" - assert self.parser.get_format(sor_path) == ( - self.parser.name, - self.parser.version, - ) - def test_scan(self, sor_path): """Scan returns expected SR-4731 metadata.""" fixed = _expected_fixed_values(_get_block_payload(sor_path, "FxdParams")) @@ -264,21 +257,6 @@ def test_select(self, sor_path, sor_patch): assert_allclose(out.get_coord("distance").min(), distance.values[5]) assert_allclose(out.get_coord("distance").max(), distance.values[10]) - def test_out_of_range_selects_empty_spool(self, sor_path, sor_patch): - """Out-of-range selectors return an empty spool.""" - time = sor_patch.get_coord("time") - distance = sor_patch.get_coord("distance") - assert not len( - self.parser.read(sor_path, time=(time.max() + np.timedelta64(1, "s"), ...)) - ) - assert not len(self.parser.read(sor_path, distance=(distance.max() + 1, ...))) - - def test_read_stream(self, sor_path, sor_patch): - """BytesIO streams can be read.""" - bio = BytesIO(sor_path.read_bytes()) - out = self.parser.read(bio)[0] - assert out.equals(sor_patch) - def test_get_format_false_for_version_mismatch(self, sor_path): """A valid SOR with the wrong map version should not be claimed.""" data = bytearray(sor_path.read_bytes()) diff --git a/tests/test_io/test_terra15/test_terra15.py b/tests/test_io/test_terra15/test_terra15.py index beb18f94b..09ef0e32a 100644 --- a/tests/test_io/test_terra15/test_terra15.py +++ b/tests/test_io/test_terra15/test_terra15.py @@ -11,7 +11,6 @@ import pytest import dascore as dc -from dascore.io.terra15 import Terra15FormatterV4 from dascore.io.terra15.utils import _get_version_data_node @@ -33,19 +32,6 @@ def test_missing_gps_time(self, missing_gps_terra15_hdf5): assert isinstance(patch, dc.Patch) assert not np.any(pd.isnull(patch.coords.get_array("time"))) - def test_time_slice(self, terra15_v6_path): - """Ensure time slice within the file works.""" - info = dc.scan_to_df(terra15_v6_path).iloc[0] - file_t1, file_t2 = info["time_min"], info["time_max"] - dur = file_t2 - file_t1 - new_dur = dur / 4 - t1, t2 = file_t1 + new_dur, file_t1 + 2 * new_dur - out = dc.read(terra15_v6_path, time=(t1, t2))[0] - assert isinstance(out, dc.Patch) - time_summary = out.summary.get_coord_summary("time") - assert time_summary.min >= t1 - assert time_summary.max <= t2 - def test_time_slice_no_snap(self, terra15_v6_path): """Ensure no snapping returns raw time.""" info = dc.scan_to_df(terra15_v6_path).iloc[0] @@ -86,11 +72,6 @@ def test_units(self, terra15_das_patch): == patch.summary.get_coord_summary("distance").units ) - def test_hdf5file_not_terra15(self, generic_hdf5): - """Assert that the generic hdf5 file is not a terra15.""" - parser = Terra15FormatterV4() - assert not parser.get_format(generic_hdf5) - def test_unsupported_version_error(self): """Test that unsupported Terra15 version raises NotImplementedError.""" diff --git a/tests/test_io/test_xml_binary/test_xml_binary.py b/tests/test_io/test_xml_binary/test_xml_binary.py index dfd7c0bdc..7acf0c91d 100644 --- a/tests/test_io/test_xml_binary/test_xml_binary.py +++ b/tests/test_io/test_xml_binary/test_xml_binary.py @@ -254,12 +254,6 @@ def test_doesnt_reindex(self, binary_xml_directory): new_spool = spool.update() assert len(new_spool) == 2 - def test_simple_spool(self, binary_xml_directory): - """Ensure the simple path can be read into a spool.""" - spool = dc.spool(binary_xml_directory).update() - assert isinstance(spool, dc.BaseSpool) - assert len(spool) == 2 - def test_read_with_other_files(self, binary_xml_with_other_files): """Ensure other files are also included/indexed.""" spool = dc.spool(binary_xml_with_other_files).update() diff --git a/tests/test_proc/test_basic.py b/tests/test_proc/test_basic.py index 247560ed5..2014eea61 100644 --- a/tests/test_proc/test_basic.py +++ b/tests/test_proc/test_basic.py @@ -209,19 +209,20 @@ def test_zero_channels(self, random_patch): assert np.all(norm.data[0, :] == 0.0) assert np.all(norm.data[:, 0] == 0.0) + # One dimension: normalize reduces along an axis, and which axis that + # is has its own tests above; what these two are about is the nans. @pytest.mark.parametrize("norm", ["l1", "l2", "max"]) - @pytest.mark.parametrize("dim", ["time", "distance"]) - def test_nan_does_not_contaminate_slice(self, random_patch, dim, norm): + def test_nan_does_not_contaminate_slice(self, random_patch, norm): """A single NaN should not blank every value sharing its slice.""" patch = _patch_with_nan(random_patch) - out = patch.normalize(dim, norm=norm) + out = patch.normalize("time", norm=norm) assert np.isnan(out.data).sum() == 1 @pytest.mark.filterwarnings("ignore:All-NaN slice encountered") @pytest.mark.parametrize("norm", ["l1", "l2", "max"]) - @pytest.mark.parametrize("dim", ["time", "distance"]) - def test_all_nan_slice_stays_null(self, random_patch, dim, norm): + def test_all_nan_slice_stays_null(self, random_patch, norm): """A completely null slice should stay null rather than become zeros.""" + dim = "time" data = np.asarray(random_patch.data, dtype=np.float64).copy() # Null the first slice reduced by norm (patch is 2D, so the other axis). other_axis = 1 - random_patch.get_axis(dim) diff --git a/tests/test_proc/test_filter.py b/tests/test_proc/test_filter.py index de379b854..ad9fe3bf2 100644 --- a/tests/test_proc/test_filter.py +++ b/tests/test_proc/test_filter.py @@ -228,24 +228,34 @@ def test_sobel_runs(self, random_patch): assert not np.any(pd.isnull(out.data)) +@pytest.mark.parametrize("name", ["median_filter", "notch_filter", "savgol_filter"]) +def test_filters_validate_their_dims(random_patch, name): + """Each filter routes its dimension arguments through the shared check. + + pass_filter has its own check and its own error (TestPassFilterChecks), + so it cannot stand in for these three. + """ + kwargs = {"savgol_filter": {"polyorder": 2}, "notch_filter": {"q": 30}} + with pytest.raises(ParameterError, match="You must"): + getattr(random_patch, name)(**kwargs.get(name, {})) + + class TestMedianFilter: """Simple tests on median filter.""" - def test_median_no_kwargs_raises(self, random_patch): - """Apply default values.""" - msg = "You must" - with pytest.raises(ParameterError, match=msg): - random_patch.median_filter() - - def test_median_filter_time(self, random_patch): + def test_median_filter_time(self): """Test median filter in time dimension.""" - out = random_patch.median_filter(time=0.5) + # A median filter costs the window size times the sample count, so a + # small patch says the same thing much sooner. + patch = dc.get_example_patch("random_das", shape=(30, 200)) + out = patch.median_filter(time=0.5) assert isinstance(out, dc.Patch) assert not np.any(pd.isnull(out.data)) - def test_median_filter_time_distance(self, random_patch): + def test_median_filter_time_distance(self): """Apply default values.""" - out = random_patch.median_filter(time=0.05, distance=2) + patch = dc.get_example_patch("random_das", shape=(30, 200)) + out = patch.median_filter(time=0.05, distance=2) assert isinstance(out, dc.Patch) assert not np.any(pd.isnull(out.data)) @@ -258,12 +268,6 @@ def test_median_filter_ones(self, random_patch): class TestNotchFilter: """Tests for the notch filter.""" - def test_notch_no_kwargs_raises(self, random_patch): - """Test that no dimension raises an appropriate error.""" - msg = "You must" - with pytest.raises(ParameterError, match=msg): - random_patch.notch_filter(q=30) - def test_notch_filter_time(self, random_patch): """Test the notch filter along the time dimension.""" filtered_patch = random_patch.notch_filter(time=60, q=30) @@ -323,15 +327,12 @@ def test_unitless_coord_with_quantity_raises(self, random_patch, value): class TestSavgolFilter: """Simple tests on Savgol filter.""" - def test_savgol_no_kwargs_raises(self, random_patch): - """Ensure no kwargs raises.""" - msg = "You must" - with pytest.raises(ParameterError, match=msg): - random_patch.savgol_filter(polyorder=2) - - def test_savgol_filter_time(self, random_patch): + def test_savgol_filter_time(self): """Test savgol filter in time dimension.""" - out = random_patch.savgol_filter(polyorder=2, time=5) + # time=0.5 rather than 5 with the smaller patch: the window is a + # count of samples, and 5 seconds of it no longer fits. + patch = dc.get_example_patch("random_das", shape=(30, 200)) + out = patch.savgol_filter(polyorder=2, time=0.5) assert isinstance(out, dc.Patch) assert not np.any(pd.isnull(out.data)) diff --git a/tests/test_proc/test_mute.py b/tests/test_proc/test_mute.py index 511abce60..d38c88f8a 100644 --- a/tests/test_proc/test_mute.py +++ b/tests/test_proc/test_mute.py @@ -59,9 +59,17 @@ def _assert_point_values( @pytest.fixture(scope="session") -def patch_ones(random_patch): - """Return a patch filled with ones.""" - return random_patch.new(data=np.ones_like(random_patch.data)) +def patch_ones(): + """Return a patch filled with ones. + + A quarter of the default patch's pixels over the same 300 m by 8 s + extent: the tests below mute along lines given in metres and seconds, + so the extent has to stay while the sampling need not. + """ + patch = dc.get_example_patch( + "random_das", shape=(150, 1000), distance_step=2, time_step=0.008 + ) + return patch.new(data=np.ones_like(patch.data)) @pytest.fixture(scope="module") diff --git a/tests/test_proc/test_proc_inventory.py b/tests/test_proc/test_proc_inventory.py index 0195825ad..2eea72638 100644 --- a/tests/test_proc/test_proc_inventory.py +++ b/tests/test_proc/test_proc_inventory.py @@ -191,14 +191,6 @@ def test_missing_named_attr_warn(self, patch, inventory): ) assert "pulse_rate" not in dict(out.attrs) - def test_missing_named_coord_warn(self, patch, inventory): - """The coordinate half honors the same policy.""" - with pytest.warns(UserWarning, match="defines no 'nope'"): - out = patch.enrich( - inventory, attrs=False, coords=("nope",), on_missing="warn" - ) - assert "nope" not in out.coords.coord_map - @pytest.mark.parametrize("kwargs", [{"attrs": None}, {"coords": None}]) def test_none_is_not_the_off_switch(self, patch, inventory, kwargs): """False turns a half off; None is no longer a second spelling.""" @@ -518,13 +510,6 @@ def test_missing_coord_null(self, patch, inventory): out = patch.enrich(inventory, attrs=False, coords=("nope",), on_missing="null") assert np.isnan(out.get_coord("nope").values).all() - def test_missing_coord_ignore(self, patch, inventory): - """Or omitted entirely.""" - out = patch.enrich( - inventory, attrs=False, coords=("nope",), on_missing="ignore" - ) - assert "nope" not in set(out.coords.coord_map) - def test_blanket_without_geometry(self, patch, inventory): """A path with no geometry has no axes to project.""" inv = _replace_path(inventory, geometry=()) diff --git a/tests/test_proc/test_rolling.py b/tests/test_proc/test_rolling.py index 3a7eb7f51..1566a11c2 100644 --- a/tests/test_proc/test_rolling.py +++ b/tests/test_proc/test_rolling.py @@ -150,11 +150,11 @@ def test_center(self, random_patch): last_label = np.take(out.data, 0, axis=time_ax) assert np.all(np.isnan(last_label)) - @pytest.mark.parametrize("_", list(range(5))) - def test_compare_to_pandas(self, range_patch, _): + def test_compare_to_pandas(self, range_patch): """Test the apply method of PatchRoller when distance coordinate is entered and the first axis is distance. """ + # Seeded, so the five trials this used to run were one trial five times. random = np.random.RandomState(42) patch = range_patch axis = patch.get_axis("distance") @@ -235,20 +235,23 @@ def percentile_plus(frame, q, offset=0, axis=None): ).apply(lambda frame, axis=None: np.percentile(frame, 80, axis=axis) + 1) assert all_close(out, expected) - def test_pandas_apply_with_args_kwargs(self, random_patch): + def test_pandas_apply_with_args_kwargs(self): """Ensure pandas rolling apply supports extra function arguments.""" def percentile_plus(frame, q, offset=0, axis=None): """Get percentile with an offset.""" return np.percentile(frame, q, axis=axis) + offset - dt = random_patch.get_coord("time").step - out = random_patch.rolling(time=10 * dt, step=10 * dt, engine="pandas").apply( + # A small patch: pandas applies a python function per window, so this + # test's cost is the number of windows, not what is in them. + patch = dc.get_example_patch("random_das", shape=(30, 200)) + dt = patch.get_coord("time").step + out = patch.rolling(time=10 * dt, step=10 * dt, engine="pandas").apply( percentile_plus, 80, offset=1 ) - expected = random_patch.rolling( - time=10 * dt, step=10 * dt, engine="pandas" - ).apply(lambda frame: np.percentile(frame, 80) + 1) + expected = patch.rolling(time=10 * dt, step=10 * dt, engine="pandas").apply( + lambda frame: np.percentile(frame, 80) + 1 + ) assert all_close(out, expected) diff --git a/tests/test_proc/test_taper.py b/tests/test_proc/test_taper.py index eab4bb327..7c9de7372 100644 --- a/tests/test_proc/test_taper.py +++ b/tests/test_proc/test_taper.py @@ -23,17 +23,41 @@ def patch_ones(random_patch): return patch -@pytest.fixture(scope="session", params=sorted(WINDOW_FUNCTIONS)) +# Three shapes run through taper, rather than all thirteen; the window each +# name resolves to is checked for every entry in test_every_window_tapers. +TAPER_WINDOWS = ("hann", "triang", "blackmanharris") + + +@pytest.fixture(scope="session", params=TAPER_WINDOWS) def time_tapered_patch(request, patch_ones): """Return a tapered trace.""" - if "boxcar" in str(request.param): - pytest.skip("boxcar doesn't actually apply taper.") # first get a patch with all ones for easy testing patch = patch_ones.update(data=np.ones_like(patch_ones.data)) out = taper(patch, time=0.05, window_type=request.param) return out +def test_every_window_tapers(): + """Each name in the table reaches the window scipy has for it. + + The entries are lazy imports, and two of them are aliases (`cos` for + hann, `ramp` for triang), so a name pointing at the wrong scipy symbol + resolves fine and returns an array of the right length. Asserting the + shape of the window is what catches that: every one of them tapers to + near zero at both ends, hamming's 0.08 being the highest, and boxcar + is the one which does not taper at all. + """ + assert set(TAPER_WINDOWS) <= set(WINDOW_FUNCTIONS) + for name, func in WINDOW_FUNCTIONS.items(): + window = np.asarray(func(64)) + assert window.shape == (64,), name + assert np.max(window) <= 1.0 + 1e-9, name + if name == "boxcar": + assert np.all(window == 1.0), name + else: + assert window[0] < 0.09 and window[-1] < 0.09, name + + def _get_start_end_indices(patch, dim): """Helper function to get indices for slicing start/end of data.""" axis = patch.get_axis(dim) diff --git a/tests/test_transform/test_hilbert.py b/tests/test_transform/test_hilbert.py index ea3dd5aca..a1cc8fad7 100644 --- a/tests/test_transform/test_hilbert.py +++ b/tests/test_transform/test_hilbert.py @@ -67,7 +67,7 @@ def test_hilbert_sine_wave(self): class TestEnvelope: """Tests for the envelope function.""" - @pytest.fixture(autouse=True) + @pytest.fixture(scope="class", autouse=True) def modulated_patch_and_envelope(self): """Return a modulated patch""" # Create AM signal: A(t) * cos(w*t) where A(t) is the envelope diff --git a/tests/test_transform/test_tau_p.py b/tests/test_transform/test_tau_p.py index cc898a55a..e8ec0616d 100644 --- a/tests/test_transform/test_tau_p.py +++ b/tests/test_transform/test_tau_p.py @@ -107,8 +107,12 @@ def test_slowness_vals(self): """Ensures correct slowness and tau values are computed""" pytest.importorskip("numba") test_vels = np.linspace(1000, 3000, 101) - nch = 1000 - nt = 2000 + # The assertions below allow 20 m/s, which is exactly one step of + # test_vels, so argmax has to land on the right bin outright. At this + # aperture the runner-up peaks at 0.73 of the winner across the four + # cases; at 200x600 it reaches 0.996, close enough to tie. + nch = 400 + nt = 1000 # positive slope vel = 1500 diff --git a/tests/test_utils/test_array_api.py b/tests/test_utils/test_array_api.py index ee54014d0..917219dcb 100644 --- a/tests/test_utils/test_array_api.py +++ b/tests/test_utils/test_array_api.py @@ -280,9 +280,15 @@ def numpy_array(self): array[3, :] = np.nan return array + # Every pair but (keepdims=True, axis=0): what keepdims does to a + # reduction over the first axis, axis=1 already says. Keep the rest -- + # min/max over axis 1 with keepdims is the only cell which notices the + # mask shape at array_api.py's all-nan check. @pytest.mark.parametrize("name", names) - @pytest.mark.parametrize("axis", [0, 1, None]) - @pytest.mark.parametrize("keepdims", [True, False]) + @pytest.mark.parametrize( + ("axis", "keepdims"), + [(0, False), (1, False), (None, False), (1, True), (None, True)], + ) def test_matches_numpy(self, name, axis, keepdims, numpy_array, to_array): """The reductions agree with numpy, including on all-nan slices.""" array = to_array(numpy_array) diff --git a/tests/test_utils/test_chunk.py b/tests/test_utils/test_chunk.py index f9a963e36..ed015cf07 100644 --- a/tests/test_utils/test_chunk.py +++ b/tests/test_utils/test_chunk.py @@ -232,13 +232,6 @@ def test_nan_in_df(self, contiguous_df): ) assert expected_start in set(plan.outputs["time_min"]) - def test_all_nan(self, contiguous_df): - """When all rows lack the dim (and are dropped) the plan is empty.""" - nat = dc.to_datetime64("NaT") - df = contiguous_df.assign(time_min=nat, time_max=nat) - plan = build_chunk_plan(df, missing_dim="drop", time=dc.to_timedelta64(1.2)) - assert plan.outputs.empty - def test_nan_in_sample_ok(self, contiguous_df): """Ensure a NaN in the sampling rate is ok.""" df = contiguous_df.assign(time_step=dc.to_timedelta64("NaT")) @@ -248,26 +241,10 @@ def test_nan_in_sample_ok(self, contiguous_df): assert len(chunk_df) == 2 * len(contiguous_df) assert np.all(pd.isnull(chunk_df["time_step"])) - def test_unknown_dim_raises(self, contiguous_df): - """An unknown chunk dimension raises a clear error.""" - with pytest.raises(ChunkError, match="Time"): - build_chunk_plan(contiguous_df, Time=10) - - def test_invalid_conflict_raises(self, contiguous_df): - """An unsupported conflict value raises at the chunk call. See #804.""" - with pytest.raises(ParameterError, match="conflict must be one of"): - build_chunk_plan(contiguous_df, time=None, conflict="banana") - class TestChunkPlanToMerge: """Merge-mode planning on raw dataframes.""" - def test_chunk_can_merge(self, contiguous_df): - """Ensure chunk can be used to merge unspecified segment lengths.""" - out = build_chunk_plan(contiguous_df, time=None).outputs - assert len(out) == 1 - assert out["time_min"].min() == contiguous_df["time_min"].min() - def test_doesnt_merge_gappy_df(self, gapy_df): """Ensure the gappy dataframe doesn't get merged.""" out = build_chunk_plan(gapy_df, time=None).outputs @@ -309,14 +286,6 @@ def test_forced_merge_warns(self, contiguous_df): plan = build_chunk_plan(df, time=None, tolerance=10) assert len(plan.outputs) == 1 - def test_modified_flag_after_merge(self, contiguous_df): - """The modified flag shows False for a simple contiguous merge.""" - df = contiguous_df.assign(time_max=lambda x: x["time_max"] - x["time_step"]) - plan = build_chunk_plan(df, time=None) - assert len(plan.outputs) == 1 - assert plan.outputs["time_min"].min() == df["time_min"].min() - assert not plan.members["_modified"].any() - class TestPlanMembers: """Sanity checks on the members (instruction) table.""" @@ -987,11 +956,6 @@ def test_group_colliding_with_dim_column(self, contiguous_df): with pytest.raises(ParameterError, match="collide"): build_gap_frame(contiguous_df, "time", group="time_step") - def test_bad_missing_dim_raises(self, contiguous_df): - """A typo in missing_dim raises rather than silently dropping.""" - with pytest.raises(ParameterError, match="missing_dim"): - build_gap_frame(contiguous_df, "time", missing_dim="rasie") - class TestBuildCoverageFrame: """Coverage summaries on raw dataframes.""" diff --git a/tests/test_utils/test_gc_pause.py b/tests/test_utils/test_gc_pause.py index 2d29a3ba8..0fea44626 100644 --- a/tests/test_utils/test_gc_pause.py +++ b/tests/test_utils/test_gc_pause.py @@ -92,15 +92,26 @@ def loop_thread(): server = threading.Thread(target=loop_thread, daemon=True) server.start() - pause_gc() + # A collection has to be due for the pause to be what prevents it. + # Counting on the default gen-0 threshold means counting rounds + # against a number CPython is free to change -- and 8 rounds of 200 + # under the current 2000 never reaches it, so the test would pass + # whether or not gc was paused. + threshold = gc.get_threshold() + gc.set_threshold(100) try: - for _ in range(20): + # Inside the try: pause_gc counts the pause before it warns, so a + # warning raised as an error would otherwise leave the session + # with collection off and this threshold in place. + pause_gc() + for _ in range(8): with phil: # h5py holds its lock across the fetch request.release() assert answer.acquire(timeout=20), "deadlocked" finally: stop.set() resume_gc() + gc.set_threshold(*threshold) server.join(timeout=5) gc.collect() diff --git a/tests/test_utils/test_patch_utils.py b/tests/test_utils/test_patch_utils.py index 307496f41..b6962f99b 100644 --- a/tests/test_utils/test_patch_utils.py +++ b/tests/test_utils/test_patch_utils.py @@ -334,11 +334,6 @@ def test_stack_history_disabled(self, random_patch): class TestPatchMergeWorkflow: """Tests for the supported patch merge workflow.""" - def test_spool_chunk_replacement(self, random_patch): - """Ensure spool.chunk remains the supported merge path.""" - out = dc.spool([random_patch]).chunk(time=None) - assert len(out) == 1 - def test_merge_compatible_coords_attrs_ignores_private_attrs(self, random_patch): """Private attrs should not make otherwise compatible patches fail.""" patch_1 = random_patch.update_attrs(_source_patch_key="one") @@ -914,15 +909,6 @@ def test_spool_up(self, random_patch): out = func([random_patch] * 3, time=None) assert isinstance(out, dc.BaseSpool) - def test_new_dim_spool(self, random_patch): - """Ensure a patch with new dim can be retrieved from spool.""" - spool = dc.spool([random_patch, random_patch]) - spool_concat = spool.concatenate(wave_rank=None) - assert len(spool_concat) == 1 - patch = spool_concat[0] - assert "wave_rank" in patch.dims - assert len(patch.get_coord("wave_rank")) == len(spool) - def test_patch_with_gap(self, random_patch): """Ensure a patch with a time gap still concats.""" # Create a spool with patches that have a large gap diff --git a/tests/test_workflow/test_patch_op.py b/tests/test_workflow/test_patch_op.py index 82e3aa841..f95f0d16e 100644 --- a/tests/test_workflow/test_patch_op.py +++ b/tests/test_workflow/test_patch_op.py @@ -137,38 +137,16 @@ def test_the_op_does_what_the_call_did(self, call): args = resolve(args) func = _function(name) target = get_patch(key) - assert _same_patch( - func(target, *args, **kwargs), func.op(*args, **kwargs)(target) - ) - - @pytest.mark.parametrize("call", CALLS, ids=IDS) - def test_the_history_is_the_same(self, call): - """ - Down to what the patch says was done to it. - - `_same_patch` compares attrs, and history is one, so this is - already covered -- said again here because it is the property - which catches an operation that skipped the decorator, and it - should not quietly go away if that comparison is ever loosened. - """ - name, key, args, kwargs = call - args = resolve(args) - func = _function(name) - target = get_patch(key) direct = func(target, *args, **kwargs) - if not isinstance(direct, dc.Patch): - pytest.skip(f"{name} does not return a patch") - assert func.op(*args, **kwargs)(target).attrs.history == direct.attrs.history - - @pytest.mark.parametrize("call", CALLS, ids=IDS) - def test_the_call_fingerprints_alike(self, call): - """One call has one fingerprint, whichever route asks for it.""" - name, _, args, kwargs = call - args = resolve(args) - func = _function(name) - assert ( - fingerprint_call(func, args, kwargs) == func.op(*args, **kwargs).fingerprint - ) + through_op = func.op(*args, **kwargs)(target) + assert _same_patch(direct, through_op) + # `_same_patch` compares attrs and history is one, so the history + # is already covered -- said again because it is the property + # which catches an operation that skipped the decorator, and it + # should not quietly go away if that comparison is loosened. Not + # every patch function returns a patch (the viz ones do not). + if isinstance(direct, dc.Patch): + assert through_op.attrs.history == direct.attrs.history @pytest.mark.parametrize("call", CALLS, ids=IDS) def test_the_op_is_written_down(self, call): @@ -200,25 +178,18 @@ def test_an_operation_a_document_cannot_hold(self, name): op.to_dict() @pytest.mark.parametrize("call", CALLS, ids=IDS) - def test_the_op_pickles(self, call): - """An operation handed to another process carries what it is.""" - name, _, args, kwargs = call - args = resolve(args) - op = _function(name).op(*args, **kwargs) - assert pickle.loads(pickle.dumps(op)) == op - - @pytest.mark.parametrize("call", CALLS, ids=IDS) - def test_the_version_is_the_functions(self, call): - """ - The operation reports the version its function is declared at. - - Not the class's: `PatchOp` stands for every patch function, so its - own version says nothing about any of them. - """ + def test_the_op_is_well_formed(self, call): + """One call is one fingerprint, one version, and pickles as itself.""" name, _, args, kwargs = call args = resolve(args) func = _function(name) - assert _function(name).op(*args, **kwargs).version == func.__version__ + op = func.op(*args, **kwargs) + assert fingerprint_call(func, args, kwargs) == op.fingerprint + # An operation handed to another process carries what it is. + assert pickle.loads(pickle.dumps(op)) == op + # The version is the function's, not PatchOp's: the class stands + # for every patch function, so its own says nothing about any. + assert op.version == func.__version__ def _function(name): diff --git a/tests/test_workflow/test_task.py b/tests/test_workflow/test_task.py index b3d3530fd..78c5d80b3 100644 --- a/tests/test_workflow/test_task.py +++ b/tests/test_workflow/test_task.py @@ -183,16 +183,6 @@ def test_time_units_normalized(self): nanosecond = TimedValueTask(when=np.datetime64("2020-01-01T00:00:00.000000000")) assert day.fingerprint == nanosecond.fingerprint - def test_none_parameters_dropped(self): - """A parameter left at None is the same call as one left out.""" - assert TimedValueTask(when=None).fingerprint == TimedValueTask().fingerprint - - def test_array_parameter(self): - """An array parameter is hashed by its values.""" - first = TimedValueTask(when=np.arange(3)) - assert first.fingerprint != TimedValueTask(when=np.arange(4)).fingerprint - assert first.fingerprint == TimedValueTask(when=np.arange(3)).fingerprint - def test_nested_task_parameter(self): """A task given to a task is part of its parameters.""" first = TimedValueTask(when=ScaleTask(factor=2))