diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7c1a641..b4b6208 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,6 +3,8 @@ on: workflow_dispatch: {} push: branches: [main] # branch to trigger deployment +permissions: + contents: write jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 79e5ca6..2431278 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: version: 1.8.5 - name: Install dependencies and run tests run: | - export CI_CD=1 + export BIH_HPC=0 poetry install poetry run task coverage-test - name: Generate coverage badge diff --git a/.gitignore b/.gitignore index ff1b4f4..b98547c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *__pycache__ *.pyc +dist/ data/* !*.gitkeep +docs/build/ diff --git a/README.md b/README.md index eff702b..7a31c2a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -![UDM logo](res/img/logo.svg) +

+ UDM logo +

# UDM - Unified Data Module @@ -21,11 +23,11 @@ The documentation is available at [luisherrmann.github.io/udm](https://luisherrm ### Installation from GitHub --- -You can install the package with `pip` directly from github: +You can install the package with `pip` directly from GitHub: ``` -pip install git+https://:@github.com/luisherrmann/udm.git#egg=udm --extra-index-url https://download.pytorch.org/whl/cu116 +pip install "udm @ git+https://github.com/luisherrmann/udm.git" --extra-index-url https://download.pytorch.org/whl/cu116 ``` -Note that the `--extra-index-url` parameter is required to install the correct pytorch dependencies. +Note that the `--extra-index-url` parameter is required to install the correct PyTorch dependencies. If you are using `poetry` for dependency management, you can also install the package by adding these lines @@ -37,21 +39,21 @@ url = "https://download.pytorch.org/whl/cu116" secondary = true [tool.poetry.dependencies] -udm = {git = "https://:@github.com/luisherrmann/udm.git"} +udm = {git = "https://github.com/luisherrmann/udm.git"} ``` -to the `pyproject.toml` of your project, and running either `poetry install` or `poetry update` in your project. where `` should be replaced with your GitHub username and `` with your private access token. +to the `pyproject.toml` of your project, and run either `poetry install` or `poetry update` in your project. ### Installation from local source --- -If you want to install the UDM a local source, activate the Python environment where you wish to install the UDM (e.g. if you are managing your environments with `conda`, use `conda activate ` to activate the environment) and install the UDM using +If you want to install the UDM from a local source, activate the Python environment where you wish to install the UDM (e.g. if you are managing your environments with `conda`, use `conda activate ` to activate the environment) and install the UDM using ``` bash pip install -e ``` -It is highly recommended that you install in development mode (i.e. providing the `-e` editable-flag), since it may be necessary to introduce additions or modifications for your own project, requiring the package to be editable. +It is highly recommended that you install in development mode (i.e. providing the `-e` editable flag), since it may be necessary to introduce additions or modifications for your own project, requiring the package to be editable. If you are using `poetry` for dependency and environment management, include the following line @@ -67,13 +69,13 @@ in your `pyproject.toml` and install or update the dependencies. ### Post installation --- -If your project is already using hydra configs, it is recommended that you use the hydra configs in the `config` folder in a starting point for building your own hydra configs by cloning the config folder into your respective `config` subfolder. For instance, in the OGM/Umbrella project, the config templates of the UDM were included through +If your project is already using Hydra configs, it is recommended that you use the Hydra configs in the `config` folder as a starting point for building your own Hydra configs by cloning the config folder into your respective `config` subfolder. For instance, in the OGM/Umbrella project, the config templates of the UDM were included through ``` bash cp -r config umbrella/config/run/datamodule ``` -so the config of the UDM becomes a subconfig `datamodule` of the `run` config. +so the config of the UDM becomes a `datamodule` subconfig of the `run` config. ## Overview: How it works --- @@ -86,13 +88,13 @@ The UDM relies on the following three classes 2. `GeneralDataset` 3. `GeneralDatamodule` -Additional helper classes are provide to enable +Additional helper classes are provided to enable 4. Transforms (`PluginTransform`, `DatasetTransform`) 5. Filtering (`PluginRowFilter`, `PluginColFilter` and more) ### 1. DataPlugin -The `DataPlugin` class is an abstract class that defines the general interface for interacting with different data modalities. The idea is that for each data modality or each way of interacting with the data, there should be a class extending from `DataPlugin`. For instance there could be a `GeneticPlugin`, `CovariatePlugin`, ... class implemeting the interface of `DataPlugin` and providing additional methods specfic to the respective data modality. There is also a pre-existing generic class `TabularPlugin` which can be used for reading generic tabular data from a `.feather` file. +The `DataPlugin` class is an abstract class that defines the general interface for interacting with different data modalities. The idea is that for each data modality or each way of interacting with the data, there should be a class extending from `DataPlugin`. For instance, there could be a `GeneticPlugin`, `CovariatePlugin`, ... class implementing the interface of `DataPlugin` and providing additional methods specific to the respective data modality. There is also a pre-existing generic class `TabularPlugin` which can be used for reading generic tabular data from a `.feather` file. The `__getitem__()` method is called with an eid of the `DataPlugin` instance and an optional dictionary of feature selections and returns the respective data sample corresponding to that eid for the features provided. @@ -101,18 +103,18 @@ An important aspect to keep in mind is that the eids between different applicati 1. ***native eids*** are ids that are native to the application used by the `DataPlugin`. 2. ***master eids*** are the ids that are used to retrieve elements from the `DataPlugin`. -The user needs to ensure that all `DataPlugin`s use the same ***master eids***. If they use different ***native eids***, one set of eids are taken as the master eids and the `DataPlugin`s using different native eids need to be provided with a `.csv` mapping file to map native eids to master eids. This can be done using the `eid_map_path` option to specify the mapping file, as well as `eid_map_from` and `eid_map_to` to specify the columns containing the native and master eids, respectively. +The user needs to ensure that all `DataPlugin`s use the same ***master eids***. If they use different ***native eids***, one set of eids is taken as the master eids and the `DataPlugin`s using different native eids need to be provided with a `.csv` mapping file to map native eids to master eids. This can be done using the `eid_map_path` option to specify the mapping file, as well as `eid_map_from` and `eid_map_to` to specify the columns containing the native and master eids, respectively. -If the user needs to retrieve metadata from the dataplugin, this can be done using the `get_metadata()` method. This method should provide at least the following info: +If the user needs to retrieve metadata from the `DataPlugin`, this can be done using the `get_metadata()` method. This method should provide at least the following info: 1. `'features'`: A list of the feature names of the data controlled by the `DataPlugin`. -2. `feature_types`: A list of the data type used by the respective feature names. +2. `'feature_types'`: A list of the data types used by the respective feature names. 3. `'eids'`: A list of all the master eids controlled by this `DataPlugin`. -4. `'tags'`: A list of strings that can be used to tag `DataPlugin` for later identification and metadata aggregation across multiple `DataPlugin` instances. These are passed to the `DataPlugin` during initialization. +4. `'tags'`: A list of strings that can be used to tag the `DataPlugin` for later identification and metadata aggregation across multiple `DataPlugin` instances. These are passed to the `DataPlugin` during initialization. ### 2. GeneralDataset -The `GeneralDataset` class extends from the torch `Dataset` (`torch.utils.data`) and allows for the creation of a dataset from which to build a `DataLoader`. It expects a list of (optionally named) `DataPlugin` instances and a list of eids to use from those plugins. For instance, given +The `GeneralDataset` class extends from the PyTorch `Dataset` (`torch.utils.data`) and allows for the creation of a dataset from which to build a `DataLoader`. It expects a list of (optionally named) `DataPlugin` instances and a list of eids to use from those plugins. For instance, given ``` python >>> plugins = { @@ -121,7 +123,7 @@ The `GeneralDataset` class extends from the torch `Dataset` (`torch.utils.data`) ... } ``` -containing data for hypothetical eids `[1, 2, ..., 100]`, one could define datasets representing train-, validation- and test-split through `GeneralDataset` instances: +containing data for hypothetical eids `[1, 2, ..., 100]`, one could define datasets representing train, validation, and test splits through `GeneralDataset` instances: ``` python >>> train_ds = GeneralDataset(plugins, eids=[0, ..., 80], ...) @@ -165,7 +167,7 @@ The `get_metadata()` method returns metadata of the respective `DataPlugins` as ### 3. GeneralDatamodule -The `GeneralDatamodule` extends from the `LightningDatamodule` of Pytorch Lightning. It is initialized with a config of plugins, as well as mapping of eids denoting he respective splits. For instance, a `GeneralDatamodule` using the `CovariatePlugin` and the `GeneticPlugin` might be initialized by something like +The `GeneralDatamodule` extends from the `LightningDatamodule` of PyTorch Lightning. It is initialized with a config of plugins, as well as a mapping of eids denoting the respective splits. For instance, a `GeneralDatamodule` using the `CovariatePlugin` and the `GeneticPlugin` might be initialized by something like ``` python datamodule = GeneralDatamodule( @@ -190,19 +192,19 @@ datamodule = GeneralDatamodule( ``` -Ideally, the eids of all the plugins should align. However, when they do not, eids available from the DataPlugin will be obtained through `intersect` or through `union` operation of the respective eid sets (the behaviour is controlled by the `combine_eids_as` parameter). +Ideally, the eids of all the plugins should align. However, when they do not, eids available from the `DataPlugin` will be obtained through an `intersect` or `union` operation of the respective eid sets (the behaviour is controlled by the `combine_eids_as` parameter). Another thing to take into account is that the `GeneralDatamodule` supports passing of multiple splits. The respective datasets will all be set during the setup of the module. The splits to be used for training, validation and testing can be reassigned at any point during the lifecycle of the UDM. However, you will have to rerun `prepare_data()` and `setup()` for the new splits to be used rather than the old ones. ### 4. Transforms -All `DataPlugin` and `GeneralDataset` subclasses can be instantiated with additional transformations that are applied on top of the data of the `DataPlugin` or `GeneralDataset` every time an single element (or batch thereof) is sampled from the respective instance. The `DataPlugin` needs to be provided with an instance of `PluginTransform`, while a `GeneralDataset` must be provided with a `DatasetTransform` at initialization. +All `DataPlugin` and `GeneralDataset` subclasses can be instantiated with additional transformations that are applied on top of the data of the `DataPlugin` or `GeneralDataset` every time a single element (or batch thereof) is sampled from the respective instance. The `DataPlugin` needs to be provided with an instance of `PluginTransform`, while a `GeneralDataset` must be provided with a `DatasetTransform` at initialization. These transforms can be understood as follows: - `TensorTransform` -A function that takes a tensor as input and returns a tensor as output `torch.Tensor -> torch.Tensor`. You can use the `get_transform()` function from the `transforms.tensor_transforms.factory` package to instantiate transformations from hydra configurations. By default, all transforms from `torchvision.transforms` and `torch.nn.functional` are also supported. +A function that takes a tensor as input and returns a tensor as output `torch.Tensor -> torch.Tensor`. You can use the `get_transform()` function from the `transforms.tensor_transforms.factory` package to instantiate transformations from Hydra configurations. By default, all transforms from `torchvision.transforms` and `torch.nn.functional` are also supported. For example: ```python from omegaconf import DictConfig @@ -219,7 +221,7 @@ data_ = transform(data) ``` - `PluginTransform` -A mapping of component names to `TensorTransform` instances to be applied to the respective component. For instance, the `EHRPlugin` could be equipped with a single transformation `{records: cnormalize}`, where `cnormalize = lambda x: normalize(x, mean=(0.0, 0.5), std=(0.278, 1.023)`. Every time an element gets sampled from the dataplugin, the tensor of component `records` is modified by the `cnormalize` function. +A mapping of component names to `TensorTransform` instances to be applied to the respective component. For instance, the `EHRPlugin` could be equipped with a single transformation `{records: cnormalize}`, where `cnormalize = lambda x: normalize(x, mean=(0.0, 0.5), std=(0.278, 1.023))`. Every time an element gets sampled from the `DataPlugin`, the tensor of component `records` is modified by the `cnormalize` function. For example: ```python data = { @@ -237,14 +239,14 @@ data_ = transform(data) ``` - `DatasetTransform` -A mapping of plugin names to `PluginTransform` instances to be applied to the respective `DataPlugin`s when sampling. Every time an element gets sampled from the dataset, the tensor of every plugin is modified by the respective `PluginTransform`. Note that if a `DataPlugin` `plugin1`, controlled by the dataset `ds` was initialized with a `PluginTransform` `plugin1_trafo`, then the transformation will be applied in any dataset controlling `plugin1`. If a dataset `ds` has an own plugin `ds_plugin1_trafo`, then sampling an original element `x` from `ds` will see lead transformations `x[plugin1] -> plugin1_trafo -> ds_plugin1_trafo` of the original output. +A mapping of plugin names to `PluginTransform` instances to be applied to the respective `DataPlugin`s when sampling. Every time an element gets sampled from the dataset, the tensor of every plugin is modified by the respective `PluginTransform`. Note that if a `DataPlugin` `plugin1`, controlled by the dataset `ds`, was initialized with a `PluginTransform` `plugin1_trafo`, then the transformation will be applied in any dataset controlling `plugin1`. If a dataset `ds` has its own plugin transform `ds_plugin1_trafo`, then sampling an original element `x` from `ds` will lead to transformations `x[plugin1] -> plugin1_trafo -> ds_plugin1_trafo` of the original output. ### 5. Filtering -All `DataPlugin` instances can be provided with an instance of `PluginRowFilter` or `PluginColFilter` to filter rows or columns of the data, respectively. The filter will be applied once to the entire dataset during the `setup()` of the respective plugin, and the dataset will only keep rows and filters which satisfy the filtering condition (assuming a non-empty `row_filter` or `col_filter` argument is provided). Subsequent sampling of filtered eids will cause a `KeyError`, as will the selection of filtered columns. The hierarchy of filters works as follows: +All `DataPlugin` instances can be provided with an instance of `PluginRowFilter` or `PluginColFilter` to filter rows or columns of the data, respectively. The filter will be applied once to the entire dataset during the `setup()` of the respective plugin, and the dataset will only keep rows and columns which satisfy the filtering condition (assuming a non-empty `row_filter` or `col_filter` argument is provided). Subsequent sampling of filtered eids will cause a `KeyError`, as will the selection of filtered columns. The hierarchy of filters works as follows: - `Filter` -A filter is essentially a function instantiated with certain parameters which can be **called with a 2D tensor** $X$ **of shape** $M \times N$, and which **returns a boolean tensor** mask $\mu$ of length $M$, where $\mu_i$ indicates whether row $X_{i,:}$ should be kept or not. Column filtering for 2D tensor $X$ can be handled analogously by calling a filter with $X^T$. +A filter is essentially a function instantiated with certain parameters which can be **called with a 2D tensor** $X$ **of shape** $M \times N$, and which **returns a boolean tensor** mask $\mu$ of length $M$, where $\mu_i$ indicates whether row $X_{i,:}$ should be kept or not. Column filtering for a 2D tensor $X$ can be handled analogously by calling a filter with $X^T$. For example: ```python v_filter = AnyNan() @@ -312,7 +314,7 @@ mask = row_filter(eids, data) ``` - `PluginColFilter` -Instantiated with an optional mapping of component name to `KeyFilter`, and an optional mapping of component name to `Filter`. The key filters and value filters are applied to each component of the provided data separately and the filter returns a dictionary mapping components to masks to be applied to each component separately. Only columns satisfying their respective `KeyFilter` and `Filter` are marked to preservation. +Instantiated with an optional mapping of component name to `KeyFilter`, and an optional mapping of component name to `Filter`. The key filters and value filters are applied to each component of the provided data separately and the filter returns a dictionary mapping components to masks to be applied to each component separately. Only columns satisfying their respective `KeyFilter` and `Filter` are marked for preservation. Consider the following example: ```python features = { @@ -352,7 +354,7 @@ masks = col_filters(data) # } ``` -**NOTE**: Some of the plugins, such as the `H5adPlugin` had their own systems for filtering columns in place (e.g `usecols` and `dropcols` arguments). These individual filtering systems are still in place for backward compatibility reasons, but will eventually be removed. +**NOTE**: Some of the plugins, such as the `H5adPlugin`, had their own systems for filtering columns in place (e.g. `usecols` and `dropcols` arguments). These individual filtering systems are still in place for backward compatibility reasons, but will eventually be removed. ## Extending the UDM --- @@ -375,7 +377,8 @@ class ProteomicsPlugin(DataPlugin): ... ``` -which extends from `DataPlugin` that enables the use of proteomics data, and that the code for this class is in a file called `proteomics_plugin.py`. Following Python convention, the source code for each DataPlugin subclass should be in a file containing no other classes than the subclass itself, and the classname should be written in camel case (e.g. `ProteomicsPlugin`), while the file name should be written in snake case (e.g. `proteomics_plugin`). NOTE: Please make sure the name of the plugin does not match the name of any other pre-existing DataPlugin in the project! +which extends from `DataPlugin`, enables the use of proteomics data, and has its code in a file called `proteomics_plugin.py`. Following Python convention, the source code for each `DataPlugin` subclass should be in a file containing no other classes than the subclass itself, and the class name should be written in camel case (e.g. `ProteomicsPlugin`), while the file name should be written in snake case (e.g. `proteomics_plugin`). +**NOTE**: Please make sure the name of the plugin does not match the name of any other pre-existing `DataPlugin` in the project! To include this DataPlugin in the repository, you could put the file `proteomics_plugin.py` in a subdirectory of `plugins` like so: @@ -405,7 +408,7 @@ Or better yet: └── ... ``` -Arbitrary levels of nesting are possible, as the GeneralDatamoulde will automatically discover all subclasses of `DataPlugin` within the `plugins` package. However, we encourage you to apply a low amount of nesting to keep a clean directory structure. +Arbitrary levels of nesting are possible, as the `GeneralDatamodule` will automatically discover all subclasses of `DataPlugin` within the `plugins` package. However, we encourage you to use a low amount of nesting to keep a clean directory structure. ### 2. Adding config files --- @@ -420,9 +423,9 @@ src_path: memmap: false ``` -i.e. the fields `name` giving the name of the class, the field `__init__` giving the method by which to initialize an instance of the class and default values for the arguments to be passed to the init function of the class. The first two values are mandatory, because they are required by the `DataModule` to know which DataPlugins to prepare, and what method to prepare for the initialization. By default, the initialization method will be the regular `__init__` method, but in some cases it may be useful to define different `__init__` methods for different ways of initializing the DataPlugin for interfacing with the respective data. +i.e. the field `name` gives the name of the class, the field `__init__` gives the method by which to initialize an instance of the class, and the remaining fields give default values for the arguments to be passed to the init function of the class. The first two values are mandatory, because they are required by the `DataModule` to know which `DataPlugin`s to prepare, and what method to use for the initialization. By default, the initialization method will be the regular `__init__` method, but in some cases it may be useful to define different `__init__` methods for different ways of initializing the `DataPlugin` for interfacing with the respective data. -For example, let's say the proteomics data to be accessed through the `ProteomicsPlugin` can also be retrieved from a database. Then, a good pattern would be to enable the `ProteomicsPlugin` to be initialized through another method `from_db()`, where database connection arguments are provided. It would be recommendable to have a separate default configuration for this case, e.g.: +For example, let's say the proteomics data to be accessed through the `ProteomicsPlugin` can also be retrieved from a database. Then, a good pattern would be to enable the `ProteomicsPlugin` to be initialized through another method `from_db()`, where database connection arguments are provided. It would be recommended to have a separate default configuration for this case, e.g.: #### *`from_db.yaml`* ``` yaml @@ -433,7 +436,7 @@ db_pass: locked db_table: ukbb_processed ``` -Putting everything together, these two config files should be placed in a subdirectory of configs, preferrably mirroring the directory structure of `udm/plugins`, like so: +Putting everything together, these two config files should be placed in a subdirectory of configs, preferably mirroring the directory structure of `udm/plugins`, like so: ``` . @@ -454,9 +457,9 @@ Putting everything together, these two config files should be placed in a subdir └── ... ``` -And that's it, you can now create DataModule configurations +And that's it, you can now create your own DataModule configurations. -### 3. Add tests +### 3. Adding tests In order to ensure that your `DataPlugin` subclass works properly, it is highly encouraged that you write unit tests to check that your plugin works as intended on small test datasets. diff --git a/config/plugins/genetic/debug.yaml b/config/plugins/genetic/debug.yaml index 8947334..4086187 100644 --- a/config/plugins/genetic/debug.yaml +++ b/config/plugins/genetic/debug.yaml @@ -1,6 +1,6 @@ name: GeneticPlugin __init__: __init__ -src: ['all_chrs_extended_tr.fincad100', 'all_chrs_valid.fincad100', 'all_chrs_test.fincad100'] +src: ['genetic_extended_tr', 'genetic_valid', 'genetic_test'] src_names: ['train', 'valid', 'test'] genetic_json: 'res/ukb_genetic.json' ontology: 'molecular_function' diff --git a/poetry.lock b/poetry.lock index a61edd5..9ee9067 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "addict" @@ -6,6 +6,7 @@ version = "2.4.0" description = "Addict is a dictionary whose items can be set using both attribute and item syntax." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "addict-2.4.0-py3-none-any.whl", hash = "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc"}, {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, @@ -17,6 +18,7 @@ version = "3.8.4" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, @@ -117,7 +119,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet ; python_version < \"3.10\""] [[package]] name = "aiosignal" @@ -125,6 +127,7 @@ version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -139,6 +142,7 @@ version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" optional = false python-versions = ">=3.6" +groups = ["docs"] files = [ {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, @@ -150,6 +154,7 @@ version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, ] @@ -160,6 +165,7 @@ version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, @@ -171,6 +177,7 @@ version = "4.0.2" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, @@ -182,6 +189,7 @@ version = "22.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, @@ -192,7 +200,7 @@ cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] dev = ["attrs[docs,tests]"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] +tests-no-zope = ["cloudpickle ; platform_python_implementation == \"CPython\"", "cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990) ; platform_python_implementation == \"CPython\"", "mypy (>=0.971,<0.990) ; platform_python_implementation == \"CPython\"", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version < \"3.11\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version < \"3.11\"", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] [[package]] name = "babel" @@ -200,6 +208,7 @@ version = "2.12.1" description = "Internationalization utilities" optional = false python-versions = ">=3.7" +groups = ["docs"] files = [ {file = "Babel-2.12.1-py3-none-any.whl", hash = "sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610"}, {file = "Babel-2.12.1.tar.gz", hash = "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455"}, @@ -214,6 +223,7 @@ version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "docs"] files = [ {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, @@ -225,6 +235,8 @@ version = "1.15.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\"" files = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, @@ -301,6 +313,7 @@ version = "3.3.1" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.6.1" +groups = ["dev"] files = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, @@ -312,6 +325,7 @@ version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main", "docs"] files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, @@ -396,6 +410,7 @@ version = "8.1.3" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, @@ -410,6 +425,7 @@ version = "2.2.1" description = "Extended pickling support for Python objects" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, @@ -421,6 +437,8 @@ version = "3.26.3" description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "cmake-3.26.3-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9d38ea5b4999f8f042a071bea3e213f085bac26d7ab54cb5a4c6a193c4baf132"}, {file = "cmake-3.26.3-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:6e5fcd1cfaac33d015e2709e0dd1b7ad352a315367012ac359c9adc062cf075b"}, @@ -450,10 +468,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev", "docs"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\" or sys_platform == \"win32\"", docs = "sys_platform == \"win32\""} [[package]] name = "coverage" @@ -461,6 +481,7 @@ version = "7.2.7" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, @@ -525,7 +546,7 @@ files = [ ] [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "dask" @@ -533,6 +554,7 @@ version = "2023.3.2" description = "Parallel PyData with Task Scheduling" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "dask-2023.3.2-py3-none-any.whl", hash = "sha256:5e64763d62feb18afd3ad66f364e0b4f456f7ac92e894fcc87950af75029ecdf"}, {file = "dask-2023.3.2.tar.gz", hash = "sha256:51009e92ba9a280bd417633d1ae84f3ed23a8940f0a19594a4b7797ef226fff4"}, @@ -543,7 +565,7 @@ click = ">=7.0" cloudpickle = ">=1.1.1" fsspec = ">=0.6.0" importlib-metadata = ">=4.13.0" -numpy = {version = ">=1.21", optional = true, markers = "extra == \"array\""} +numpy = {version = ">=1.21", optional = true, markers = "extra == \"array\" or extra == \"dataframe\""} packaging = ">=20.0" pandas = {version = ">=1.3", optional = true, markers = "extra == \"dataframe\""} partd = ">=1.2.0" @@ -564,6 +586,7 @@ version = "0.2.0" description = "Generalized Linear Models with Dask" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "dask-glm-0.2.0.tar.gz", hash = "sha256:58b86cebf04fe5b9e58092e1c467e32e60d01e11b71fdc628baaa9fc6d1adee5"}, {file = "dask_glm-0.2.0-py2.py3-none-any.whl", hash = "sha256:a116c36a830cc20660c0815c4c8d67239814931952e8695d784604cb4839ea51"}, @@ -585,6 +608,7 @@ version = "2023.3.24" description = "A library for distributed and parallel machine learning" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "dask-ml-2023.3.24.tar.gz", hash = "sha256:96c090db6d32836536e3f09ca64deb599e864587aa8e3d4d2c6b4af588733307"}, {file = "dask_ml-2023.3.24-py3-none-any.whl", hash = "sha256:ac34d556cdf3c4b25096d3f3806bbb2bed1ff7937f0c04e14c71d11a6c925055"}, @@ -615,6 +639,7 @@ version = "0.3.6" description = "serialize all of python" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, @@ -629,6 +654,7 @@ version = "0.3.6" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, @@ -640,6 +666,7 @@ version = "2023.3.2.1" description = "Distributed scheduler for Dask" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "distributed-2023.3.2.1-py3-none-any.whl", hash = "sha256:a7756a4b952ec5a7fd3163e93aef99aaf8b0000568fa9ee7c000113a470d7f8e"}, {file = "distributed-2023.3.2.1.tar.gz", hash = "sha256:f40c4578622a15261bb59676ca8d7024f7d108aecc58406a5482bdfb7e69ce99"}, @@ -668,6 +695,7 @@ version = "0.4.0" description = "Python bindings for the docker credentials store API" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "docker-pycreds-0.4.0.tar.gz", hash = "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4"}, {file = "docker_pycreds-0.4.0-py2.py3-none-any.whl", hash = "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49"}, @@ -682,6 +710,7 @@ version = "0.6.2" description = "Pythonic argument parser, that will make you smile" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] @@ -692,6 +721,7 @@ version = "0.18.1" description = "Docutils -- Python Documentation Utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["docs"] files = [ {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, @@ -703,6 +733,7 @@ version = "1.1.0" description = "An implementation of lxml.xmlfile for the standard library" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, @@ -714,6 +745,8 @@ version = "1.1.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] +markers = "python_version <= \"3.10\"" files = [ {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, @@ -728,6 +761,7 @@ version = "3.12.2" description = "A platform independent file lock." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, @@ -743,6 +777,7 @@ version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, @@ -826,6 +861,7 @@ version = "2023.4.0" description = "File-system specification" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fsspec-2023.4.0-py3-none-any.whl", hash = "sha256:f398de9b49b14e9d84d2c2d11b7b67121bc072fe97b930c4e5668ac3917d8307"}, {file = "fsspec-2023.4.0.tar.gz", hash = "sha256:bf064186cd8808f0b2f6517273339ba0a0c8fb1b7048991c28bc67f58b8b67cd"}, @@ -865,6 +901,7 @@ version = "4.0.10" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, @@ -879,6 +916,7 @@ version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "GitPython-3.1.31-py3-none-any.whl", hash = "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"}, {file = "GitPython-3.1.31.tar.gz", hash = "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573"}, @@ -893,6 +931,7 @@ version = "1.3.1" description = "Python scripts to find enrichment of GO terms" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "goatools-1.3.1-py3-none-any.whl", hash = "sha256:eb87f7293fbe7b84482b46b1f8f9471c1389bbd39a088a2deae510ef9d77e77f"}, {file = "goatools-1.3.1.tar.gz", hash = "sha256:c25cb7b6044308d359a010fc55a9b9f2f58dd64e4e2afd181d22d665d781a3dc"}, @@ -915,6 +954,7 @@ version = "1.0.1" description = "a heap with decrease-key and increase-key operations" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "HeapDict-1.0.1-py3-none-any.whl", hash = "sha256:6065f90933ab1bb7e50db403b90cab653c853690c5992e69294c2de2b253fc92"}, {file = "HeapDict-1.0.1.tar.gz", hash = "sha256:8495f57b3e03d8e46d5f1b2cc62ca881aca392fd5cc048dc0aa2e1a6d23ecdb6"}, @@ -926,6 +966,7 @@ version = "1.3.2" description = "A framework for elegantly configuring complex applications" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "hydra-core-1.3.2.tar.gz", hash = "sha256:8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824"}, {file = "hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b"}, @@ -943,6 +984,7 @@ version = "2.5.22" description = "File identification library for Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "identify-2.5.22-py2.py3-none-any.whl", hash = "sha256:f0faad595a4687053669c112004178149f6c326db71ee999ae4636685753ad2f"}, {file = "identify-2.5.22.tar.gz", hash = "sha256:f7a93d6cf98e29bd07663c60728e7a4057615068d7a639d132dc883b2d54d31e"}, @@ -957,6 +999,7 @@ version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" +groups = ["main", "docs"] files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, @@ -968,6 +1011,7 @@ version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["docs"] files = [ {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, @@ -979,10 +1023,12 @@ version = "6.3.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.7" +groups = ["main", "docs"] files = [ {file = "importlib_metadata-6.3.0-py3-none-any.whl", hash = "sha256:8f8bd2af397cf33bd344d35cfe7f489219b7d14fc79a3f854b75b8417e9226b0"}, {file = "importlib_metadata-6.3.0.tar.gz", hash = "sha256:23c2bcae4762dfb0bbe072d358faec24957901d75b6c4ab11172c0c982532402"}, ] +markers = {docs = "python_version < \"3.10\""} [package.dependencies] zipp = ">=0.5" @@ -990,7 +1036,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)"] [[package]] name = "importlib-resources" @@ -998,6 +1044,8 @@ version = "5.12.0" description = "Read resources from Python packages" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.8\"" files = [ {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, @@ -1008,7 +1056,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "iniconfig" @@ -1016,6 +1064,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -1027,6 +1076,7 @@ version = "3.1.2" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main", "docs"] files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, @@ -1044,6 +1094,7 @@ version = "1.2.0" description = "Lightweight pipelining with Python functions" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, @@ -1051,24 +1102,71 @@ files = [ [[package]] name = "lightning-utilities" -version = "0.8.0" -description = "PyTorch Lightning Sample project." +version = "0.11.9" +description = "Lightning toolbox for across the our ecosystem." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.8\" or sys_platform != \"linux\" and python_version < \"3.10\"" files = [ - {file = "lightning-utilities-0.8.0.tar.gz", hash = "sha256:8e5d95c7c57f026cdfed7c154303e88c93a7a5e868c9944cb02cf71f1db29720"}, - {file = "lightning_utilities-0.8.0-py3-none-any.whl", hash = "sha256:22aa107b51c8f50ccef54d08885eb370903eb04148cddb2891b9c65c59de2a6e"}, + {file = "lightning_utilities-0.11.9-py3-none-any.whl", hash = "sha256:ac6d4e9e28faf3ff4be997876750fee10dc604753dbc429bf3848a95c5d7e0d2"}, + {file = "lightning_utilities-0.11.9.tar.gz", hash = "sha256:f5052b81344cc2684aa9afd74b7ce8819a8f49a858184ec04548a5a109dfd053"}, ] [package.dependencies] packaging = ">=17.1" +setuptools = "*" typing-extensions = "*" [package.extras] cli = ["fire"] -docs = ["sphinx (>=4.0,<5.0)"] -test = ["coverage (==6.5.0)"] -typing = ["mypy (>=1.0.0)"] +docs = ["requests (>=2.0.0)"] +typing = ["mypy (>=1.0.0)", "types-setuptools"] + +[[package]] +name = "lightning-utilities" +version = "0.15.2" +description = "Lightning toolbox for across the our ecosystem." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.9\" and sys_platform == \"linux\"" +files = [ + {file = "lightning_utilities-0.15.2-py3-none-any.whl", hash = "sha256:ad3ab1703775044bbf880dbf7ddaaac899396c96315f3aa1779cec9d618a9841"}, + {file = "lightning_utilities-0.15.2.tar.gz", hash = "sha256:cdf12f530214a63dacefd713f180d1ecf5d165338101617b4742e8f22c032e24"}, +] + +[package.dependencies] +packaging = ">=17.1" +setuptools = "*" +typing_extensions = "*" + +[package.extras] +cli = ["jsonargparse[signatures] (>=4.38.0)", "tomlkit"] +docs = ["requests (>=2.0.0)"] +typing = ["mypy (>=1.0.0)", "types-setuptools"] + +[[package]] +name = "lightning-utilities" +version = "0.15.3" +description = "Lightning toolbox for across the our ecosystem." +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "lightning_utilities-0.15.3-py3-none-any.whl", hash = "sha256:6c55f1bee70084a1cbeaa41ada96e4b3a0fea5909e844dd335bd80f5a73c5f91"}, + {file = "lightning_utilities-0.15.3.tar.gz", hash = "sha256:792ae0204c79f6859721ac7f386c237a33b0ed06ba775009cb894e010a842033"}, +] + +[package.dependencies] +packaging = ">=22" +typing_extensions = "*" + +[package.extras] +cli = ["jsonargparse[signatures] (>=4.38.0)", "tomlkit"] +docs = ["requests (>=2.0.0)"] +typing = ["mypy (>=1.0.0)", "types-setuptools"] [[package]] name = "lit" @@ -1076,6 +1174,8 @@ version = "16.0.1" description = "A Software Testing Tool" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "lit-16.0.1.tar.gz", hash = "sha256:630a47291b714cb115015df23ab04267c24fe59aec7ecd7e637d5c75cdb45c91"}, ] @@ -1086,6 +1186,7 @@ version = "0.40.1" description = "lightweight wrapper around basic LLVM functionality" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "llvmlite-0.40.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:84ce9b1c7a59936382ffde7871978cddcda14098e5a76d961e204523e5c372fb"}, {file = "llvmlite-0.40.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3673c53cb21c65d2ff3704962b5958e967c6fc0bd0cff772998face199e8d87b"}, @@ -1119,6 +1220,7 @@ version = "1.0.0" description = "File-based locks for Python on Linux and Windows" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"}, {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"}, @@ -1130,6 +1232,7 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -1154,6 +1257,7 @@ version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" +groups = ["main", "docs"] files = [ {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, @@ -1213,6 +1317,7 @@ version = "0.4.0" description = "Collection of plugins for markdown-it-py" optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "mdit_py_plugins-0.4.0-py3-none-any.whl", hash = "sha256:b51b3bb70691f57f974e257e367107857a93b36f322a9e6d44ca5bf28ec2def9"}, {file = "mdit_py_plugins-0.4.0.tar.gz", hash = "sha256:d8ab27e9aed6c38aa716819fedfde15ca275715955f8a185a8e1cf90fb1d2c1b"}, @@ -1232,6 +1337,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["docs"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1243,6 +1349,7 @@ version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, @@ -1251,7 +1358,7 @@ files = [ [package.extras] develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] +gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] tests = ["pytest (>=4.6)"] [[package]] @@ -1260,6 +1367,7 @@ version = "1.0.5" description = "MessagePack serializer" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, @@ -1332,6 +1440,8 @@ version = "0.3.0" description = "shlex for windows" optional = false python-versions = ">=3.5" +groups = ["dev"] +markers = "sys_platform == \"win32\"" files = [ {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, @@ -1343,6 +1453,7 @@ version = "6.0.4" description = "multidict implementation" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, @@ -1426,6 +1537,7 @@ version = "0.6.0" description = "Multiple dispatch" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "multipledispatch-0.6.0-py2-none-any.whl", hash = "sha256:407e6d8c5fa27075968ba07c4db3ef5f02bea4e871e959570eeb69ee39a6565b"}, {file = "multipledispatch-0.6.0-py3-none-any.whl", hash = "sha256:a55c512128fb3f7c2efd2533f2550accb93c35f1045242ef74645fc92a2c3cba"}, @@ -1441,6 +1553,7 @@ version = "2.0.0" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "myst_parser-2.0.0-py3-none-any.whl", hash = "sha256:7c36344ae39c8e740dad7fdabf5aa6fc4897a813083c6cc9990044eb93656b14"}, {file = "myst_parser-2.0.0.tar.gz", hash = "sha256:ea929a67a6a0b1683cdbe19b8d2e724cd7643f8aa3e7bb18dd65beac3483bead"}, @@ -1457,7 +1570,7 @@ sphinx = ">=6,<8" [package.extras] code-style = ["pre-commit (>=3.0,<4.0)"] linkify = ["linkify-it-py (>=2.0,<3.0)"] -rtd = ["ipython", "pydata-sphinx-theme (==v0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.8.2,<0.9.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] +rtd = ["ipython", "pydata-sphinx-theme (==0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.8.2,<0.9.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=7,<8)", "pytest-cov", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "sphinx-pytest"] testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4,<0.4.0)"] @@ -1467,6 +1580,7 @@ version = "3.1" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, @@ -1485,6 +1599,7 @@ version = "1.7.0" description = "Node.js virtual environment builder" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +groups = ["dev"] files = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, @@ -1499,6 +1614,7 @@ version = "0.57.1" description = "compiling Python code using LLVM" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "numba-0.57.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db8268eb5093cae2288942a8cbd69c9352f6fe6e0bfa0a9a27679436f92e4248"}, {file = "numba-0.57.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:643cb09a9ba9e1bd8b060e910aeca455e9442361e80fce97690795ff9840e681"}, @@ -1537,6 +1653,7 @@ version = "1.24.2" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, @@ -1574,6 +1691,8 @@ version = "11.10.3.66" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, @@ -1589,6 +1708,8 @@ version = "11.7.101" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895"}, {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-win_amd64.whl", hash = "sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0"}, @@ -1604,6 +1725,8 @@ version = "11.7.99" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, @@ -1620,6 +1743,8 @@ version = "11.7.99" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, @@ -1635,6 +1760,8 @@ version = "8.5.0.96" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, @@ -1650,8 +1777,12 @@ version = "10.9.0.58" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl", hash = "sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81"}, + {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux2014_aarch64.whl", hash = "sha256:34b7315104e615b230dc3c2d1861f13bff9ec465c5d3b4bb65b4986d03a1d8d4"}, + {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e21037259995243cc370dd63c430d77ae9280bedb68d5b5a18226bfc92e5d748"}, {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-win_amd64.whl", hash = "sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03"}, ] @@ -1661,6 +1792,8 @@ version = "10.2.10.91" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c"}, {file = "nvidia_curand_cu11-10.2.10.91-py3-none-win_amd64.whl", hash = "sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417"}, @@ -1676,6 +1809,8 @@ version = "11.4.0.1" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412"}, {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99"}, @@ -1692,6 +1827,8 @@ version = "11.7.4.91" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d"}, {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-win_amd64.whl", hash = "sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9"}, @@ -1707,6 +1844,8 @@ version = "2.14.3" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl", hash = "sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb"}, ] @@ -1717,6 +1856,8 @@ version = "11.7.91" description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac"}, {file = "nvidia_nvtx_cu11-11.7.91-py3-none-win_amd64.whl", hash = "sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064"}, @@ -1732,6 +1873,7 @@ version = "2.3.0" description = "A flexible configuration library" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, @@ -1747,6 +1889,7 @@ version = "3.1.2" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, @@ -1761,6 +1904,7 @@ version = "23.1" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "docs"] files = [ {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, @@ -1772,6 +1916,7 @@ version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, @@ -1805,7 +1950,7 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.21.0", markers = "python_version == \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.1" @@ -1820,6 +1965,7 @@ version = "1.4.0" description = "Appendable key-value storage" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "partd-1.4.0-py3-none-any.whl", hash = "sha256:7a63529348cf0dff14b986db641cd1b83c16b5cb9fc647c2851779db03282ef8"}, {file = "partd-1.4.0.tar.gz", hash = "sha256:aa0ff35dbbcc807ae374db56332f4c1b39b46f67bf2975f5151e0b4186aed0d5"}, @@ -1838,6 +1984,7 @@ version = "0.1.2" description = "File system general utilities" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pathtools-0.1.2.tar.gz", hash = "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0"}, ] @@ -1848,6 +1995,7 @@ version = "0.5.3" description = "A Python package for describing statistical models and for building design matrices." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "patsy-0.5.3-py2.py3-none-any.whl", hash = "sha256:7eb5349754ed6aa982af81f636479b1b8db9d5b1a6e957a6016ec0534b5c86b7"}, {file = "patsy-0.5.3.tar.gz", hash = "sha256:bdc18001875e319bc91c812c1eb6a10be4bb13cb81eb763f466179dca3b67277"}, @@ -1866,6 +2014,7 @@ version = "9.5.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, @@ -1945,6 +2094,7 @@ version = "3.8.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "platformdirs-3.8.1-py3-none-any.whl", hash = "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c"}, {file = "platformdirs-3.8.1.tar.gz", hash = "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528"}, @@ -1960,6 +2110,7 @@ version = "0.13.1" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main", "dev"] files = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, @@ -1974,6 +2125,7 @@ version = "0.9.1" description = "A collection of helpful Python tools!" optional = false python-versions = "*" +groups = ["docs"] files = [ {file = "pockets-0.9.1-py2.py3-none-any.whl", hash = "sha256:68597934193c08a08eb2bf6a1d85593f627c22f9b065cc727a4f03f669d96d86"}, {file = "pockets-0.9.1.tar.gz", hash = "sha256:9320f1a3c6f7a9133fe3b571f283bcf3353cd70249025ae8d618e40e9f7e92b3"}, @@ -1988,6 +2140,7 @@ version = "2.21.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, @@ -2006,6 +2159,7 @@ version = "3.7.0" description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "prettytable-3.7.0-py3-none-any.whl", hash = "sha256:f4aaf2ed6e6062a82fd2e6e5289bbbe705ec2788fe401a3a1f62a1cea55526d2"}, {file = "prettytable-3.7.0.tar.gz", hash = "sha256:ef8334ee40b7ec721651fc4d37ecc7bb2ef55fde5098d994438f0dfdaa385c0c"}, @@ -2023,6 +2177,7 @@ version = "4.22.3" description = "" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, {file = "protobuf-4.22.3-cp310-abi3-win_amd64.whl", hash = "sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51"}, @@ -2045,6 +2200,7 @@ version = "5.9.4" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main", "dev"] files = [ {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"}, @@ -2063,7 +2219,7 @@ files = [ ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""] [[package]] name = "pyarrow" @@ -2071,6 +2227,7 @@ version = "10.0.1" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, @@ -2108,6 +2265,8 @@ version = "2.21" description = "C parser in Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\"" files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -2119,6 +2278,7 @@ version = "1.4.2" description = "Python interface to Graphviz's Dot" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "pydot-1.4.2-py2.py3-none-any.whl", hash = "sha256:66c98190c65b8d2e2382a441b4c0edfdb4f4c025ef9cb9874de478fb0793a451"}, {file = "pydot-1.4.2.tar.gz", hash = "sha256:248081a39bcb56784deb018977e428605c1c758f10897a339fce1dd728ff007d"}, @@ -2133,13 +2293,14 @@ version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" +groups = ["docs"] files = [ {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] -plugins = ["importlib-metadata"] +plugins = ["importlib-metadata ; python_version < \"3.8\""] [[package]] name = "pyparsing" @@ -2147,6 +2308,7 @@ version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" +groups = ["main"] files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -2161,6 +2323,7 @@ version = "7.3.0" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, @@ -2183,6 +2346,7 @@ version = "0.6.3" description = "It helps to use fixtures in pytest.mark.parametrize" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"}, {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"}, @@ -2197,6 +2361,7 @@ version = "2.8.2" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2207,38 +2372,35 @@ six = ">=1.5" [[package]] name = "pytorch-lightning" -version = "1.9.5" +version = "2.3.3" description = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "pytorch-lightning-1.9.5.tar.gz", hash = "sha256:925fe7b80ddf04859fa385aa493b260be4000b11a2f22447afb4a932d1f07d26"}, - {file = "pytorch_lightning-1.9.5-py3-none-any.whl", hash = "sha256:06821558158623c5d2ecf5d3d0374dc8bd661e0acd3acf54a6d6f71737c156c5"}, + {file = "pytorch-lightning-2.3.3.tar.gz", hash = "sha256:5f974015425af6873b5689246c5495ca12686b446751479273c154b73aeea843"}, + {file = "pytorch_lightning-2.3.3-py3-none-any.whl", hash = "sha256:4365e3f2874e223e63cb42628d24c88c2bdc8d1794453cac38c0619b31115fba"}, ] [package.dependencies] -fsspec = {version = ">2021.06.0", extras = ["http"]} -lightning-utilities = ">=0.6.0.post0" +fsspec = {version = ">=2022.5.0", extras = ["http"]} +lightning-utilities = ">=0.10.0" numpy = ">=1.17.2" -packaging = ">=17.1" +packaging = ">=20.0" PyYAML = ">=5.4" -torch = ">=1.10.0" +torch = ">=2.0.0" torchmetrics = ">=0.7.0" tqdm = ">=4.57.0" -typing-extensions = ">=4.0.0" +typing-extensions = ">=4.4.0" [package.extras] -all = ["colossalai (>=0.2.0)", "deepspeed (>=0.6.0)", "fairscale (>=0.4.5)", "gym[classic-control] (>=0.17.0)", "hivemind (==1.1.5)", "horovod (>=0.21.2,!=0.24.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.7.1)", "jsonargparse[signatures] (>=4.18.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "rich (>=10.14.0,!=10.15.0.a)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.11.1)"] -colossalai = ["colossalai (>=0.2.0)"] -deepspeed = ["deepspeed (>=0.6.0)"] -dev = ["cloudpickle (>=1.3)", "codecov (==2.1.12)", "colossalai (>=0.2.0)", "coverage (==6.5.0)", "deepspeed (>=0.6.0)", "fairscale (>=0.4.5)", "fastapi (<0.87.0)", "gym[classic-control] (>=0.17.0)", "hivemind (==1.1.5)", "horovod (>=0.21.2,!=0.24.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.7.1)", "jsonargparse[signatures] (>=4.18.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "onnx (<1.14.0)", "onnxruntime (<1.14.0)", "pandas (>1.0)", "pre-commit (==2.20.0)", "protobuf (<=3.20.1)", "psutil (<5.9.5)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-forked (==1.4.0)", "pytest-rerunfailures (==10.3)", "rich (>=10.14.0,!=10.15.0.a)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.11.1)", "uvicorn (<0.19.1)"] -examples = ["gym[classic-control] (>=0.17.0)", "ipython[all] (<8.7.1)", "torchmetrics (>=0.10.0)", "torchvision (>=0.11.1)"] -extra = ["hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.18.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "rich (>=10.14.0,!=10.15.0.a)", "tensorboardX (>=2.2)"] -fairscale = ["fairscale (>=0.4.5)"] -hivemind = ["hivemind (==1.1.5)"] -horovod = ["horovod (>=0.21.2,!=0.24.0)"] -strategies = ["colossalai (>=0.2.0)", "deepspeed (>=0.6.0)", "fairscale (>=0.4.5)", "hivemind (==1.1.5)", "horovod (>=0.21.2,!=0.24.0)"] -test = ["cloudpickle (>=1.3)", "codecov (==2.1.12)", "coverage (==6.5.0)", "fastapi (<0.87.0)", "onnx (<1.14.0)", "onnxruntime (<1.14.0)", "pandas (>1.0)", "pre-commit (==2.20.0)", "protobuf (<=3.20.1)", "psutil (<5.9.5)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-forked (==1.4.0)", "pytest-rerunfailures (==10.3)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "uvicorn (<0.19.1)"] +all = ["bitsandbytes (>=0.42.0)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.15.0)"] +deepspeed = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] +dev = ["bitsandbytes (>=0.42.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "fastapi", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "onnx (>=0.14.0)", "onnxruntime (>=0.15.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.15.0)", "uvicorn"] +examples = ["ipython[all] (<8.15.0)", "lightning-utilities (>=0.8.0)", "requests (<2.32.0)", "torchmetrics (>=0.10.0)", "torchvision (>=0.15.0)"] +extra = ["bitsandbytes (>=0.42.0)", "hydra-core (>=1.2.0)", "jsonargparse[signatures] (>=4.27.7)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "rich (>=12.3.0)", "tensorboardX (>=2.2)"] +strategies = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] +test = ["cloudpickle (>=1.3)", "coverage (==7.3.1)", "fastapi", "onnx (>=0.14.0)", "onnxruntime (>=0.15.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "uvicorn"] [[package]] name = "pytz" @@ -2246,10 +2408,12 @@ version = "2023.3" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main", "docs"] files = [ {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +markers = {docs = "python_version == \"3.8\""} [[package]] name = "pyyaml" @@ -2257,6 +2421,7 @@ version = "6.0" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" +groups = ["main", "dev", "docs"] files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, @@ -2306,6 +2471,7 @@ version = "2.28.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.7, <4" +groups = ["main", "docs"] files = [ {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, @@ -2327,6 +2493,7 @@ version = "1.2.2" description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "scikit-learn-1.2.2.tar.gz", hash = "sha256:8429aea30ec24e7a8c7ed8a3fa6213adf3814a6efbea09e16e0a0c71e1a1a3d7"}, {file = "scikit_learn-1.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99cc01184e347de485bf253d19fcb3b1a3fb0ee4cea5ee3c43ec0cc429b6d29f"}, @@ -2369,6 +2536,7 @@ version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, @@ -2407,6 +2575,7 @@ version = "1.19.1" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sentry-sdk-1.19.1.tar.gz", hash = "sha256:7ae78bd921981a5010ab540d6bdf3b793659a4db8cccf7f16180702d48a80d84"}, {file = "sentry_sdk-1.19.1-py2.py3-none-any.whl", hash = "sha256:885a11c69df23e53eb281d003b9ff15a5bdfa43d8a2a53589be52104a1b4582f"}, @@ -2448,6 +2617,7 @@ version = "1.3.2" description = "A Python module to customize the process title" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "setproctitle-1.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:288943dec88e178bb2fd868adf491197cc0fc8b6810416b1c6775e686bab87fe"}, {file = "setproctitle-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:630f6fe5e24a619ccf970c78e084319ee8be5be253ecc9b5b216b0f474f5ef18"}, @@ -2532,6 +2702,7 @@ version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, @@ -2539,7 +2710,7 @@ files = [ [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov ; platform_python_implementation != \"PyPy\"", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2548,6 +2719,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main", "docs"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2559,6 +2731,7 @@ version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, @@ -2570,6 +2743,7 @@ version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." optional = false python-versions = "*" +groups = ["docs"] files = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, @@ -2581,6 +2755,7 @@ version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -2592,6 +2767,7 @@ version = "6.2.1" description = "Python documentation generator" optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "Sphinx-6.2.1.tar.gz", hash = "sha256:6d56a34697bb749ffa0152feafc4b19836c755d90a7c59b72bc7dfd371b9cc6b"}, {file = "sphinx-6.2.1-py3-none-any.whl", hash = "sha256:97787ff1fa3256a3eef9eda523a63dbf299f7b47e053cfcf684a1c2a8380c912"}, @@ -2627,6 +2803,7 @@ version = "2.1.6" description = "Type hints (PEP 484) support for the Sphinx autodoc extension" optional = false python-versions = "*" +groups = ["docs"] files = [ {file = "sphinx-autodoc-napoleon-typehints-2.1.6.tar.gz", hash = "sha256:67e36dc22d03362ab95c41d33b3a86fd8a7561a73648df112cdb57f989e1dae9"}, {file = "sphinx_autodoc_napoleon_typehints-2.1.6-py2.py3-none-any.whl", hash = "sha256:ad2f6a89eff950b30b65097f41a9f7065df168998a5fab16bb94df2d6b45e087"}, @@ -2643,6 +2820,7 @@ version = "1.2.4" description = "sphinx-nested-apidoc: When flattened is not enough" optional = false python-versions = ">=3.8,<4.0" +groups = ["docs"] files = [ {file = "sphinx_nested_apidoc-1.2.4-py3-none-any.whl", hash = "sha256:e33ccdea1fdaae9a5f47432750461aaa5aa6d391761cad3e4cb14425685d434f"}, {file = "sphinx_nested_apidoc-1.2.4.tar.gz", hash = "sha256:5ae842d35a918d3289ed3e3c899bf059e56b038bcfe061db3cead0c2b9a9f42b"}, @@ -2657,6 +2835,7 @@ version = "1.3.0rc1" description = "Read the Docs theme for Sphinx" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["docs"] files = [ {file = "sphinx_rtd_theme-1.3.0rc1-py2.py3-none-any.whl", hash = "sha256:ace3640f8951a93fd514fccd02071abac340c28fb0c907f180a7608c416e99a2"}, {file = "sphinx_rtd_theme-1.3.0rc1.tar.gz", hash = "sha256:3e321023694842feae0baed4f34004c4c925d812df9c93db49769c5887496d13"}, @@ -2676,6 +2855,7 @@ version = "1.0.4" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, @@ -2691,6 +2871,7 @@ version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." optional = false python-versions = ">=3.5" +groups = ["docs"] files = [ {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, @@ -2706,6 +2887,7 @@ version = "2.0.1" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = false python-versions = ">=3.8" +groups = ["docs"] files = [ {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, @@ -2721,6 +2903,7 @@ version = "4.1" description = "Extension to include jQuery on newer Sphinx releases" optional = false python-versions = ">=2.7" +groups = ["docs"] files = [ {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, @@ -2735,6 +2918,7 @@ version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" optional = false python-versions = ">=3.5" +groups = ["docs"] files = [ {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, @@ -2749,6 +2933,7 @@ version = "0.7" description = "Sphinx \"napoleon\" extension." optional = false python-versions = "*" +groups = ["docs"] files = [ {file = "sphinxcontrib-napoleon-0.7.tar.gz", hash = "sha256:407382beed396e9f2d7f3043fad6afda95719204a1e1a231ac865f40abcbfcf8"}, {file = "sphinxcontrib_napoleon-0.7-py2.py3-none-any.whl", hash = "sha256:711e41a3974bdf110a484aec4c1a556799eb0b3f3b897521a018ad7e2db13fef"}, @@ -2764,6 +2949,7 @@ version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." optional = false python-versions = ">=3.5" +groups = ["docs"] files = [ {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, @@ -2779,6 +2965,7 @@ version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." optional = false python-versions = ">=3.5" +groups = ["docs"] files = [ {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, @@ -2794,6 +2981,8 @@ version = "0.13.2" description = "Statistical computations and models for Python" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "platform_system == \"Windows\" and platform_machine == \"x86\" and python_version < \"3.10\"" files = [ {file = "statsmodels-0.13.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e7ca5b7e678c0bb7a24f5c735d58ac104a50eb61b17c484cce0e221a095560f"}, {file = "statsmodels-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:066a75d5585378b2df972f81a90b9a3da5e567b7d4833300c1597438c1a35e29"}, @@ -2832,58 +3021,14 @@ build = ["cython (>=0.29.26)"] develop = ["cython (>=0.29.26)"] docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"] -[[package]] -name = "statsmodels" -version = "0.13.3" -description = "Statistical computations and models for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "statsmodels-0.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b71bb64c6d4087dd6192eadfad390fbeb4074f676ef34c7e56579cead8c478e7"}, - {file = "statsmodels-0.13.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:658b634c273c2f287a0086e56a5d6b95ec3ddac991cbb020b34f731e932de0bd"}, - {file = "statsmodels-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab9f63f432889b179967ab645aea7480e28731823a3b99850d7f7a561b624f93"}, - {file = "statsmodels-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f432fb7f54ce5edccc83aa36566653cd04ee35bbbefdf0a2b7bd9c97c5da443"}, - {file = "statsmodels-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:4cd64076c3ad366b10fd4e6f8ca6aeb1e398ec5480bddb65fba8889dd9eb550d"}, - {file = "statsmodels-0.13.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:33f9caff2dbdfef22505678407d2f549b32a4a2729eb8675b60eb2932fc0e883"}, - {file = "statsmodels-0.13.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:393f6a7ec85f65be9ac1a13be152dd14c65084436c48bcdf94cb21ef0b6cb79c"}, - {file = "statsmodels-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12b56d13d9a2af7a1aadc3fe9f3d3c18a5727a651323d94e7c2047177adfb9ce"}, - {file = "statsmodels-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a61e0652f62b01981d8e857aa77550b42cf316c9d8e569b559869c248e3de834"}, - {file = "statsmodels-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:5368bccd471bb8cef0a8957ba5f2a3e5b5ecc433b0783d9f602039df45c780d3"}, - {file = "statsmodels-0.13.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1ecfb191958de187ba44b93316f4953b8b6588b5f68dcab218f76498a862dd7c"}, - {file = "statsmodels-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ea2b481b15e9e501904a1c36efc5f9a202f87529e600a99c364fd7e4598ae88"}, - {file = "statsmodels-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d270a11aac6720a8024e1136ab44036d0878f62995617bb5b9fc5c77ea3d3b8"}, - {file = "statsmodels-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2185ed356823cd1c258c09b790f0c21d2fd49321e82c79f8f6dc546f1c671d7a"}, - {file = "statsmodels-0.13.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9da39a36d114abcdcf8ebd351ed69229e23cb12b8a607996cb6511fa88e78b4d"}, - {file = "statsmodels-0.13.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3b3a9942d0b462af4c68c3895095d304869cbec9d97f3c268f19a6ba7ba294dc"}, - {file = "statsmodels-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fff0316420bc4f6fbd80dd77eb74f3834fcd0e4ca98ba9611b8a6d41ebbb979"}, - {file = "statsmodels-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352041bc04eaf90232e54a86861a460365ef45f34f58529578487e6f640dadf3"}, - {file = "statsmodels-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:61a0f39848ebacf5560e1539ca0037b8fc25cc9d1d7444bbef5bdc0a3c56087b"}, - {file = "statsmodels-0.13.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:78cd12b0ee543fa955d2bace18518fc7d2b57f13c65929b54445bf3e54955b08"}, - {file = "statsmodels-0.13.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:afccb80e3ddc969bfb5285f846ac2622861ffe192423087214d60e4c6e40e384"}, - {file = "statsmodels-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3609824e1ced44722bd905564d8ce94df29d24e32a6dd67cc9255932aedcd7b"}, - {file = "statsmodels-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81f8e71963a7bd169338fbb1472e34ec85ae4447414ac37bdae5cf6d1ac223bb"}, - {file = "statsmodels-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:000c7a1ce6780834f5fbb63f9ae07a00863a00f602c7c470c942153692f5bbc3"}, - {file = "statsmodels-0.13.3.tar.gz", hash = "sha256:ed71df887334b1d332e71d33215122bdd54494dcb2248606b30bcfa6112e860a"}, -] - -[package.dependencies] -numpy = {version = ">=1.17", markers = "python_version != \"3.10\" or platform_system != \"Windows\" or platform_python_implementation == \"PyPy\""} -packaging = ">=21.3" -pandas = ">=0.25" -patsy = ">=0.5.2" -scipy = {version = ">=1.3", markers = "(python_version > \"3.7\" or platform_system != \"Windows\" or platform_machine != \"x86\") and python_version < \"3.12\""} - -[package.extras] -build = ["cython (>=0.29.32)"] -develop = ["Jinja2", "colorama", "cython (>=0.29.32)", "cython (>=0.29.32,<3.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.0.1,<7.1.0)", "pytest-randomly", "pytest-xdist", "pywinpty", "setuptools-scm[toml] (>=7.0.0,<7.1.0)"] -docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"] - [[package]] name = "statsmodels" version = "0.13.5" description = "Statistical computations and models for Python" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "platform_machine != \"x86\" or platform_system != \"Windows\" or python_version >= \"3.10\"" files = [ {file = "statsmodels-0.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75319fddded9507cc310fc3980e4ae4d64e3ff37b322ad5e203a84f89d85203"}, {file = "statsmodels-0.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f148920ef27c7ba69a5735724f65de9422c0c8bcef71b50c846b823ceab8840"}, @@ -2924,7 +3069,7 @@ scipy = {version = ">=1.3", markers = "(python_version > \"3.9\" or platform_sys [package.extras] build = ["cython (>=0.29.32)"] -develop = ["Jinja2", "colorama", "cython (>=0.29.32)", "cython (>=0.29.32,<3.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.0.1,<7.1.0)", "pytest-randomly", "pytest-xdist", "pywinpty", "setuptools-scm[toml] (>=7.0.0,<7.1.0)"] +develop = ["Jinja2", "colorama", "cython (>=0.29.32)", "cython (>=0.29.32,<3.0.0)", "flake8", "isort", "joblib", "matplotlib (>=3)", "oldest-supported-numpy (>=2022.4.18)", "pytest (>=7.0.1,<7.1.0)", "pytest-randomly", "pytest-xdist", "pywinpty ; os_name == \"nt\"", "setuptools-scm[toml] (>=7.0.0,<7.1.0)"] docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"] [[package]] @@ -2933,6 +3078,7 @@ version = "1.11.1" description = "Computer algebra system (CAS) in Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "sympy-1.11.1-py3-none-any.whl", hash = "sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf"}, {file = "sympy-1.11.1.tar.gz", hash = "sha256:e32380dce63cb7c0108ed525570092fd45168bdae2faa17e528221ef72e88658"}, @@ -2947,6 +3093,7 @@ version = "1.10.4" description = "tasks runner for python projects" optional = false python-versions = ">=3.6,<4.0" +groups = ["dev"] files = [ {file = "taskipy-1.10.4-py3-none-any.whl", hash = "sha256:b96245d7f2956d36821435acaa822143ef6d2ff6cfecc48cf0a48c4b95456c14"}, {file = "taskipy-1.10.4.tar.gz", hash = "sha256:0006429f708f530fc7add28ca51fd41f6c6e1fbe86763ff05125e1af3e8bdc7e"}, @@ -2964,6 +3111,7 @@ version = "1.7.0" description = "Traceback serialization library." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] files = [ {file = "tblib-1.7.0-py2.py3-none-any.whl", hash = "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"}, {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, @@ -2975,6 +3123,7 @@ version = "0.1.35" description = "Read and write large, multi-dimensional arrays" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "tensorstore-0.1.35-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:4e1b4210b777c4a585a183bdd435bfa8aa5628c46075cb64adcd9b4bdd124e35"}, {file = "tensorstore-0.1.35-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285cfc4816bb0cc305cddc11f25f81a14bd84af0c8bbd39e42c81413c0bf242e"}, @@ -3004,6 +3153,7 @@ version = "3.1.0" description = "threadpoolctl" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "threadpoolctl-3.1.0-py3-none-any.whl", hash = "sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b"}, {file = "threadpoolctl-3.1.0.tar.gz", hash = "sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380"}, @@ -3015,10 +3165,12 @@ version = "2.0.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +markers = {main = "python_version <= \"3.10\""} [[package]] name = "toolz" @@ -3026,6 +3178,7 @@ version = "0.12.0" description = "List processing tools and functional utilities" optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, @@ -3037,6 +3190,7 @@ version = "2.0.0" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"}, {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"}, @@ -3092,6 +3246,7 @@ version = "2.0.1" description = "An audio package for PyTorch" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "torchaudio-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5d21ebbb55e7040d418d5062b0e882f9660d68b477b38fd436fa6c92ccbb52a"}, {file = "torchaudio-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dbcd93b29d71a2f500f36a34ea5e467f510f773da85322098e6bdd8c9dc9948"}, @@ -3124,6 +3279,7 @@ version = "0.11.4" description = "PyTorch native Metrics" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "torchmetrics-0.11.4-py3-none-any.whl", hash = "sha256:45f892f3534e91f3ad9e2488d1b05a93b7cb76b7d037969435a41a1f24750d9a"}, {file = "torchmetrics-0.11.4.tar.gz", hash = "sha256:1fe45a14b44dd65d90199017dd5a4b5a128d56a8a311da7916c402c18c671494"}, @@ -3150,6 +3306,7 @@ version = "0.15.1" description = "image and video datasets and models for torch deep learning" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "torchvision-0.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc10d48e9a60d006d0c1b48dea87f1ec9b63d856737d592f7c5c44cd87f3f4b7"}, {file = "torchvision-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3708d3410fdcaf6280e358cda9de2a4ab06cc0b4c0fd9aeeac550ec2563a887e"}, @@ -3188,6 +3345,7 @@ version = "6.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">= 3.7" +groups = ["main"] files = [ {file = "tornado-6.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72"}, {file = "tornado-6.2-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9"}, @@ -3208,6 +3366,7 @@ version = "4.65.0" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, @@ -3228,6 +3387,8 @@ version = "2.0.0" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, @@ -3238,15 +3399,6 @@ files = [ {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, - {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, - {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, - {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, - {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, - {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, - {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, - {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, - {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, - {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, ] [package.dependencies] @@ -3265,6 +3417,7 @@ version = "3.7.4.3" description = "Type Hints for Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["docs"] files = [ {file = "typing-3.7.4.3-py2-none-any.whl", hash = "sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5"}, {file = "typing-3.7.4.3.tar.gz", hash = "sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9"}, @@ -3276,6 +3429,7 @@ version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, @@ -3287,14 +3441,15 @@ version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["main", "docs"] files = [ {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +brotli = ["brotli (>=1.0.9) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -3303,6 +3458,7 @@ version = "20.23.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "virtualenv-20.23.1-py3-none-any.whl", hash = "sha256:34da10f14fea9be20e0fd7f04aba9732f84e593dac291b757ce42e3368a39419"}, {file = "virtualenv-20.23.1.tar.gz", hash = "sha256:8ff19a38c1021c742148edc4f81cb43d7f8c6816d2ede2ab72af5b84c749ade1"}, @@ -3315,7 +3471,7 @@ platformdirs = ">=3.5.1,<4" [package.extras] docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezer (>=0.4.6)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.8)", "time-machine (>=2.9)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezer (>=0.4.6) ; platform_python_implementation == \"PyPy\"", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.8)", "time-machine (>=2.9) ; platform_python_implementation == \"CPython\""] [[package]] name = "wandb" @@ -3323,6 +3479,7 @@ version = "0.13.11" description = "A CLI and library for interacting with the Weights and Biases API." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "wandb-0.13.11-py3-none-any.whl", hash = "sha256:e584ac7fb0097fba8a030d5aedc5fea157286aba69a2e6e6e689b794155fcc8f"}, {file = "wandb-0.13.11.tar.gz", hash = "sha256:f2b065d1c732e8e1828fabe720dc82f12e6109d0d2bbfbcccc17eaefae0fd7c9"}, @@ -3364,6 +3521,7 @@ version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, @@ -3375,6 +3533,8 @@ version = "0.40.0" description = "A built-package format for Python" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" files = [ {file = "wheel-0.40.0-py3-none-any.whl", hash = "sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247"}, {file = "wheel-0.40.0.tar.gz", hash = "sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873"}, @@ -3389,6 +3549,7 @@ version = "3.1.0" description = "A Python module for creating Excel XLSX files." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "XlsxWriter-3.1.0-py3-none-any.whl", hash = "sha256:b70a147d36235d1ee835cfd037396f789db1f76740a0e5c917d54137169341de"}, {file = "XlsxWriter-3.1.0.tar.gz", hash = "sha256:02913b50b74c00f165933d5da3e3a02cab4204cb4932722a1b342c5c71034122"}, @@ -3400,6 +3561,7 @@ version = "1.8.2" description = "Yet another URL library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, @@ -3487,6 +3649,7 @@ version = "2.2.0" description = "Mutable mapping tools" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "zict-2.2.0-py2.py3-none-any.whl", hash = "sha256:dabcc8c8b6833aa3b6602daad50f03da068322c1a90999ff78aed9eecc8fa92c"}, {file = "zict-2.2.0.tar.gz", hash = "sha256:d7366c2e2293314112dcf2432108428a67b927b00005619feefc310d12d833f3"}, @@ -3501,14 +3664,16 @@ version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.7" +groups = ["main", "docs"] files = [ {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, ] +markers = {docs = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8 ; python_version < \"3.12\"", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "zstandard" @@ -3516,6 +3681,7 @@ version = "0.19.0" description = "Zstandard bindings for Python" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, @@ -3577,6 +3743,6 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.8" -content-hash = "279566202222450bbd6667dd6188e5b783a17ea9f69ba070400b71405db008cc" +content-hash = "d5636486f7dd86a5bd8046dc6a7256d455cab2d62751ba06062bd3de465c5f0a" diff --git a/pyproject.toml b/pyproject.toml index 70b1270..a680997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "udm" -version = "0.2.0" +version = "1.0.0" description = "DataModule used for unified access to UKBB data across BIH projects." authors = [ "Luis Herrmann ", @@ -48,7 +48,7 @@ scipy = {version = "^1.9.3", source = "pypi"} tqdm = {version = "^4.64.1", source = "pypi"} wandb = {version = "^0.13.6", source = "pypi"} zstandard = {version = "^0.19.0", source = "pypi"} -pytorch-lightning = {version = "^1.8.3.post1", source = "pypi"} +pytorch-lightning = {version = "^2.3.3.0", source = "pypi"} pytest = "^7.2.1" dill = "^0.3.6" tensorstore = "^0.1.33" diff --git a/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.X.npy b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..130c2af Binary files /dev/null and b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.obs.csv b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.var.csv b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_extended_tr.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.X.npy b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.X.npy new file mode 100644 index 0000000..11e59b0 Binary files /dev/null and b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.X.npy differ diff --git a/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.obs.csv b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.var.csv b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_test.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.X.npy b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.X.npy new file mode 100644 index 0000000..5859ab1 Binary files /dev/null and b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.obs.csv b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.var.csv b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/additional_pheno/additional_pheno_valid.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..ce3e618 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..0032efc --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_extended_tr.h5ad.var.csv @@ -0,0 +1,10 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,genetic_principal_components_f22009_0_1,float,"{""zscore"": {""std"": [2948.33267068], ""mean"": [-1.12466556]}}" +1,2,genetic_principal_components_f22009_0_2,float,"{""zscore"": {""std"": [776.16257928], ""mean"": [0.35880318]}}" +2,3,genetic_principal_components_f22009_0_3,float,"{""zscore"": {""std"": [221.79799481], ""mean"": [-0.08920376]}}" +3,4,genetic_principal_components_f22009_0_4,float,"{""zscore"": {""std"": [108.2274961], ""mean"": [0.12935057]}}" +4,5,genetic_principal_components_f22009_0_5,float,"{""zscore"": {""std"": [57.29823059], ""mean"": [0.06411575]}}" +5,6,age,float,"{""zscore"": {""std"": [65.83748872], ""mean"": [68.67234809]}}" +6,7,batch,uint8, +7,8,sex_f31_0_0_Female,uint8, +8,9,sex_f31_0_0_Male,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_base_test.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.X.npy new file mode 100644 index 0000000..f4ba1d8 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_base_test.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/covariate/covariates_new_base_test.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.var.csv new file mode 100644 index 0000000..0032efc --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_test.h5ad.var.csv @@ -0,0 +1,10 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,genetic_principal_components_f22009_0_1,float,"{""zscore"": {""std"": [2948.33267068], ""mean"": [-1.12466556]}}" +1,2,genetic_principal_components_f22009_0_2,float,"{""zscore"": {""std"": [776.16257928], ""mean"": [0.35880318]}}" +2,3,genetic_principal_components_f22009_0_3,float,"{""zscore"": {""std"": [221.79799481], ""mean"": [-0.08920376]}}" +3,4,genetic_principal_components_f22009_0_4,float,"{""zscore"": {""std"": [108.2274961], ""mean"": [0.12935057]}}" +4,5,genetic_principal_components_f22009_0_5,float,"{""zscore"": {""std"": [57.29823059], ""mean"": [0.06411575]}}" +5,6,age,float,"{""zscore"": {""std"": [65.83748872], ""mean"": [68.67234809]}}" +6,7,batch,uint8, +7,8,sex_f31_0_0_Female,uint8, +8,9,sex_f31_0_0_Male,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_base_train.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.X.npy new file mode 100644 index 0000000..820c0ad Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_base_train.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_base_train.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.var.csv new file mode 100644 index 0000000..0032efc --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_train.h5ad.var.csv @@ -0,0 +1,10 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,genetic_principal_components_f22009_0_1,float,"{""zscore"": {""std"": [2948.33267068], ""mean"": [-1.12466556]}}" +1,2,genetic_principal_components_f22009_0_2,float,"{""zscore"": {""std"": [776.16257928], ""mean"": [0.35880318]}}" +2,3,genetic_principal_components_f22009_0_3,float,"{""zscore"": {""std"": [221.79799481], ""mean"": [-0.08920376]}}" +3,4,genetic_principal_components_f22009_0_4,float,"{""zscore"": {""std"": [108.2274961], ""mean"": [0.12935057]}}" +4,5,genetic_principal_components_f22009_0_5,float,"{""zscore"": {""std"": [57.29823059], ""mean"": [0.06411575]}}" +5,6,age,float,"{""zscore"": {""std"": [65.83748872], ""mean"": [68.67234809]}}" +6,7,batch,uint8, +7,8,sex_f31_0_0_Female,uint8, +8,9,sex_f31_0_0_Male,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.X.npy new file mode 100644 index 0000000..18b7c83 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.var.csv new file mode 100644 index 0000000..0032efc --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_base_valid.h5ad.var.csv @@ -0,0 +1,10 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,genetic_principal_components_f22009_0_1,float,"{""zscore"": {""std"": [2948.33267068], ""mean"": [-1.12466556]}}" +1,2,genetic_principal_components_f22009_0_2,float,"{""zscore"": {""std"": [776.16257928], ""mean"": [0.35880318]}}" +2,3,genetic_principal_components_f22009_0_3,float,"{""zscore"": {""std"": [221.79799481], ""mean"": [-0.08920376]}}" +3,4,genetic_principal_components_f22009_0_4,float,"{""zscore"": {""std"": [108.2274961], ""mean"": [0.12935057]}}" +4,5,genetic_principal_components_f22009_0_5,float,"{""zscore"": {""std"": [57.29823059], ""mean"": [0.06411575]}}" +5,6,age,float,"{""zscore"": {""std"": [65.83748872], ""mean"": [68.67234809]}}" +6,7,batch,uint8, +7,8,sex_f31_0_0_Female,uint8, +8,9,sex_f31_0_0_Male,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..7b49035 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..62c15c7 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_extended_tr.h5ad.var.csv @@ -0,0 +1,38 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,medications_drugs_for_acid_related_disorders,uint8, +1,2,medications_vitamins,uint8, +2,3,medications_diuretics,uint8, +3,4,medications_beta_blocking_agents,uint8, +4,5,medications_agents_acting_on_the_renin-angiotensin_system,uint8, +5,6,medications_lipid_modifying_agents,uint8, +6,7,medications_thyroid_therapy,uint8, +7,8,medications_antiinflammatory_and_antirheumatic_products,uint8, +8,9,medications_analgesics,uint8, +9,10,medications_psychoanaleptics,uint8, +10,11,medications_nasal_preparations,uint8, +11,12,medications_drugs_for_obstructive_airway_diseases,uint8, +12,13,medications_statins,uint8, +13,14,family_fh_alzheimer's_disease/dementia,uint8, +14,15,family_fh_bowel_cancer,uint8, +15,16,family_fh_breast_cancer,uint8, +16,17,family_fh_chronic_bronchitis/emphysema,uint8, +17,18,family_fh_diabetes,uint8, +18,19,family_fh_heart_disease,uint8, +19,20,family_fh_high_blood_pressure,uint8, +20,21,family_fh_lung_cancer,uint8, +21,22,family_fh_parkinson's_disease,uint8, +22,23,family_fh_severe_depression,uint8, +23,24,family_fh_stroke,uint8, +24,25,questionaire_overall_health_rating_f2178_0_0_Poor,uint8, +25,26,questionaire_overall_health_rating_f2178_0_0_Fair,uint8, +26,27,questionaire_overall_health_rating_f2178_0_0_Good,uint8, +27,28,questionaire_overall_health_rating_f2178_0_0_Excellent,uint8, +28,29,questionaire_smoking_status_f20116_0_0_Current,uint8, +29,30,questionaire_smoking_status_f20116_0_0_Previous,uint8, +30,31,questionaire_smoking_status_f20116_0_0_Never,uint8, +31,32,questionaire_alcohol_intake_frequency_f1558_0_0_Daily or almost daily,uint8, +32,33,questionaire_alcohol_intake_frequency_f1558_0_0_Three or four times a week,uint8, +33,34,questionaire_alcohol_intake_frequency_f1558_0_0_Once or twice a week,uint8, +34,35,questionaire_alcohol_intake_frequency_f1558_0_0_One to three times a month,uint8, +35,36,questionaire_alcohol_intake_frequency_f1558_0_0_Special occasions only,uint8, +36,37,questionaire_alcohol_intake_frequency_f1558_0_0_Never,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.X.npy new file mode 100644 index 0000000..3afc149 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.var.csv new file mode 100644 index 0000000..62c15c7 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_test.h5ad.var.csv @@ -0,0 +1,38 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,medications_drugs_for_acid_related_disorders,uint8, +1,2,medications_vitamins,uint8, +2,3,medications_diuretics,uint8, +3,4,medications_beta_blocking_agents,uint8, +4,5,medications_agents_acting_on_the_renin-angiotensin_system,uint8, +5,6,medications_lipid_modifying_agents,uint8, +6,7,medications_thyroid_therapy,uint8, +7,8,medications_antiinflammatory_and_antirheumatic_products,uint8, +8,9,medications_analgesics,uint8, +9,10,medications_psychoanaleptics,uint8, +10,11,medications_nasal_preparations,uint8, +11,12,medications_drugs_for_obstructive_airway_diseases,uint8, +12,13,medications_statins,uint8, +13,14,family_fh_alzheimer's_disease/dementia,uint8, +14,15,family_fh_bowel_cancer,uint8, +15,16,family_fh_breast_cancer,uint8, +16,17,family_fh_chronic_bronchitis/emphysema,uint8, +17,18,family_fh_diabetes,uint8, +18,19,family_fh_heart_disease,uint8, +19,20,family_fh_high_blood_pressure,uint8, +20,21,family_fh_lung_cancer,uint8, +21,22,family_fh_parkinson's_disease,uint8, +22,23,family_fh_severe_depression,uint8, +23,24,family_fh_stroke,uint8, +24,25,questionaire_overall_health_rating_f2178_0_0_Poor,uint8, +25,26,questionaire_overall_health_rating_f2178_0_0_Fair,uint8, +26,27,questionaire_overall_health_rating_f2178_0_0_Good,uint8, +27,28,questionaire_overall_health_rating_f2178_0_0_Excellent,uint8, +28,29,questionaire_smoking_status_f20116_0_0_Current,uint8, +29,30,questionaire_smoking_status_f20116_0_0_Previous,uint8, +30,31,questionaire_smoking_status_f20116_0_0_Never,uint8, +31,32,questionaire_alcohol_intake_frequency_f1558_0_0_Daily or almost daily,uint8, +32,33,questionaire_alcohol_intake_frequency_f1558_0_0_Three or four times a week,uint8, +33,34,questionaire_alcohol_intake_frequency_f1558_0_0_Once or twice a week,uint8, +34,35,questionaire_alcohol_intake_frequency_f1558_0_0_One to three times a month,uint8, +35,36,questionaire_alcohol_intake_frequency_f1558_0_0_Special occasions only,uint8, +36,37,questionaire_alcohol_intake_frequency_f1558_0_0_Never,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.X.npy new file mode 100644 index 0000000..3afc149 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.var.csv new file mode 100644 index 0000000..62c15c7 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_train.h5ad.var.csv @@ -0,0 +1,38 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,medications_drugs_for_acid_related_disorders,uint8, +1,2,medications_vitamins,uint8, +2,3,medications_diuretics,uint8, +3,4,medications_beta_blocking_agents,uint8, +4,5,medications_agents_acting_on_the_renin-angiotensin_system,uint8, +5,6,medications_lipid_modifying_agents,uint8, +6,7,medications_thyroid_therapy,uint8, +7,8,medications_antiinflammatory_and_antirheumatic_products,uint8, +8,9,medications_analgesics,uint8, +9,10,medications_psychoanaleptics,uint8, +10,11,medications_nasal_preparations,uint8, +11,12,medications_drugs_for_obstructive_airway_diseases,uint8, +12,13,medications_statins,uint8, +13,14,family_fh_alzheimer's_disease/dementia,uint8, +14,15,family_fh_bowel_cancer,uint8, +15,16,family_fh_breast_cancer,uint8, +16,17,family_fh_chronic_bronchitis/emphysema,uint8, +17,18,family_fh_diabetes,uint8, +18,19,family_fh_heart_disease,uint8, +19,20,family_fh_high_blood_pressure,uint8, +20,21,family_fh_lung_cancer,uint8, +21,22,family_fh_parkinson's_disease,uint8, +22,23,family_fh_severe_depression,uint8, +23,24,family_fh_stroke,uint8, +24,25,questionaire_overall_health_rating_f2178_0_0_Poor,uint8, +25,26,questionaire_overall_health_rating_f2178_0_0_Fair,uint8, +26,27,questionaire_overall_health_rating_f2178_0_0_Good,uint8, +27,28,questionaire_overall_health_rating_f2178_0_0_Excellent,uint8, +28,29,questionaire_smoking_status_f20116_0_0_Current,uint8, +29,30,questionaire_smoking_status_f20116_0_0_Previous,uint8, +30,31,questionaire_smoking_status_f20116_0_0_Never,uint8, +31,32,questionaire_alcohol_intake_frequency_f1558_0_0_Daily or almost daily,uint8, +32,33,questionaire_alcohol_intake_frequency_f1558_0_0_Three or four times a week,uint8, +33,34,questionaire_alcohol_intake_frequency_f1558_0_0_Once or twice a week,uint8, +34,35,questionaire_alcohol_intake_frequency_f1558_0_0_One to three times a month,uint8, +35,36,questionaire_alcohol_intake_frequency_f1558_0_0_Special occasions only,uint8, +36,37,questionaire_alcohol_intake_frequency_f1558_0_0_Never,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.X.npy new file mode 100644 index 0000000..3afc149 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.var.csv new file mode 100644 index 0000000..62c15c7 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_categorical_valid.h5ad.var.csv @@ -0,0 +1,38 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,medications_drugs_for_acid_related_disorders,uint8, +1,2,medications_vitamins,uint8, +2,3,medications_diuretics,uint8, +3,4,medications_beta_blocking_agents,uint8, +4,5,medications_agents_acting_on_the_renin-angiotensin_system,uint8, +5,6,medications_lipid_modifying_agents,uint8, +6,7,medications_thyroid_therapy,uint8, +7,8,medications_antiinflammatory_and_antirheumatic_products,uint8, +8,9,medications_analgesics,uint8, +9,10,medications_psychoanaleptics,uint8, +10,11,medications_nasal_preparations,uint8, +11,12,medications_drugs_for_obstructive_airway_diseases,uint8, +12,13,medications_statins,uint8, +13,14,family_fh_alzheimer's_disease/dementia,uint8, +14,15,family_fh_bowel_cancer,uint8, +15,16,family_fh_breast_cancer,uint8, +16,17,family_fh_chronic_bronchitis/emphysema,uint8, +17,18,family_fh_diabetes,uint8, +18,19,family_fh_heart_disease,uint8, +19,20,family_fh_high_blood_pressure,uint8, +20,21,family_fh_lung_cancer,uint8, +21,22,family_fh_parkinson's_disease,uint8, +22,23,family_fh_severe_depression,uint8, +23,24,family_fh_stroke,uint8, +24,25,questionaire_overall_health_rating_f2178_0_0_Poor,uint8, +25,26,questionaire_overall_health_rating_f2178_0_0_Fair,uint8, +26,27,questionaire_overall_health_rating_f2178_0_0_Good,uint8, +27,28,questionaire_overall_health_rating_f2178_0_0_Excellent,uint8, +28,29,questionaire_smoking_status_f20116_0_0_Current,uint8, +29,30,questionaire_smoking_status_f20116_0_0_Previous,uint8, +30,31,questionaire_smoking_status_f20116_0_0_Never,uint8, +31,32,questionaire_alcohol_intake_frequency_f1558_0_0_Daily or almost daily,uint8, +32,33,questionaire_alcohol_intake_frequency_f1558_0_0_Three or four times a week,uint8, +33,34,questionaire_alcohol_intake_frequency_f1558_0_0_Once or twice a week,uint8, +34,35,questionaire_alcohol_intake_frequency_f1558_0_0_One to three times a month,uint8, +35,36,questionaire_alcohol_intake_frequency_f1558_0_0_Special occasions only,uint8, +36,37,questionaire_alcohol_intake_frequency_f1558_0_0_Never,uint8, diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..7fc9967 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..ca6f7fd --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_extended_tr.h5ad.var.csv @@ -0,0 +1,82 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,questionaire_townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,measurements_body_mass_index_bmi_f21001_0_0,float,"{""zscore"": {""std"": [23.06975408], ""mean"": [27.43260556]}}" +2,3,measurements_weight_f21002_0_0,float,"{""zscore"": {""std"": [254.26876274], ""mean"": [78.05452345]}}" +3,4,measurements_pulse_wave_arterial_stiffness_index_f21021_0_0,float,"{""zscore"": {""std"": [16.36903814], ""mean"": [9.33767979]}}" +4,5,measurements_pulse_wave_reflection_index_f4195_0_0,float,"{""zscore"": {""std"": [1035.5975347], ""mean"": [67.728354]}}" +5,6,measurements_waist_circumference_f48_0_0,float,"{""zscore"": {""std"": [181.86851834], ""mean"": [90.31081898]}}" +6,7,measurements_hip_circumference_f49_0_0,float,"{""zscore"": {""std"": [85.55373922], ""mean"": [103.40158394]}}" +7,8,measurements_standing_height_f50_0_0,float,"{""zscore"": {""std"": [86.10089505], ""mean"": [168.44354526]}}" +8,9,measurements_trunk_fat_percentage_f23127_0_0,float,"{""zscore"": {""std"": [64.13611621], ""mean"": [31.17342071]}}" +9,10,measurements_body_fat_percentage_f23099_0_0,float,"{""zscore"": {""std"": [73.06088003], ""mean"": [31.45178684]}}" +10,11,measurements_basal_metabolic_rate_f23105_0_0,float,"{""zscore"": {""std"": [1861301.06090653], ""mean"": [6616.13365882]}}" +11,12,measurements_forced_vital_capacity_fvc_best_measure_f20151_0_0,float,"{""zscore"": {""std"": [0.96988716], ""mean"": [3.77788852]}}" +12,13,measurements_forced_expiratory_volume_in_1second_fev1_best_measure_f20150_0_0,float,"{""zscore"": {""std"": [0.60848128], ""mean"": [2.85074123]}}" +13,14,measurements_fev1_fvc_ratio_zscore_f20258_0_0,float,"{""zscore"": {""std"": [0.78797162], ""mean"": [0.41824227]}}" +14,15,measurements_peak_expiratory_flow_pef_f3064_0_2,float,"{""zscore"": {""std"": [23559.06009175], ""mean"": [325.81174618]}}" +15,16,measurements_peak_expiratory_flow_pef_f3064_0_1,float,"{""zscore"": {""std"": [19178.83974267], ""mean"": [372.7427792]}}" +16,17,measurements_peak_expiratory_flow_pef_f3064_0_0,float,"{""zscore"": {""std"": [18383.4861072], ""mean"": [388.39422424]}}" +17,18,measurements_systolic_blood_pressure_automated_reading_f4080,float,"{""zscore"": {""std"": [348.89967781], ""mean"": [137.82152088]}}" +18,19,measurements_diastolic_blood_pressure_automated_reading_f4079,float,"{""zscore"": {""std"": [103.14531112], ""mean"": [82.21547217]}}" +19,20,measurements_pulse_rate_automated_reading_f102,float,"{""zscore"": {""std"": [126.85754528], ""mean"": [69.3559183]}}" +20,21,labs_basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +21,22,labs_basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +22,23,labs_eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +23,24,labs_eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +24,25,labs_haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +25,26,labs_haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +26,27,labs_high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +27,28,labs_high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +28,29,labs_immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +29,30,labs_lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +30,31,labs_lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +31,32,labs_mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +32,33,labs_mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +33,34,labs_mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +34,35,labs_mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +35,36,labs_mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +36,37,labs_mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +37,38,labs_monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +38,39,labs_monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +39,40,labs_neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +40,41,labs_neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +41,42,labs_nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +42,43,labs_nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +43,44,labs_platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +44,45,labs_platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +45,46,labs_platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +46,47,labs_red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +47,48,labs_red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +48,49,labs_reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +49,50,labs_reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +50,51,labs_white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +51,52,labs_alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +52,53,labs_albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +53,54,labs_alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +54,55,labs_apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +55,56,labs_apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +56,57,labs_aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +57,58,labs_creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +58,59,labs_calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +59,60,labs_cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +60,61,labs_creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +61,62,labs_cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +62,63,labs_direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +63,64,labs_gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +64,65,labs_glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +65,66,labs_glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +66,67,labs_hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +67,68,labs_igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +68,69,labs_ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +69,70,labs_lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +70,71,labs_oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +71,72,labs_phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +72,73,labs_rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +73,74,labs_shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +74,75,labs_testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +75,76,labs_total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +76,77,labs_total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +77,78,labs_triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +78,79,labs_urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +79,80,labs_urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +80,81,labs_vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.X.npy new file mode 100644 index 0000000..d66e427 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.var.csv new file mode 100644 index 0000000..ca6f7fd --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_test.h5ad.var.csv @@ -0,0 +1,82 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,questionaire_townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,measurements_body_mass_index_bmi_f21001_0_0,float,"{""zscore"": {""std"": [23.06975408], ""mean"": [27.43260556]}}" +2,3,measurements_weight_f21002_0_0,float,"{""zscore"": {""std"": [254.26876274], ""mean"": [78.05452345]}}" +3,4,measurements_pulse_wave_arterial_stiffness_index_f21021_0_0,float,"{""zscore"": {""std"": [16.36903814], ""mean"": [9.33767979]}}" +4,5,measurements_pulse_wave_reflection_index_f4195_0_0,float,"{""zscore"": {""std"": [1035.5975347], ""mean"": [67.728354]}}" +5,6,measurements_waist_circumference_f48_0_0,float,"{""zscore"": {""std"": [181.86851834], ""mean"": [90.31081898]}}" +6,7,measurements_hip_circumference_f49_0_0,float,"{""zscore"": {""std"": [85.55373922], ""mean"": [103.40158394]}}" +7,8,measurements_standing_height_f50_0_0,float,"{""zscore"": {""std"": [86.10089505], ""mean"": [168.44354526]}}" +8,9,measurements_trunk_fat_percentage_f23127_0_0,float,"{""zscore"": {""std"": [64.13611621], ""mean"": [31.17342071]}}" +9,10,measurements_body_fat_percentage_f23099_0_0,float,"{""zscore"": {""std"": [73.06088003], ""mean"": [31.45178684]}}" +10,11,measurements_basal_metabolic_rate_f23105_0_0,float,"{""zscore"": {""std"": [1861301.06090653], ""mean"": [6616.13365882]}}" +11,12,measurements_forced_vital_capacity_fvc_best_measure_f20151_0_0,float,"{""zscore"": {""std"": [0.96988716], ""mean"": [3.77788852]}}" +12,13,measurements_forced_expiratory_volume_in_1second_fev1_best_measure_f20150_0_0,float,"{""zscore"": {""std"": [0.60848128], ""mean"": [2.85074123]}}" +13,14,measurements_fev1_fvc_ratio_zscore_f20258_0_0,float,"{""zscore"": {""std"": [0.78797162], ""mean"": [0.41824227]}}" +14,15,measurements_peak_expiratory_flow_pef_f3064_0_2,float,"{""zscore"": {""std"": [23559.06009175], ""mean"": [325.81174618]}}" +15,16,measurements_peak_expiratory_flow_pef_f3064_0_1,float,"{""zscore"": {""std"": [19178.83974267], ""mean"": [372.7427792]}}" +16,17,measurements_peak_expiratory_flow_pef_f3064_0_0,float,"{""zscore"": {""std"": [18383.4861072], ""mean"": [388.39422424]}}" +17,18,measurements_systolic_blood_pressure_automated_reading_f4080,float,"{""zscore"": {""std"": [348.89967781], ""mean"": [137.82152088]}}" +18,19,measurements_diastolic_blood_pressure_automated_reading_f4079,float,"{""zscore"": {""std"": [103.14531112], ""mean"": [82.21547217]}}" +19,20,measurements_pulse_rate_automated_reading_f102,float,"{""zscore"": {""std"": [126.85754528], ""mean"": [69.3559183]}}" +20,21,labs_basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +21,22,labs_basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +22,23,labs_eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +23,24,labs_eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +24,25,labs_haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +25,26,labs_haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +26,27,labs_high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +27,28,labs_high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +28,29,labs_immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +29,30,labs_lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +30,31,labs_lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +31,32,labs_mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +32,33,labs_mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +33,34,labs_mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +34,35,labs_mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +35,36,labs_mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +36,37,labs_mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +37,38,labs_monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +38,39,labs_monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +39,40,labs_neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +40,41,labs_neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +41,42,labs_nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +42,43,labs_nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +43,44,labs_platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +44,45,labs_platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +45,46,labs_platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +46,47,labs_red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +47,48,labs_red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +48,49,labs_reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +49,50,labs_reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +50,51,labs_white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +51,52,labs_alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +52,53,labs_albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +53,54,labs_alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +54,55,labs_apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +55,56,labs_apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +56,57,labs_aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +57,58,labs_creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +58,59,labs_calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +59,60,labs_cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +60,61,labs_creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +61,62,labs_cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +62,63,labs_direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +63,64,labs_gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +64,65,labs_glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +65,66,labs_glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +66,67,labs_hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +67,68,labs_igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +68,69,labs_ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +69,70,labs_lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +70,71,labs_oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +71,72,labs_phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +72,73,labs_rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +73,74,labs_shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +74,75,labs_testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +75,76,labs_total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +76,77,labs_total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +77,78,labs_triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +78,79,labs_urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +79,80,labs_urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +80,81,labs_vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.X.npy new file mode 100644 index 0000000..a6e6dcc Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.var.csv new file mode 100644 index 0000000..ca6f7fd --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_train.h5ad.var.csv @@ -0,0 +1,82 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,questionaire_townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,measurements_body_mass_index_bmi_f21001_0_0,float,"{""zscore"": {""std"": [23.06975408], ""mean"": [27.43260556]}}" +2,3,measurements_weight_f21002_0_0,float,"{""zscore"": {""std"": [254.26876274], ""mean"": [78.05452345]}}" +3,4,measurements_pulse_wave_arterial_stiffness_index_f21021_0_0,float,"{""zscore"": {""std"": [16.36903814], ""mean"": [9.33767979]}}" +4,5,measurements_pulse_wave_reflection_index_f4195_0_0,float,"{""zscore"": {""std"": [1035.5975347], ""mean"": [67.728354]}}" +5,6,measurements_waist_circumference_f48_0_0,float,"{""zscore"": {""std"": [181.86851834], ""mean"": [90.31081898]}}" +6,7,measurements_hip_circumference_f49_0_0,float,"{""zscore"": {""std"": [85.55373922], ""mean"": [103.40158394]}}" +7,8,measurements_standing_height_f50_0_0,float,"{""zscore"": {""std"": [86.10089505], ""mean"": [168.44354526]}}" +8,9,measurements_trunk_fat_percentage_f23127_0_0,float,"{""zscore"": {""std"": [64.13611621], ""mean"": [31.17342071]}}" +9,10,measurements_body_fat_percentage_f23099_0_0,float,"{""zscore"": {""std"": [73.06088003], ""mean"": [31.45178684]}}" +10,11,measurements_basal_metabolic_rate_f23105_0_0,float,"{""zscore"": {""std"": [1861301.06090653], ""mean"": [6616.13365882]}}" +11,12,measurements_forced_vital_capacity_fvc_best_measure_f20151_0_0,float,"{""zscore"": {""std"": [0.96988716], ""mean"": [3.77788852]}}" +12,13,measurements_forced_expiratory_volume_in_1second_fev1_best_measure_f20150_0_0,float,"{""zscore"": {""std"": [0.60848128], ""mean"": [2.85074123]}}" +13,14,measurements_fev1_fvc_ratio_zscore_f20258_0_0,float,"{""zscore"": {""std"": [0.78797162], ""mean"": [0.41824227]}}" +14,15,measurements_peak_expiratory_flow_pef_f3064_0_2,float,"{""zscore"": {""std"": [23559.06009175], ""mean"": [325.81174618]}}" +15,16,measurements_peak_expiratory_flow_pef_f3064_0_1,float,"{""zscore"": {""std"": [19178.83974267], ""mean"": [372.7427792]}}" +16,17,measurements_peak_expiratory_flow_pef_f3064_0_0,float,"{""zscore"": {""std"": [18383.4861072], ""mean"": [388.39422424]}}" +17,18,measurements_systolic_blood_pressure_automated_reading_f4080,float,"{""zscore"": {""std"": [348.89967781], ""mean"": [137.82152088]}}" +18,19,measurements_diastolic_blood_pressure_automated_reading_f4079,float,"{""zscore"": {""std"": [103.14531112], ""mean"": [82.21547217]}}" +19,20,measurements_pulse_rate_automated_reading_f102,float,"{""zscore"": {""std"": [126.85754528], ""mean"": [69.3559183]}}" +20,21,labs_basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +21,22,labs_basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +22,23,labs_eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +23,24,labs_eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +24,25,labs_haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +25,26,labs_haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +26,27,labs_high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +27,28,labs_high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +28,29,labs_immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +29,30,labs_lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +30,31,labs_lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +31,32,labs_mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +32,33,labs_mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +33,34,labs_mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +34,35,labs_mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +35,36,labs_mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +36,37,labs_mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +37,38,labs_monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +38,39,labs_monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +39,40,labs_neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +40,41,labs_neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +41,42,labs_nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +42,43,labs_nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +43,44,labs_platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +44,45,labs_platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +45,46,labs_platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +46,47,labs_red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +47,48,labs_red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +48,49,labs_reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +49,50,labs_reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +50,51,labs_white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +51,52,labs_alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +52,53,labs_albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +53,54,labs_alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +54,55,labs_apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +55,56,labs_apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +56,57,labs_aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +57,58,labs_creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +58,59,labs_calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +59,60,labs_cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +60,61,labs_creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +61,62,labs_cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +62,63,labs_direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +63,64,labs_gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +64,65,labs_glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +65,66,labs_glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +66,67,labs_hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +67,68,labs_igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +68,69,labs_ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +69,70,labs_lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +70,71,labs_oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +71,72,labs_phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +72,73,labs_rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +73,74,labs_shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +74,75,labs_testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +75,76,labs_total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +76,77,labs_total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +77,78,labs_triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +78,79,labs_urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +79,80,labs_urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +80,81,labs_vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.X.npy new file mode 100644 index 0000000..0efd78f Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.var.csv new file mode 100644 index 0000000..ca6f7fd --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_continous_valid.h5ad.var.csv @@ -0,0 +1,82 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,questionaire_townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,measurements_body_mass_index_bmi_f21001_0_0,float,"{""zscore"": {""std"": [23.06975408], ""mean"": [27.43260556]}}" +2,3,measurements_weight_f21002_0_0,float,"{""zscore"": {""std"": [254.26876274], ""mean"": [78.05452345]}}" +3,4,measurements_pulse_wave_arterial_stiffness_index_f21021_0_0,float,"{""zscore"": {""std"": [16.36903814], ""mean"": [9.33767979]}}" +4,5,measurements_pulse_wave_reflection_index_f4195_0_0,float,"{""zscore"": {""std"": [1035.5975347], ""mean"": [67.728354]}}" +5,6,measurements_waist_circumference_f48_0_0,float,"{""zscore"": {""std"": [181.86851834], ""mean"": [90.31081898]}}" +6,7,measurements_hip_circumference_f49_0_0,float,"{""zscore"": {""std"": [85.55373922], ""mean"": [103.40158394]}}" +7,8,measurements_standing_height_f50_0_0,float,"{""zscore"": {""std"": [86.10089505], ""mean"": [168.44354526]}}" +8,9,measurements_trunk_fat_percentage_f23127_0_0,float,"{""zscore"": {""std"": [64.13611621], ""mean"": [31.17342071]}}" +9,10,measurements_body_fat_percentage_f23099_0_0,float,"{""zscore"": {""std"": [73.06088003], ""mean"": [31.45178684]}}" +10,11,measurements_basal_metabolic_rate_f23105_0_0,float,"{""zscore"": {""std"": [1861301.06090653], ""mean"": [6616.13365882]}}" +11,12,measurements_forced_vital_capacity_fvc_best_measure_f20151_0_0,float,"{""zscore"": {""std"": [0.96988716], ""mean"": [3.77788852]}}" +12,13,measurements_forced_expiratory_volume_in_1second_fev1_best_measure_f20150_0_0,float,"{""zscore"": {""std"": [0.60848128], ""mean"": [2.85074123]}}" +13,14,measurements_fev1_fvc_ratio_zscore_f20258_0_0,float,"{""zscore"": {""std"": [0.78797162], ""mean"": [0.41824227]}}" +14,15,measurements_peak_expiratory_flow_pef_f3064_0_2,float,"{""zscore"": {""std"": [23559.06009175], ""mean"": [325.81174618]}}" +15,16,measurements_peak_expiratory_flow_pef_f3064_0_1,float,"{""zscore"": {""std"": [19178.83974267], ""mean"": [372.7427792]}}" +16,17,measurements_peak_expiratory_flow_pef_f3064_0_0,float,"{""zscore"": {""std"": [18383.4861072], ""mean"": [388.39422424]}}" +17,18,measurements_systolic_blood_pressure_automated_reading_f4080,float,"{""zscore"": {""std"": [348.89967781], ""mean"": [137.82152088]}}" +18,19,measurements_diastolic_blood_pressure_automated_reading_f4079,float,"{""zscore"": {""std"": [103.14531112], ""mean"": [82.21547217]}}" +19,20,measurements_pulse_rate_automated_reading_f102,float,"{""zscore"": {""std"": [126.85754528], ""mean"": [69.3559183]}}" +20,21,labs_basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +21,22,labs_basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +22,23,labs_eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +23,24,labs_eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +24,25,labs_haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +25,26,labs_haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +26,27,labs_high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +27,28,labs_high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +28,29,labs_immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +29,30,labs_lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +30,31,labs_lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +31,32,labs_mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +32,33,labs_mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +33,34,labs_mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +34,35,labs_mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +35,36,labs_mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +36,37,labs_mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +37,38,labs_monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +38,39,labs_monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +39,40,labs_neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +40,41,labs_neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +41,42,labs_nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +42,43,labs_nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +43,44,labs_platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +44,45,labs_platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +45,46,labs_platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +46,47,labs_red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +47,48,labs_red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +48,49,labs_reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +49,50,labs_reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +50,51,labs_white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +51,52,labs_alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +52,53,labs_albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +53,54,labs_alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +54,55,labs_apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +55,56,labs_apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +56,57,labs_aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +57,58,labs_creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +58,59,labs_calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +59,60,labs_cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +60,61,labs_creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +61,62,labs_cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +62,63,labs_direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +63,64,labs_gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +64,65,labs_glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +65,66,labs_glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +66,67,labs_hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +67,68,labs_igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +68,69,labs_ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +69,70,labs_lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +70,71,labs_oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +71,72,labs_phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +72,73,labs_rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +73,74,labs_shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +74,75,labs_testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +75,76,labs_total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +76,77,labs_total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +77,78,labs_triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +78,79,labs_urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +79,80,labs_urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +80,81,labs_vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" diff --git a/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..43f35cf Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..4c46479 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_extended_tr.h5ad.var.csv @@ -0,0 +1,100 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,drugs_for_acid_related_disorders,float, +2,3,vitamins,float, +3,4,diuretics,float, +4,5,beta_blocking_agents,float, +5,6,agents_acting_on_the_renin-angiotensin_system,float, +6,7,lipid_modifying_agents,float, +7,8,thyroid_therapy,float, +8,9,antiinflammatory_and_antirheumatic_products,float, +9,10,analgesics,float, +10,11,psychoanaleptics,float, +11,12,nasal_preparations,float, +12,13,drugs_for_obstructive_airway_diseases,float, +13,14,statins,float, +14,15,fh_alzheimer's_disease/dementia,float, +15,16,fh_bowel_cancer,float, +16,17,fh_breast_cancer,float, +17,18,fh_chronic_bronchitis/emphysema,float, +18,19,fh_diabetes,float, +19,20,fh_heart_disease,float, +20,21,fh_high_blood_pressure,float, +21,22,fh_lung_cancer,float, +22,23,fh_parkinson's_disease,float, +23,24,fh_severe_depression,float, +24,25,fh_stroke,float, +25,26,basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +26,27,basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +27,28,eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +28,29,eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +29,30,haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +30,31,haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +31,32,high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +32,33,high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +33,34,immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +34,35,lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +35,36,lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +36,37,mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +37,38,mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +38,39,mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +39,40,mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +40,41,mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +41,42,mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +42,43,monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +43,44,monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +44,45,neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +45,46,neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +46,47,nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +47,48,nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +48,49,platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +49,50,platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +50,51,platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +51,52,red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +52,53,red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +53,54,reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +54,55,reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +55,56,white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +56,57,alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +57,58,albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +58,59,alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +59,60,apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +60,61,apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +61,62,aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +62,63,creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +63,64,calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +64,65,cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +65,66,creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +66,67,cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +67,68,direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +68,69,gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +69,70,glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +70,71,glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +71,72,hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +72,73,igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +73,74,ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +74,75,lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +75,76,oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +76,77,phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +77,78,rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +78,79,shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +79,80,testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +80,81,total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +81,82,total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +82,83,triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +83,84,urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +84,85,urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +85,86,vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" +86,87,overall_health_rating_f2178_0_0_Poor,float, +87,88,overall_health_rating_f2178_0_0_Fair,float, +88,89,overall_health_rating_f2178_0_0_Good,float, +89,90,overall_health_rating_f2178_0_0_Excellent,float, +90,91,smoking_status_f20116_0_0_Current,float, +91,92,smoking_status_f20116_0_0_Previous,float, +92,93,smoking_status_f20116_0_0_Never,float, +93,94,alcohol_intake_frequency_f1558_0_0_Daily or almost daily,float, +94,95,alcohol_intake_frequency_f1558_0_0_Three or four times a week,float, +95,96,alcohol_intake_frequency_f1558_0_0_Once or twice a week,float, +96,97,alcohol_intake_frequency_f1558_0_0_One to three times a month,float, +97,98,alcohol_intake_frequency_f1558_0_0_Special occasions only,float, +98,99,alcohol_intake_frequency_f1558_0_0_Never,float, diff --git a/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.X.npy new file mode 100644 index 0000000..9293d28 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.var.csv new file mode 100644 index 0000000..4c46479 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_test.h5ad.var.csv @@ -0,0 +1,100 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,drugs_for_acid_related_disorders,float, +2,3,vitamins,float, +3,4,diuretics,float, +4,5,beta_blocking_agents,float, +5,6,agents_acting_on_the_renin-angiotensin_system,float, +6,7,lipid_modifying_agents,float, +7,8,thyroid_therapy,float, +8,9,antiinflammatory_and_antirheumatic_products,float, +9,10,analgesics,float, +10,11,psychoanaleptics,float, +11,12,nasal_preparations,float, +12,13,drugs_for_obstructive_airway_diseases,float, +13,14,statins,float, +14,15,fh_alzheimer's_disease/dementia,float, +15,16,fh_bowel_cancer,float, +16,17,fh_breast_cancer,float, +17,18,fh_chronic_bronchitis/emphysema,float, +18,19,fh_diabetes,float, +19,20,fh_heart_disease,float, +20,21,fh_high_blood_pressure,float, +21,22,fh_lung_cancer,float, +22,23,fh_parkinson's_disease,float, +23,24,fh_severe_depression,float, +24,25,fh_stroke,float, +25,26,basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +26,27,basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +27,28,eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +28,29,eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +29,30,haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +30,31,haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +31,32,high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +32,33,high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +33,34,immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +34,35,lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +35,36,lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +36,37,mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +37,38,mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +38,39,mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +39,40,mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +40,41,mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +41,42,mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +42,43,monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +43,44,monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +44,45,neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +45,46,neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +46,47,nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +47,48,nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +48,49,platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +49,50,platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +50,51,platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +51,52,red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +52,53,red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +53,54,reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +54,55,reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +55,56,white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +56,57,alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +57,58,albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +58,59,alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +59,60,apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +60,61,apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +61,62,aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +62,63,creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +63,64,calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +64,65,cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +65,66,creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +66,67,cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +67,68,direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +68,69,gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +69,70,glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +70,71,glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +71,72,hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +72,73,igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +73,74,ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +74,75,lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +75,76,oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +76,77,phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +77,78,rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +78,79,shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +79,80,testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +80,81,total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +81,82,total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +82,83,triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +83,84,urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +84,85,urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +85,86,vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" +86,87,overall_health_rating_f2178_0_0_Poor,float, +87,88,overall_health_rating_f2178_0_0_Fair,float, +88,89,overall_health_rating_f2178_0_0_Good,float, +89,90,overall_health_rating_f2178_0_0_Excellent,float, +90,91,smoking_status_f20116_0_0_Current,float, +91,92,smoking_status_f20116_0_0_Previous,float, +92,93,smoking_status_f20116_0_0_Never,float, +93,94,alcohol_intake_frequency_f1558_0_0_Daily or almost daily,float, +94,95,alcohol_intake_frequency_f1558_0_0_Three or four times a week,float, +95,96,alcohol_intake_frequency_f1558_0_0_Once or twice a week,float, +96,97,alcohol_intake_frequency_f1558_0_0_One to three times a month,float, +97,98,alcohol_intake_frequency_f1558_0_0_Special occasions only,float, +98,99,alcohol_intake_frequency_f1558_0_0_Never,float, diff --git a/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.X.npy new file mode 100644 index 0000000..cd295c0 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.var.csv new file mode 100644 index 0000000..4c46479 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_train.h5ad.var.csv @@ -0,0 +1,100 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,drugs_for_acid_related_disorders,float, +2,3,vitamins,float, +3,4,diuretics,float, +4,5,beta_blocking_agents,float, +5,6,agents_acting_on_the_renin-angiotensin_system,float, +6,7,lipid_modifying_agents,float, +7,8,thyroid_therapy,float, +8,9,antiinflammatory_and_antirheumatic_products,float, +9,10,analgesics,float, +10,11,psychoanaleptics,float, +11,12,nasal_preparations,float, +12,13,drugs_for_obstructive_airway_diseases,float, +13,14,statins,float, +14,15,fh_alzheimer's_disease/dementia,float, +15,16,fh_bowel_cancer,float, +16,17,fh_breast_cancer,float, +17,18,fh_chronic_bronchitis/emphysema,float, +18,19,fh_diabetes,float, +19,20,fh_heart_disease,float, +20,21,fh_high_blood_pressure,float, +21,22,fh_lung_cancer,float, +22,23,fh_parkinson's_disease,float, +23,24,fh_severe_depression,float, +24,25,fh_stroke,float, +25,26,basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +26,27,basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +27,28,eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +28,29,eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +29,30,haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +30,31,haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +31,32,high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +32,33,high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +33,34,immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +34,35,lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +35,36,lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +36,37,mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +37,38,mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +38,39,mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +39,40,mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +40,41,mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +41,42,mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +42,43,monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +43,44,monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +44,45,neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +45,46,neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +46,47,nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +47,48,nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +48,49,platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +49,50,platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +50,51,platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +51,52,red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +52,53,red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +53,54,reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +54,55,reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +55,56,white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +56,57,alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +57,58,albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +58,59,alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +59,60,apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +60,61,apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +61,62,aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +62,63,creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +63,64,calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +64,65,cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +65,66,creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +66,67,cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +67,68,direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +68,69,gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +69,70,glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +70,71,glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +71,72,hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +72,73,igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +73,74,ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +74,75,lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +75,76,oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +76,77,phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +77,78,rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +78,79,shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +79,80,testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +80,81,total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +81,82,total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +82,83,triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +83,84,urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +84,85,urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +85,86,vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" +86,87,overall_health_rating_f2178_0_0_Poor,float, +87,88,overall_health_rating_f2178_0_0_Fair,float, +88,89,overall_health_rating_f2178_0_0_Good,float, +89,90,overall_health_rating_f2178_0_0_Excellent,float, +90,91,smoking_status_f20116_0_0_Current,float, +91,92,smoking_status_f20116_0_0_Previous,float, +92,93,smoking_status_f20116_0_0_Never,float, +93,94,alcohol_intake_frequency_f1558_0_0_Daily or almost daily,float, +94,95,alcohol_intake_frequency_f1558_0_0_Three or four times a week,float, +95,96,alcohol_intake_frequency_f1558_0_0_Once or twice a week,float, +96,97,alcohol_intake_frequency_f1558_0_0_One to three times a month,float, +97,98,alcohol_intake_frequency_f1558_0_0_Special occasions only,float, +98,99,alcohol_intake_frequency_f1558_0_0_Never,float, diff --git a/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.X.npy b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.X.npy new file mode 100644 index 0000000..0bea0b8 Binary files /dev/null and b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.obs.csv b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.var.csv b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.var.csv new file mode 100644 index 0000000..4c46479 --- /dev/null +++ b/res/datasets/fake/covariate/covariates_new_extended_valid.h5ad.var.csv @@ -0,0 +1,100 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,townsend_deprivation_index_at_recruitment_f189_0_0,float,"{""zscore"": {""std"": [9.57936802], ""mean"": [-1.29373508]}}" +1,2,drugs_for_acid_related_disorders,float, +2,3,vitamins,float, +3,4,diuretics,float, +4,5,beta_blocking_agents,float, +5,6,agents_acting_on_the_renin-angiotensin_system,float, +6,7,lipid_modifying_agents,float, +7,8,thyroid_therapy,float, +8,9,antiinflammatory_and_antirheumatic_products,float, +9,10,analgesics,float, +10,11,psychoanaleptics,float, +11,12,nasal_preparations,float, +12,13,drugs_for_obstructive_airway_diseases,float, +13,14,statins,float, +14,15,fh_alzheimer's_disease/dementia,float, +15,16,fh_bowel_cancer,float, +16,17,fh_breast_cancer,float, +17,18,fh_chronic_bronchitis/emphysema,float, +18,19,fh_diabetes,float, +19,20,fh_heart_disease,float, +20,21,fh_high_blood_pressure,float, +21,22,fh_lung_cancer,float, +22,23,fh_parkinson's_disease,float, +23,24,fh_severe_depression,float, +24,25,fh_stroke,float, +25,26,basophill_count_f30160_0_0,float,"{""zscore"": {""std"": [0.00266338], ""mean"": [0.03405815]}}" +26,27,basophill_percentage_f30220_0_0,float,"{""zscore"": {""std"": [0.37265851], ""mean"": [0.56968157]}}" +27,28,eosinophill_count_f30150_0_0,float,"{""zscore"": {""std"": [0.0192038], ""mean"": [0.17487986]}}" +28,29,eosinophill_percentage_f30210_0_0,float,"{""zscore"": {""std"": [3.51812781], ""mean"": [2.5724458]}}" +29,30,haematocrit_percentage_f30030_0_0,float,"{""zscore"": {""std"": [12.64533238], ""mean"": [41.08495569]}}" +30,31,haemoglobin_concentration_f30020_0_0,float,"{""zscore"": {""std"": [1.5550051], ""mean"": [14.17475889]}}" +31,32,high_light_scatter_reticulocyte_count_f30300_0_0,float,"{""zscore"": {""std"": [0.00010482], ""mean"": [0.01812101]}}" +32,33,high_light_scatter_reticulocyte_percentage_f30290_0_0,float,"{""zscore"": {""std"": [0.11003644], ""mean"": [0.40206229]}}" +33,34,immature_reticulocyte_fraction_f30280_0_0,float,"{""zscore"": {""std"": [0.00372685], ""mean"": [0.29079299]}}" +34,35,lymphocyte_count_f30120_0_0,float,"{""zscore"": {""std"": [1.37439394], ""mean"": [1.9659849]}}" +35,36,lymphocyte_percentage_f30180_0_0,float,"{""zscore"": {""std"": [56.23817223], ""mean"": [28.91320743]}}" +36,37,mean_corpuscular_haemoglobin_f30050_0_0,float,"{""zscore"": {""std"": [3.69242513], ""mean"": [31.45094984]}}" +37,38,mean_corpuscular_haemoglobin_concentration_f30060_0_0,float,"{""zscore"": {""std"": [1.15784293], ""mean"": [34.51283944]}}" +38,39,mean_corpuscular_volume_f30040_0_0,float,"{""zscore"": {""std"": [21.2169815], ""mean"": [91.12005201]}}" +39,40,mean_platelet_thrombocyte_volume_f30100_0_0,float,"{""zscore"": {""std"": [1.17719054], ""mean"": [9.33308081]}}" +40,41,mean_reticulocyte_volume_f30260_0_0,float,"{""zscore"": {""std"": [61.3739669], ""mean"": [105.91970733]}}" +41,42,mean_sphered_cell_volume_f30270_0_0,float,"{""zscore"": {""std"": [28.24987108], ""mean"": [82.86684851]}}" +42,43,monocyte_count_f30130_0_0,float,"{""zscore"": {""std"": [0.07448446], ""mean"": [0.47595457]}}" +43,44,monocyte_percentage_f30190_0_0,float,"{""zscore"": {""std"": [7.27999142], ""mean"": [7.06432279]}}" +44,45,neutrophill_count_f30140_0_0,float,"{""zscore"": {""std"": [2.02154927], ""mean"": [4.22788442]}}" +45,46,neutrophill_percentage_f30200_0_0,float,"{""zscore"": {""std"": [72.68854331], ""mean"": [60.88044484]}}" +46,47,nucleated_red_blood_cell_count_f30170_0_0,float,"{""zscore"": {""std"": [0.00110169], ""mean"": [0.00221747]}}" +47,48,nucleated_red_blood_cell_percentage_f30230_0_0,float,"{""zscore"": {""std"": [0.19995331], ""mean"": [0.03519064]}}" +48,49,platelet_count_f30080_0_0,float,"{""zscore"": {""std"": [3605.79971447], ""mean"": [252.98900892]}}" +49,50,platelet_crit_f30090_0_0,float,"{""zscore"": {""std"": [0.00240224], ""mean"": [0.23272601]}}" +50,51,platelet_distribution_width_f30110_0_0,float,"{""zscore"": {""std"": [0.27292359], ""mean"": [16.49035525]}}" +51,52,red_blood_cell_erythrocyte_count_f30010_0_0,float,"{""zscore"": {""std"": [0.17381905], ""mean"": [4.51689846]}}" +52,53,red_blood_cell_erythrocyte_distribution_width_f30070_0_0,float,"{""zscore"": {""std"": [0.97560543], ""mean"": [13.49324742]}}" +53,54,reticulocyte_count_f30250_0_0,float,"{""zscore"": {""std"": [0.00150476], ""mean"": [0.06100768]}}" +54,55,reticulocyte_percentage_f30240_0_0,float,"{""zscore"": {""std"": [0.7929912], ""mean"": [1.35029991]}}" +55,56,white_blood_cell_leukocyte_count_f30000_0_0,float,"{""zscore"": {""std"": [4.50588919], ""mean"": [6.88504953]}}" +56,57,alanine_aminotransferase_f30620_0_0,float,"{""zscore"": {""std"": [201.07699471], ""mean"": [23.54631834]}}" +57,58,albumin_f30600_0_0,float,"{""zscore"": {""std"": [6.91249045], ""mean"": [45.21007928]}}" +58,59,alkaline_phosphatase_f30610_0_0,float,"{""zscore"": {""std"": [700.12832287], ""mean"": [83.66915644]}}" +59,60,apolipoprotein_a_f30630_0_0,float,"{""zscore"": {""std"": [0.07325454], ""mean"": [1.53765083]}}" +60,61,apolipoprotein_b_f30640_0_0,float,"{""zscore"": {""std"": [0.05676348], ""mean"": [1.03202679]}}" +61,62,aspartate_aminotransferase_f30650_0_0,float,"{""zscore"": {""std"": [113.66905456], ""mean"": [26.23016083]}}" +62,63,creactive_protein_f30710_0_0,float,"{""zscore"": {""std"": [19.01959308], ""mean"": [2.59877425]}}" +63,64,calcium_f30680_0_0,float,"{""zscore"": {""std"": [0.00887144], ""mean"": [2.37988006]}}" +64,65,cholesterol_f30690_0_0,float,"{""zscore"": {""std"": [1.31088695], ""mean"": [5.69238121]}}" +65,66,creatinine_f30700_0_0,float,"{""zscore"": {""std"": [344.09198363], ""mean"": [72.30598403]}}" +66,67,cystatin_c_f30720_0_0,float,"{""zscore"": {""std"": [0.03108428], ""mean"": [0.90752038]}}" +67,68,direct_bilirubin_f30660_0_0,float,"{""zscore"": {""std"": [0.72353276], ""mean"": [1.83363042]}}" +68,69,gamma_glutamyltransferase_f30730_0_0,float,"{""zscore"": {""std"": [1771.17106396], ""mean"": [37.38983782]}}" +69,70,glucose_f30740_0_0,float,"{""zscore"": {""std"": [1.54619444], ""mean"": [5.12498858]}}" +70,71,glycated_haemoglobin_hba1c_f30750_0_0,float,"{""zscore"": {""std"": [45.91152054], ""mean"": [36.13116606]}}" +71,72,hdl_cholesterol_f30760_0_0,float,"{""zscore"": {""std"": [0.14616182], ""mean"": [1.44801291]}}" +72,73,igf1_f30770_0_0,float,"{""zscore"": {""std"": [32.48725749], ""mean"": [21.39891164]}}" +73,74,ldl_direct_f30780_0_0,float,"{""zscore"": {""std"": [0.75685629], ""mean"": [3.55636517]}}" +74,75,lipoprotein_a_f30790_0_0,float,"{""zscore"": {""std"": [2421.85806989], ""mean"": [44.64786511]}}" +75,76,oestradiol_f30800_0_0,float,"{""zscore"": {""std"": [185897.55306477], ""mean"": [461.17321908]}}" +76,77,phosphate_f30810_0_0,float,"{""zscore"": {""std"": [0.02595631], ""mean"": [1.15932899]}}" +77,78,rheumatoid_factor_f30820_0_0,float,"{""zscore"": {""std"": [394.33666119], ""mean"": [24.55629296]}}" +78,79,shbg_f30830_0_0,float,"{""zscore"": {""std"": [771.79451163], ""mean"": [51.62937739]}}" +79,80,testosterone_f30850_0_0,float,"{""zscore"": {""std"": [36.65259543], ""mean"": [6.55778001]}}" +80,81,total_bilirubin_f30840_0_0,float,"{""zscore"": {""std"": [19.59556077], ""mean"": [9.12506668]}}" +81,82,total_protein_f30860_0_0,float,"{""zscore"": {""std"": [16.93930524], ""mean"": [72.51024894]}}" +82,83,triglycerides_f30870_0_0,float,"{""zscore"": {""std"": [1.05613202], ""mean"": [1.74757812]}}" +83,84,urate_f30880_0_0,float,"{""zscore"": {""std"": [6469.30174694], ""mean"": [309.20545484]}}" +84,85,urea_f30670_0_0,float,"{""zscore"": {""std"": [1.96369959], ""mean"": [5.40344456]}}" +85,86,vitamin_d_f30890_0_0,float,"{""zscore"": {""std"": [445.66800989], ""mean"": [48.61285707]}}" +86,87,overall_health_rating_f2178_0_0_Poor,float, +87,88,overall_health_rating_f2178_0_0_Fair,float, +88,89,overall_health_rating_f2178_0_0_Good,float, +89,90,overall_health_rating_f2178_0_0_Excellent,float, +90,91,smoking_status_f20116_0_0_Current,float, +91,92,smoking_status_f20116_0_0_Previous,float, +92,93,smoking_status_f20116_0_0_Never,float, +93,94,alcohol_intake_frequency_f1558_0_0_Daily or almost daily,float, +94,95,alcohol_intake_frequency_f1558_0_0_Three or four times a week,float, +95,96,alcohol_intake_frequency_f1558_0_0_Once or twice a week,float, +96,97,alcohol_intake_frequency_f1558_0_0_One to three times a month,float, +97,98,alcohol_intake_frequency_f1558_0_0_Special occasions only,float, +98,99,alcohol_intake_frequency_f1558_0_0_Never,float, diff --git a/res/datasets/fake/geno/default_genetic_extended_tr.X.npy b/res/datasets/fake/geno/default_genetic_extended_tr.X.npy new file mode 100644 index 0000000..bfd020c Binary files /dev/null and b/res/datasets/fake/geno/default_genetic_extended_tr.X.npy differ diff --git a/res/datasets/fake/geno/default_genetic_extended_tr.obs.csv b/res/datasets/fake/geno/default_genetic_extended_tr.obs.csv new file mode 100644 index 0000000..520c362 --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_extended_tr.obs.csv @@ -0,0 +1,2049 @@ +index +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/geno/default_genetic_extended_tr.var.csv b/res/datasets/fake/geno/default_genetic_extended_tr.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_extended_tr.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/geno/default_genetic_test.X.npy b/res/datasets/fake/geno/default_genetic_test.X.npy new file mode 100644 index 0000000..ef4a156 Binary files /dev/null and b/res/datasets/fake/geno/default_genetic_test.X.npy differ diff --git a/res/datasets/fake/geno/default_genetic_test.obs.csv b/res/datasets/fake/geno/default_genetic_test.obs.csv new file mode 100644 index 0000000..f38c51f --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_test.obs.csv @@ -0,0 +1,1025 @@ +index +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/geno/default_genetic_test.var.csv b/res/datasets/fake/geno/default_genetic_test.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_test.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/geno/default_genetic_valid.X.npy b/res/datasets/fake/geno/default_genetic_valid.X.npy new file mode 100644 index 0000000..443b187 Binary files /dev/null and b/res/datasets/fake/geno/default_genetic_valid.X.npy differ diff --git a/res/datasets/fake/geno/default_genetic_valid.obs.csv b/res/datasets/fake/geno/default_genetic_valid.obs.csv new file mode 100644 index 0000000..9aa7386 --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_valid.obs.csv @@ -0,0 +1,1025 @@ +index +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/geno/default_genetic_valid.var.csv b/res/datasets/fake/geno/default_genetic_valid.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/default_genetic_valid.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/geno/genetic_extended_tr.X.npy b/res/datasets/fake/geno/genetic_extended_tr.X.npy new file mode 100644 index 0000000..6fb8f7f Binary files /dev/null and b/res/datasets/fake/geno/genetic_extended_tr.X.npy differ diff --git a/res/datasets/fake/geno/genetic_extended_tr.obs.csv b/res/datasets/fake/geno/genetic_extended_tr.obs.csv new file mode 100644 index 0000000..520c362 --- /dev/null +++ b/res/datasets/fake/geno/genetic_extended_tr.obs.csv @@ -0,0 +1,2049 @@ +index +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/geno/genetic_extended_tr.var.csv b/res/datasets/fake/geno/genetic_extended_tr.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/genetic_extended_tr.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/geno/genetic_test.X.npy b/res/datasets/fake/geno/genetic_test.X.npy new file mode 100644 index 0000000..d0920c6 Binary files /dev/null and b/res/datasets/fake/geno/genetic_test.X.npy differ diff --git a/res/datasets/fake/geno/genetic_test.obs.csv b/res/datasets/fake/geno/genetic_test.obs.csv new file mode 100644 index 0000000..f38c51f --- /dev/null +++ b/res/datasets/fake/geno/genetic_test.obs.csv @@ -0,0 +1,1025 @@ +index +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/geno/genetic_test.var.csv b/res/datasets/fake/geno/genetic_test.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/genetic_test.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/geno/genetic_valid.X.npy b/res/datasets/fake/geno/genetic_valid.X.npy new file mode 100644 index 0000000..509eeed Binary files /dev/null and b/res/datasets/fake/geno/genetic_valid.X.npy differ diff --git a/res/datasets/fake/geno/genetic_valid.obs.csv b/res/datasets/fake/geno/genetic_valid.obs.csv new file mode 100644 index 0000000..9aa7386 --- /dev/null +++ b/res/datasets/fake/geno/genetic_valid.obs.csv @@ -0,0 +1,1025 @@ +index +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/geno/genetic_valid.var.csv b/res/datasets/fake/geno/genetic_valid.var.csv new file mode 100644 index 0000000..2a2db6f --- /dev/null +++ b/res/datasets/fake/geno/genetic_valid.var.csv @@ -0,0 +1,11 @@ +Unnamed: 0,ID,GeneticAlt,ANN_AA_length,ANN_AA_pos,ANN_Allele,GeneticEffect,GeneticImpact,ANN_CDS_length,ANN_CDS_pos,ANN_Distance,ANN_Feature_ID,ANN_Feature_Type,ANN_Gene_ID,GeneticGName,ANN_HGVS_c,ANN_HGVS_p,ANN_Rank,GeneticTranscript,ANN_cDNA_length,ANN_cDNA_pos,GeneticChromosome,ID.1,GeneticPosition,GeneticRef,SCORE,_raw_id1,_raw_id2,Field,ValueType,WasProcessed,GeneticFrequency +0,rs183240359,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1746,ENST00000587126,transcript,ENSG00000237491,RP11-206L10.9,-1746C>T,,-1,lincRNA,-1,-1,1,rs183240359,738410,C,-4.1849e-05,rs183240359_T,1:738410_C_T_T,ENSG00000237491,Genetic,False,0.0063258007934729 +1,rs4970340,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,387,ENST00000446136,transcript,ENSG00000230368,FAM41C,-387T>G,,-1,lincRNA,-1,-1,1,rs4970340,812670,A,-7.105e-05,rs4970340_C,1:812670_A_C_C,ENSG00000230368,Genetic,False,0.023483015994487 +2,rs188400142,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,983,ENST00000417705,transcript,ENSG00000223764,RP11-54O7.3,-983C>A,,-1,lincRNA,-1,-1,1,rs188400142,856055,G,-0.00014018,rs188400142_T,1:856055_G_T_T,ENSG00000223764,Genetic,False,0.0003328660311374 +3,rs13303307,G,-1,-1,G,sequence_feature,LOW,-1,-1,-1,ENST00000379370,miscellaneous-site:Alternative_splice_site_to_produce_'z'_isoforms,ENSG00000188157,AGRN,5652-823A>G,,33,protein_coding,-1,-1,1,rs13303307,988310,A,-5.0612e-05,rs13303307_G,1:988310_A_G_G,ENSG00000188157,Genetic,False,0.1142669290090663 +4,rs140894808,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,1748,ENST00000427998,transcript,ENSG00000217801,RP11-465B22.3,-1748G>T,,-1,processed_transcript,-1,-1,1,rs140894808,993366,G,-4.5318e-05,rs140894808_T,1:993366_G_T_T,ENSG00000217801,Genetic,False,0.0343662072192195 +5,rs12027296,C,-1,-1,C,upstream_gene_variant,MODIFIER,-1,-1,4708,ENST00000465822,transcript,ENSG00000131591,C1orf159,-4708A>G,,-1,retained_intron,-1,-1,1,rs12027296,1032165,T,4.63e-05,rs12027296_C,1:1032165_T_C_C,ENSG00000131591,Genetic,False,0.0057320289700556 +6,rs12047655,T,-1,-1,T,upstream_gene_variant,MODIFIER,-1,-1,4364,ENST00000379319,transcript,ENSG00000131591,C1orf159,-18948G>A,,-1,protein_coding,-1,-1,1,rs12047655,1045871,C,5.3147e-05,rs12047655_T,1:1045871_C_T_T,ENSG00000131591,Genetic,False,0.0043576793928487 +7,rs3813205,T,-1,-1,T,3_prime_UTR_variant,MODIFIER,-1,-1,299,ENST00000379288,transcript,ENSG00000162571,TTLL10,*299C>T,,9,protein_coding,-1,-1,1,rs3813205,1120821,C,-5.8213e-05,rs3813205_T,1:1120821_C_T_T,ENSG00000162571,Genetic,False,0.0220802756118764 +8,rs112904239,T,-1,-1,T,downstream_gene_variant,MODIFIER,-1,-1,313,ENST00000379288,transcript,ENSG00000162571,TTLL10,*1035C>T,,-1,protein_coding,-1,-1,1,rs112904239,1121557,C,0.00011517,rs112904239_T,1:1121557_C_T_T,ENSG00000162571,Genetic,False,0.0756721796344296 +9,rs11577350,A,-1,-1,A,downstream_gene_variant,MODIFIER,-1,-1,3294,ENST00000379288,transcript,ENSG00000162571,TTLL10,*4016G>A,,-1,protein_coding,-1,-1,1,rs11577350,1124538,G,5.5078e-05,rs11577350_A,1:1124538_G_A_A,ENSG00000162571,Genetic,False,0.0608255694071457 diff --git a/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.X.npy b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..6f04847 Binary files /dev/null and b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.obs.csv b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.var.csv b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..cd357b2 --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_extended_tr.h5ad.var.csv @@ -0,0 +1,169 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,NMR_3hydroxybutyrate,float, +1,2,NMR_acetate,float, +2,3,NMR_acetoacetate,float, +3,4,NMR_acetone,float, +4,5,NMR_alanine,float, +5,6,NMR_albumin,float, +6,7,NMR_apolipoprotein_a1,float, +7,8,NMR_apolipoprotein_b,float, +8,9,NMR_average_diameter_for_hdl_particles,float, +9,10,NMR_average_diameter_for_ldl_particles,float, +10,11,NMR_average_diameter_for_vldl_particles,float, +11,12,NMR_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +12,13,NMR_cholesterol_in_idl,float, +13,14,NMR_cholesterol_in_large_hdl,float, +14,15,NMR_cholesterol_in_large_ldl,float, +15,16,NMR_cholesterol_in_large_vldl,float, +16,17,NMR_cholesterol_in_medium_hdl,float, +17,18,NMR_cholesterol_in_medium_ldl,float, +18,19,NMR_cholesterol_in_medium_vldl,float, +19,20,NMR_cholesterol_in_small_hdl,float, +20,21,NMR_cholesterol_in_small_ldl,float, +21,22,NMR_cholesterol_in_small_vldl,float, +22,23,NMR_cholesterol_in_very_large_hdl,float, +23,24,NMR_cholesterol_in_very_large_vldl,float, +24,25,NMR_cholesterol_in_very_small_vldl,float, +25,26,NMR_cholesteryl_esters_in_chylomicrons_and_extremely_large_vldl,float, +26,27,NMR_cholesteryl_esters_in_hdl,float, +27,28,NMR_cholesteryl_esters_in_idl,float, +28,29,NMR_cholesteryl_esters_in_ldl,float, +29,30,NMR_cholesteryl_esters_in_large_hdl,float, +30,31,NMR_cholesteryl_esters_in_large_ldl,float, +31,32,NMR_cholesteryl_esters_in_large_vldl,float, +32,33,NMR_cholesteryl_esters_in_medium_hdl,float, +33,34,NMR_cholesteryl_esters_in_medium_ldl,float, +34,35,NMR_cholesteryl_esters_in_medium_vldl,float, +35,36,NMR_cholesteryl_esters_in_small_hdl,float, +36,37,NMR_cholesteryl_esters_in_small_ldl,float, +37,38,NMR_cholesteryl_esters_in_small_vldl,float, +38,39,NMR_cholesteryl_esters_in_vldl,float, +39,40,NMR_cholesteryl_esters_in_very_large_hdl,float, +40,41,NMR_cholesteryl_esters_in_very_large_vldl,float, +41,42,NMR_cholesteryl_esters_in_very_small_vldl,float, +42,43,NMR_citrate,float, +43,44,NMR_clinical_ldl_cholesterol,float, +44,45,NMR_concentration_of_chylomicrons_and_extremely_large_vldl_particles,float, +45,46,NMR_concentration_of_hdl_particles,float, +46,47,NMR_concentration_of_idl_particles,float, +47,48,NMR_concentration_of_ldl_particles,float, +48,49,NMR_concentration_of_large_hdl_particles,float, +49,50,NMR_concentration_of_large_ldl_particles,float, +50,51,NMR_concentration_of_large_vldl_particles,float, +51,52,NMR_concentration_of_medium_hdl_particles,float, +52,53,NMR_concentration_of_medium_ldl_particles,float, +53,54,NMR_concentration_of_medium_vldl_particles,float, +54,55,NMR_concentration_of_small_hdl_particles,float, +55,56,NMR_concentration_of_small_ldl_particles,float, +56,57,NMR_concentration_of_small_vldl_particles,float, +57,58,NMR_concentration_of_vldl_particles,float, +58,59,NMR_concentration_of_very_large_hdl_particles,float, +59,60,NMR_concentration_of_very_large_vldl_particles,float, +60,61,NMR_concentration_of_very_small_vldl_particles,float, +61,62,NMR_creatinine,float, +62,63,NMR_degree_of_unsaturation,float, +63,64,NMR_docosahexaenoic_acid,float, +64,65,NMR_free_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +65,66,NMR_free_cholesterol_in_hdl,float, +66,67,NMR_free_cholesterol_in_idl,float, +67,68,NMR_free_cholesterol_in_ldl,float, +68,69,NMR_free_cholesterol_in_large_hdl,float, +69,70,NMR_free_cholesterol_in_large_ldl,float, +70,71,NMR_free_cholesterol_in_large_vldl,float, +71,72,NMR_free_cholesterol_in_medium_hdl,float, +72,73,NMR_free_cholesterol_in_medium_ldl,float, +73,74,NMR_free_cholesterol_in_medium_vldl,float, +74,75,NMR_free_cholesterol_in_small_hdl,float, +75,76,NMR_free_cholesterol_in_small_ldl,float, +76,77,NMR_free_cholesterol_in_small_vldl,float, +77,78,NMR_free_cholesterol_in_vldl,float, +78,79,NMR_free_cholesterol_in_very_large_hdl,float, +79,80,NMR_free_cholesterol_in_very_large_vldl,float, +80,81,NMR_free_cholesterol_in_very_small_vldl,float, +81,82,NMR_glucose,float, +82,83,NMR_glutamine,float, +83,84,NMR_glycine,float, +84,85,NMR_glycoprotein_acetyls,float, +85,86,NMR_hdl_cholesterol,float, +86,87,NMR_histidine,float, +87,88,NMR_isoleucine,float, +88,89,NMR_ldl_cholesterol,float, +89,90,NMR_lactate,float, +90,91,NMR_leucine,float, +91,92,NMR_linoleic_acid,float, +92,93,NMR_monounsaturated_fatty_acids,float, +93,94,NMR_omega3_fatty_acids,float, +94,95,NMR_omega6_fatty_acids,float, +95,96,NMR_phenylalanine,float, +96,97,NMR_phosphatidylcholines,float, +97,98,NMR_phosphoglycerides,float, +98,99,NMR_phospholipids_in_chylomicrons_and_extremely_large_vldl,float, +99,100,NMR_phospholipids_in_hdl,float, +100,101,NMR_phospholipids_in_idl,float, +101,102,NMR_phospholipids_in_ldl,float, +102,103,NMR_phospholipids_in_large_hdl,float, +103,104,NMR_phospholipids_in_large_ldl,float, +104,105,NMR_phospholipids_in_large_vldl,float, +105,106,NMR_phospholipids_in_medium_hdl,float, +106,107,NMR_phospholipids_in_medium_ldl,float, +107,108,NMR_phospholipids_in_medium_vldl,float, +108,109,NMR_phospholipids_in_small_hdl,float, +109,110,NMR_phospholipids_in_small_ldl,float, +110,111,NMR_phospholipids_in_small_vldl,float, +111,112,NMR_phospholipids_in_vldl,float, +112,113,NMR_phospholipids_in_very_large_hdl,float, +113,114,NMR_phospholipids_in_very_large_vldl,float, +114,115,NMR_phospholipids_in_very_small_vldl,float, +115,116,NMR_polyunsaturated_fatty_acids,float, +116,117,NMR_pyruvate,float, +117,118,NMR_remnant_cholesterol_nonhdl_nonldl_cholesterol,float, +118,119,NMR_saturated_fatty_acids,float, +119,120,NMR_sphingomyelins,float, +120,121,NMR_total_cholesterol,float, +121,122,NMR_total_cholesterol_minus_hdlc,float, +122,123,NMR_total_cholines,float, +123,124,NMR_total_concentration_of_branchedchain_amino_acids_leucine_isoleucine_valine,float, +124,125,NMR_total_concentration_of_lipoprotein_particles,float, +125,126,NMR_total_esterified_cholesterol,float, +126,127,NMR_total_fatty_acids,float, +127,128,NMR_total_free_cholesterol,float, +128,129,NMR_total_lipids_in_chylomicrons_and_extremely_large_vldl,float, +129,130,NMR_total_lipids_in_hdl,float, +130,131,NMR_total_lipids_in_idl,float, +131,132,NMR_total_lipids_in_ldl,float, +132,133,NMR_total_lipids_in_large_hdl,float, +133,134,NMR_total_lipids_in_large_ldl,float, +134,135,NMR_total_lipids_in_large_vldl,float, +135,136,NMR_total_lipids_in_lipoprotein_particles,float, +136,137,NMR_total_lipids_in_medium_hdl,float, +137,138,NMR_total_lipids_in_medium_ldl,float, +138,139,NMR_total_lipids_in_medium_vldl,float, +139,140,NMR_total_lipids_in_small_hdl,float, +140,141,NMR_total_lipids_in_small_ldl,float, +141,142,NMR_total_lipids_in_small_vldl,float, +142,143,NMR_total_lipids_in_vldl,float, +143,144,NMR_total_lipids_in_very_large_hdl,float, +144,145,NMR_total_lipids_in_very_large_vldl,float, +145,146,NMR_total_lipids_in_very_small_vldl,float, +146,147,NMR_total_phospholipids_in_lipoprotein_particles,float, +147,148,NMR_total_triglycerides,float, +148,149,NMR_triglycerides_in_chylomicrons_and_extremely_large_vldl,float, +149,150,NMR_triglycerides_in_hdl,float, +150,151,NMR_triglycerides_in_idl,float, +151,152,NMR_triglycerides_in_ldl,float, +152,153,NMR_triglycerides_in_large_hdl,float, +153,154,NMR_triglycerides_in_large_ldl,float, +154,155,NMR_triglycerides_in_large_vldl,float, +155,156,NMR_triglycerides_in_medium_hdl,float, +156,157,NMR_triglycerides_in_medium_ldl,float, +157,158,NMR_triglycerides_in_medium_vldl,float, +158,159,NMR_triglycerides_in_small_hdl,float, +159,160,NMR_triglycerides_in_small_ldl,float, +160,161,NMR_triglycerides_in_small_vldl,float, +161,162,NMR_triglycerides_in_vldl,float, +162,163,NMR_triglycerides_in_very_large_hdl,float, +163,164,NMR_triglycerides_in_very_large_vldl,float, +164,165,NMR_triglycerides_in_very_small_vldl,float, +165,166,NMR_tyrosine,float, +166,167,NMR_vldl_cholesterol,float, +167,168,NMR_valine,float, diff --git a/res/datasets/fake/metabolomics/metabolomics_test.h5ad.X.npy b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.X.npy new file mode 100644 index 0000000..6ead76e Binary files /dev/null and b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.X.npy differ diff --git a/res/datasets/fake/metabolomics/metabolomics_test.h5ad.obs.csv b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/metabolomics/metabolomics_test.h5ad.var.csv b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.var.csv new file mode 100644 index 0000000..cd357b2 --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_test.h5ad.var.csv @@ -0,0 +1,169 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,NMR_3hydroxybutyrate,float, +1,2,NMR_acetate,float, +2,3,NMR_acetoacetate,float, +3,4,NMR_acetone,float, +4,5,NMR_alanine,float, +5,6,NMR_albumin,float, +6,7,NMR_apolipoprotein_a1,float, +7,8,NMR_apolipoprotein_b,float, +8,9,NMR_average_diameter_for_hdl_particles,float, +9,10,NMR_average_diameter_for_ldl_particles,float, +10,11,NMR_average_diameter_for_vldl_particles,float, +11,12,NMR_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +12,13,NMR_cholesterol_in_idl,float, +13,14,NMR_cholesterol_in_large_hdl,float, +14,15,NMR_cholesterol_in_large_ldl,float, +15,16,NMR_cholesterol_in_large_vldl,float, +16,17,NMR_cholesterol_in_medium_hdl,float, +17,18,NMR_cholesterol_in_medium_ldl,float, +18,19,NMR_cholesterol_in_medium_vldl,float, +19,20,NMR_cholesterol_in_small_hdl,float, +20,21,NMR_cholesterol_in_small_ldl,float, +21,22,NMR_cholesterol_in_small_vldl,float, +22,23,NMR_cholesterol_in_very_large_hdl,float, +23,24,NMR_cholesterol_in_very_large_vldl,float, +24,25,NMR_cholesterol_in_very_small_vldl,float, +25,26,NMR_cholesteryl_esters_in_chylomicrons_and_extremely_large_vldl,float, +26,27,NMR_cholesteryl_esters_in_hdl,float, +27,28,NMR_cholesteryl_esters_in_idl,float, +28,29,NMR_cholesteryl_esters_in_ldl,float, +29,30,NMR_cholesteryl_esters_in_large_hdl,float, +30,31,NMR_cholesteryl_esters_in_large_ldl,float, +31,32,NMR_cholesteryl_esters_in_large_vldl,float, +32,33,NMR_cholesteryl_esters_in_medium_hdl,float, +33,34,NMR_cholesteryl_esters_in_medium_ldl,float, +34,35,NMR_cholesteryl_esters_in_medium_vldl,float, +35,36,NMR_cholesteryl_esters_in_small_hdl,float, +36,37,NMR_cholesteryl_esters_in_small_ldl,float, +37,38,NMR_cholesteryl_esters_in_small_vldl,float, +38,39,NMR_cholesteryl_esters_in_vldl,float, +39,40,NMR_cholesteryl_esters_in_very_large_hdl,float, +40,41,NMR_cholesteryl_esters_in_very_large_vldl,float, +41,42,NMR_cholesteryl_esters_in_very_small_vldl,float, +42,43,NMR_citrate,float, +43,44,NMR_clinical_ldl_cholesterol,float, +44,45,NMR_concentration_of_chylomicrons_and_extremely_large_vldl_particles,float, +45,46,NMR_concentration_of_hdl_particles,float, +46,47,NMR_concentration_of_idl_particles,float, +47,48,NMR_concentration_of_ldl_particles,float, +48,49,NMR_concentration_of_large_hdl_particles,float, +49,50,NMR_concentration_of_large_ldl_particles,float, +50,51,NMR_concentration_of_large_vldl_particles,float, +51,52,NMR_concentration_of_medium_hdl_particles,float, +52,53,NMR_concentration_of_medium_ldl_particles,float, +53,54,NMR_concentration_of_medium_vldl_particles,float, +54,55,NMR_concentration_of_small_hdl_particles,float, +55,56,NMR_concentration_of_small_ldl_particles,float, +56,57,NMR_concentration_of_small_vldl_particles,float, +57,58,NMR_concentration_of_vldl_particles,float, +58,59,NMR_concentration_of_very_large_hdl_particles,float, +59,60,NMR_concentration_of_very_large_vldl_particles,float, +60,61,NMR_concentration_of_very_small_vldl_particles,float, +61,62,NMR_creatinine,float, +62,63,NMR_degree_of_unsaturation,float, +63,64,NMR_docosahexaenoic_acid,float, +64,65,NMR_free_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +65,66,NMR_free_cholesterol_in_hdl,float, +66,67,NMR_free_cholesterol_in_idl,float, +67,68,NMR_free_cholesterol_in_ldl,float, +68,69,NMR_free_cholesterol_in_large_hdl,float, +69,70,NMR_free_cholesterol_in_large_ldl,float, +70,71,NMR_free_cholesterol_in_large_vldl,float, +71,72,NMR_free_cholesterol_in_medium_hdl,float, +72,73,NMR_free_cholesterol_in_medium_ldl,float, +73,74,NMR_free_cholesterol_in_medium_vldl,float, +74,75,NMR_free_cholesterol_in_small_hdl,float, +75,76,NMR_free_cholesterol_in_small_ldl,float, +76,77,NMR_free_cholesterol_in_small_vldl,float, +77,78,NMR_free_cholesterol_in_vldl,float, +78,79,NMR_free_cholesterol_in_very_large_hdl,float, +79,80,NMR_free_cholesterol_in_very_large_vldl,float, +80,81,NMR_free_cholesterol_in_very_small_vldl,float, +81,82,NMR_glucose,float, +82,83,NMR_glutamine,float, +83,84,NMR_glycine,float, +84,85,NMR_glycoprotein_acetyls,float, +85,86,NMR_hdl_cholesterol,float, +86,87,NMR_histidine,float, +87,88,NMR_isoleucine,float, +88,89,NMR_ldl_cholesterol,float, +89,90,NMR_lactate,float, +90,91,NMR_leucine,float, +91,92,NMR_linoleic_acid,float, +92,93,NMR_monounsaturated_fatty_acids,float, +93,94,NMR_omega3_fatty_acids,float, +94,95,NMR_omega6_fatty_acids,float, +95,96,NMR_phenylalanine,float, +96,97,NMR_phosphatidylcholines,float, +97,98,NMR_phosphoglycerides,float, +98,99,NMR_phospholipids_in_chylomicrons_and_extremely_large_vldl,float, +99,100,NMR_phospholipids_in_hdl,float, +100,101,NMR_phospholipids_in_idl,float, +101,102,NMR_phospholipids_in_ldl,float, +102,103,NMR_phospholipids_in_large_hdl,float, +103,104,NMR_phospholipids_in_large_ldl,float, +104,105,NMR_phospholipids_in_large_vldl,float, +105,106,NMR_phospholipids_in_medium_hdl,float, +106,107,NMR_phospholipids_in_medium_ldl,float, +107,108,NMR_phospholipids_in_medium_vldl,float, +108,109,NMR_phospholipids_in_small_hdl,float, +109,110,NMR_phospholipids_in_small_ldl,float, +110,111,NMR_phospholipids_in_small_vldl,float, +111,112,NMR_phospholipids_in_vldl,float, +112,113,NMR_phospholipids_in_very_large_hdl,float, +113,114,NMR_phospholipids_in_very_large_vldl,float, +114,115,NMR_phospholipids_in_very_small_vldl,float, +115,116,NMR_polyunsaturated_fatty_acids,float, +116,117,NMR_pyruvate,float, +117,118,NMR_remnant_cholesterol_nonhdl_nonldl_cholesterol,float, +118,119,NMR_saturated_fatty_acids,float, +119,120,NMR_sphingomyelins,float, +120,121,NMR_total_cholesterol,float, +121,122,NMR_total_cholesterol_minus_hdlc,float, +122,123,NMR_total_cholines,float, +123,124,NMR_total_concentration_of_branchedchain_amino_acids_leucine_isoleucine_valine,float, +124,125,NMR_total_concentration_of_lipoprotein_particles,float, +125,126,NMR_total_esterified_cholesterol,float, +126,127,NMR_total_fatty_acids,float, +127,128,NMR_total_free_cholesterol,float, +128,129,NMR_total_lipids_in_chylomicrons_and_extremely_large_vldl,float, +129,130,NMR_total_lipids_in_hdl,float, +130,131,NMR_total_lipids_in_idl,float, +131,132,NMR_total_lipids_in_ldl,float, +132,133,NMR_total_lipids_in_large_hdl,float, +133,134,NMR_total_lipids_in_large_ldl,float, +134,135,NMR_total_lipids_in_large_vldl,float, +135,136,NMR_total_lipids_in_lipoprotein_particles,float, +136,137,NMR_total_lipids_in_medium_hdl,float, +137,138,NMR_total_lipids_in_medium_ldl,float, +138,139,NMR_total_lipids_in_medium_vldl,float, +139,140,NMR_total_lipids_in_small_hdl,float, +140,141,NMR_total_lipids_in_small_ldl,float, +141,142,NMR_total_lipids_in_small_vldl,float, +142,143,NMR_total_lipids_in_vldl,float, +143,144,NMR_total_lipids_in_very_large_hdl,float, +144,145,NMR_total_lipids_in_very_large_vldl,float, +145,146,NMR_total_lipids_in_very_small_vldl,float, +146,147,NMR_total_phospholipids_in_lipoprotein_particles,float, +147,148,NMR_total_triglycerides,float, +148,149,NMR_triglycerides_in_chylomicrons_and_extremely_large_vldl,float, +149,150,NMR_triglycerides_in_hdl,float, +150,151,NMR_triglycerides_in_idl,float, +151,152,NMR_triglycerides_in_ldl,float, +152,153,NMR_triglycerides_in_large_hdl,float, +153,154,NMR_triglycerides_in_large_ldl,float, +154,155,NMR_triglycerides_in_large_vldl,float, +155,156,NMR_triglycerides_in_medium_hdl,float, +156,157,NMR_triglycerides_in_medium_ldl,float, +157,158,NMR_triglycerides_in_medium_vldl,float, +158,159,NMR_triglycerides_in_small_hdl,float, +159,160,NMR_triglycerides_in_small_ldl,float, +160,161,NMR_triglycerides_in_small_vldl,float, +161,162,NMR_triglycerides_in_vldl,float, +162,163,NMR_triglycerides_in_very_large_hdl,float, +163,164,NMR_triglycerides_in_very_large_vldl,float, +164,165,NMR_triglycerides_in_very_small_vldl,float, +165,166,NMR_tyrosine,float, +166,167,NMR_vldl_cholesterol,float, +167,168,NMR_valine,float, diff --git a/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.X.npy b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.X.npy new file mode 100644 index 0000000..e218c11 Binary files /dev/null and b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.obs.csv b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.var.csv b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.var.csv new file mode 100644 index 0000000..cd357b2 --- /dev/null +++ b/res/datasets/fake/metabolomics/metabolomics_valid.h5ad.var.csv @@ -0,0 +1,169 @@ +Unnamed: 0,id,Field,ValueType,Transformation +0,1,NMR_3hydroxybutyrate,float, +1,2,NMR_acetate,float, +2,3,NMR_acetoacetate,float, +3,4,NMR_acetone,float, +4,5,NMR_alanine,float, +5,6,NMR_albumin,float, +6,7,NMR_apolipoprotein_a1,float, +7,8,NMR_apolipoprotein_b,float, +8,9,NMR_average_diameter_for_hdl_particles,float, +9,10,NMR_average_diameter_for_ldl_particles,float, +10,11,NMR_average_diameter_for_vldl_particles,float, +11,12,NMR_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +12,13,NMR_cholesterol_in_idl,float, +13,14,NMR_cholesterol_in_large_hdl,float, +14,15,NMR_cholesterol_in_large_ldl,float, +15,16,NMR_cholesterol_in_large_vldl,float, +16,17,NMR_cholesterol_in_medium_hdl,float, +17,18,NMR_cholesterol_in_medium_ldl,float, +18,19,NMR_cholesterol_in_medium_vldl,float, +19,20,NMR_cholesterol_in_small_hdl,float, +20,21,NMR_cholesterol_in_small_ldl,float, +21,22,NMR_cholesterol_in_small_vldl,float, +22,23,NMR_cholesterol_in_very_large_hdl,float, +23,24,NMR_cholesterol_in_very_large_vldl,float, +24,25,NMR_cholesterol_in_very_small_vldl,float, +25,26,NMR_cholesteryl_esters_in_chylomicrons_and_extremely_large_vldl,float, +26,27,NMR_cholesteryl_esters_in_hdl,float, +27,28,NMR_cholesteryl_esters_in_idl,float, +28,29,NMR_cholesteryl_esters_in_ldl,float, +29,30,NMR_cholesteryl_esters_in_large_hdl,float, +30,31,NMR_cholesteryl_esters_in_large_ldl,float, +31,32,NMR_cholesteryl_esters_in_large_vldl,float, +32,33,NMR_cholesteryl_esters_in_medium_hdl,float, +33,34,NMR_cholesteryl_esters_in_medium_ldl,float, +34,35,NMR_cholesteryl_esters_in_medium_vldl,float, +35,36,NMR_cholesteryl_esters_in_small_hdl,float, +36,37,NMR_cholesteryl_esters_in_small_ldl,float, +37,38,NMR_cholesteryl_esters_in_small_vldl,float, +38,39,NMR_cholesteryl_esters_in_vldl,float, +39,40,NMR_cholesteryl_esters_in_very_large_hdl,float, +40,41,NMR_cholesteryl_esters_in_very_large_vldl,float, +41,42,NMR_cholesteryl_esters_in_very_small_vldl,float, +42,43,NMR_citrate,float, +43,44,NMR_clinical_ldl_cholesterol,float, +44,45,NMR_concentration_of_chylomicrons_and_extremely_large_vldl_particles,float, +45,46,NMR_concentration_of_hdl_particles,float, +46,47,NMR_concentration_of_idl_particles,float, +47,48,NMR_concentration_of_ldl_particles,float, +48,49,NMR_concentration_of_large_hdl_particles,float, +49,50,NMR_concentration_of_large_ldl_particles,float, +50,51,NMR_concentration_of_large_vldl_particles,float, +51,52,NMR_concentration_of_medium_hdl_particles,float, +52,53,NMR_concentration_of_medium_ldl_particles,float, +53,54,NMR_concentration_of_medium_vldl_particles,float, +54,55,NMR_concentration_of_small_hdl_particles,float, +55,56,NMR_concentration_of_small_ldl_particles,float, +56,57,NMR_concentration_of_small_vldl_particles,float, +57,58,NMR_concentration_of_vldl_particles,float, +58,59,NMR_concentration_of_very_large_hdl_particles,float, +59,60,NMR_concentration_of_very_large_vldl_particles,float, +60,61,NMR_concentration_of_very_small_vldl_particles,float, +61,62,NMR_creatinine,float, +62,63,NMR_degree_of_unsaturation,float, +63,64,NMR_docosahexaenoic_acid,float, +64,65,NMR_free_cholesterol_in_chylomicrons_and_extremely_large_vldl,float, +65,66,NMR_free_cholesterol_in_hdl,float, +66,67,NMR_free_cholesterol_in_idl,float, +67,68,NMR_free_cholesterol_in_ldl,float, +68,69,NMR_free_cholesterol_in_large_hdl,float, +69,70,NMR_free_cholesterol_in_large_ldl,float, +70,71,NMR_free_cholesterol_in_large_vldl,float, +71,72,NMR_free_cholesterol_in_medium_hdl,float, +72,73,NMR_free_cholesterol_in_medium_ldl,float, +73,74,NMR_free_cholesterol_in_medium_vldl,float, +74,75,NMR_free_cholesterol_in_small_hdl,float, +75,76,NMR_free_cholesterol_in_small_ldl,float, +76,77,NMR_free_cholesterol_in_small_vldl,float, +77,78,NMR_free_cholesterol_in_vldl,float, +78,79,NMR_free_cholesterol_in_very_large_hdl,float, +79,80,NMR_free_cholesterol_in_very_large_vldl,float, +80,81,NMR_free_cholesterol_in_very_small_vldl,float, +81,82,NMR_glucose,float, +82,83,NMR_glutamine,float, +83,84,NMR_glycine,float, +84,85,NMR_glycoprotein_acetyls,float, +85,86,NMR_hdl_cholesterol,float, +86,87,NMR_histidine,float, +87,88,NMR_isoleucine,float, +88,89,NMR_ldl_cholesterol,float, +89,90,NMR_lactate,float, +90,91,NMR_leucine,float, +91,92,NMR_linoleic_acid,float, +92,93,NMR_monounsaturated_fatty_acids,float, +93,94,NMR_omega3_fatty_acids,float, +94,95,NMR_omega6_fatty_acids,float, +95,96,NMR_phenylalanine,float, +96,97,NMR_phosphatidylcholines,float, +97,98,NMR_phosphoglycerides,float, +98,99,NMR_phospholipids_in_chylomicrons_and_extremely_large_vldl,float, +99,100,NMR_phospholipids_in_hdl,float, +100,101,NMR_phospholipids_in_idl,float, +101,102,NMR_phospholipids_in_ldl,float, +102,103,NMR_phospholipids_in_large_hdl,float, +103,104,NMR_phospholipids_in_large_ldl,float, +104,105,NMR_phospholipids_in_large_vldl,float, +105,106,NMR_phospholipids_in_medium_hdl,float, +106,107,NMR_phospholipids_in_medium_ldl,float, +107,108,NMR_phospholipids_in_medium_vldl,float, +108,109,NMR_phospholipids_in_small_hdl,float, +109,110,NMR_phospholipids_in_small_ldl,float, +110,111,NMR_phospholipids_in_small_vldl,float, +111,112,NMR_phospholipids_in_vldl,float, +112,113,NMR_phospholipids_in_very_large_hdl,float, +113,114,NMR_phospholipids_in_very_large_vldl,float, +114,115,NMR_phospholipids_in_very_small_vldl,float, +115,116,NMR_polyunsaturated_fatty_acids,float, +116,117,NMR_pyruvate,float, +117,118,NMR_remnant_cholesterol_nonhdl_nonldl_cholesterol,float, +118,119,NMR_saturated_fatty_acids,float, +119,120,NMR_sphingomyelins,float, +120,121,NMR_total_cholesterol,float, +121,122,NMR_total_cholesterol_minus_hdlc,float, +122,123,NMR_total_cholines,float, +123,124,NMR_total_concentration_of_branchedchain_amino_acids_leucine_isoleucine_valine,float, +124,125,NMR_total_concentration_of_lipoprotein_particles,float, +125,126,NMR_total_esterified_cholesterol,float, +126,127,NMR_total_fatty_acids,float, +127,128,NMR_total_free_cholesterol,float, +128,129,NMR_total_lipids_in_chylomicrons_and_extremely_large_vldl,float, +129,130,NMR_total_lipids_in_hdl,float, +130,131,NMR_total_lipids_in_idl,float, +131,132,NMR_total_lipids_in_ldl,float, +132,133,NMR_total_lipids_in_large_hdl,float, +133,134,NMR_total_lipids_in_large_ldl,float, +134,135,NMR_total_lipids_in_large_vldl,float, +135,136,NMR_total_lipids_in_lipoprotein_particles,float, +136,137,NMR_total_lipids_in_medium_hdl,float, +137,138,NMR_total_lipids_in_medium_ldl,float, +138,139,NMR_total_lipids_in_medium_vldl,float, +139,140,NMR_total_lipids_in_small_hdl,float, +140,141,NMR_total_lipids_in_small_ldl,float, +141,142,NMR_total_lipids_in_small_vldl,float, +142,143,NMR_total_lipids_in_vldl,float, +143,144,NMR_total_lipids_in_very_large_hdl,float, +144,145,NMR_total_lipids_in_very_large_vldl,float, +145,146,NMR_total_lipids_in_very_small_vldl,float, +146,147,NMR_total_phospholipids_in_lipoprotein_particles,float, +147,148,NMR_total_triglycerides,float, +148,149,NMR_triglycerides_in_chylomicrons_and_extremely_large_vldl,float, +149,150,NMR_triglycerides_in_hdl,float, +150,151,NMR_triglycerides_in_idl,float, +151,152,NMR_triglycerides_in_ldl,float, +152,153,NMR_triglycerides_in_large_hdl,float, +153,154,NMR_triglycerides_in_large_ldl,float, +154,155,NMR_triglycerides_in_large_vldl,float, +155,156,NMR_triglycerides_in_medium_hdl,float, +156,157,NMR_triglycerides_in_medium_ldl,float, +157,158,NMR_triglycerides_in_medium_vldl,float, +158,159,NMR_triglycerides_in_small_hdl,float, +159,160,NMR_triglycerides_in_small_ldl,float, +160,161,NMR_triglycerides_in_small_vldl,float, +161,162,NMR_triglycerides_in_vldl,float, +162,163,NMR_triglycerides_in_very_large_hdl,float, +163,164,NMR_triglycerides_in_very_large_vldl,float, +164,165,NMR_triglycerides_in_very_small_vldl,float, +165,166,NMR_tyrosine,float, +166,167,NMR_vldl_cholesterol,float, +167,168,NMR_valine,float, diff --git a/res/datasets/fake/pheno/cad_extended_tr.h5ad.X.npy b/res/datasets/fake/pheno/cad_extended_tr.h5ad.X.npy new file mode 100644 index 0000000..a8db1fa Binary files /dev/null and b/res/datasets/fake/pheno/cad_extended_tr.h5ad.X.npy differ diff --git a/res/datasets/fake/pheno/cad_extended_tr.h5ad.obs.csv b/res/datasets/fake/pheno/cad_extended_tr.h5ad.obs.csv new file mode 100644 index 0000000..088615f --- /dev/null +++ b/res/datasets/fake/pheno/cad_extended_tr.h5ad.obs.csv @@ -0,0 +1,2049 @@ +eid +5797 +5099 +1791 +7810 +1972 +3055 +9086 +5933 +1103 +4152 +9336 +2061 +5212 +5882 +1734 +7972 +8770 +8257 +7361 +8832 +3437 +3602 +8705 +2989 +8940 +1698 +1518 +7987 +4444 +4489 +7713 +4564 +8806 +5616 +3456 +4049 +5711 +3652 +4104 +7500 +3733 +2740 +2058 +2584 +2225 +4177 +1496 +1632 +3432 +2099 +7264 +4682 +3625 +6479 +2541 +4164 +2669 +1872 +4169 +2555 +5738 +8969 +9447 +9377 +7262 +7816 +4407 +8290 +6271 +5662 +9752 +2214 +2399 +5184 +6460 +4427 +5440 +2015 +6009 +6960 +8541 +8295 +1423 +9058 +9441 +6816 +7453 +9549 +8037 +7485 +8809 +4429 +2458 +2071 +9949 +2576 +6134 +1990 +4363 +7462 +7536 +6041 +4945 +4826 +7938 +8489 +7245 +9111 +9356 +6850 +3778 +7078 +4418 +8475 +8106 +5468 +3348 +2908 +1524 +2945 +4019 +1985 +7887 +5922 +6748 +4912 +7688 +2022 +3276 +6244 +6261 +9163 +4066 +4869 +4233 +9718 +3911 +6433 +4114 +6897 +5494 +3746 +5198 +9028 +2701 +4545 +6725 +8848 +3556 +2403 +7156 +8126 +1201 +9035 +6970 +2364 +1710 +7507 +8961 +8549 +6689 +7568 +8671 +9416 +3667 +5252 +5441 +5765 +8932 +1545 +8649 +3919 +1261 +9566 +2852 +8346 +5080 +1305 +2216 +3195 +7942 +4457 +8529 +7838 +9288 +7872 +8147 +6684 +4051 +7190 +7375 +1940 +1483 +1611 +1085 +1677 +9278 +5137 +9301 +4471 +1196 +3695 +1391 +2575 +1232 +5815 +9108 +5841 +8730 +1363 +9397 +6787 +8473 +2534 +8060 +9971 +3519 +4845 +5363 +2586 +8872 +1390 +3563 +8572 +1115 +4898 +6704 +8494 +3749 +4805 +2222 +2397 +2533 +5803 +5462 +3774 +2542 +7225 +7880 +5613 +4336 +1885 +6280 +6723 +5168 +9282 +7412 +6286 +1189 +8920 +1775 +5813 +1178 +7605 +6807 +9958 +7497 +2125 +4482 +5638 +3274 +6636 +8540 +7074 +1296 +8931 +3765 +2751 +2526 +3126 +6131 +3102 +9042 +6420 +2926 +9220 +3871 +2756 +6997 +7825 +1907 +7994 +1983 +3163 +2352 +8409 +2490 +3067 +6143 +4505 +1772 +4105 +8935 +4009 +3061 +6033 +1880 +7379 +5051 +5684 +2861 +5459 +6878 +1379 +9407 +9864 +7154 +4279 +3875 +2822 +8195 +3210 +9741 +7815 +7098 +8898 +4442 +6435 +4055 +2950 +5507 +8904 +2351 +4595 +9486 +5667 +1336 +2231 +2718 +9719 +3766 +6610 +6153 +1689 +4334 +3268 +6841 +8042 +8143 +7545 +8554 +3252 +3234 +3825 +4385 +3295 +9593 +3976 +4995 +8986 +7862 +7528 +5012 +3066 +7986 +8655 +4223 +6485 +9615 +5785 +1640 +5707 +5533 +6235 +4759 +7005 +2054 +3988 +9369 +4615 +5916 +4338 +4678 +7653 +6055 +5986 +4739 +3296 +8206 +1715 +9618 +8125 +3865 +2191 +4946 +2866 +5719 +5109 +7446 +2078 +2449 +9388 +4537 +1007 +6124 +4903 +9422 +4311 +6913 +5682 +9662 +5324 +9149 +6606 +8469 +2979 +7624 +4028 +3892 +1234 +2240 +1187 +7623 +3403 +7265 +1799 +4362 +3570 +7486 +6296 +6512 +4918 +5333 +9171 +3111 +6583 +5731 +8086 +3820 +4743 +2787 +8082 +6979 +1472 +7324 +3693 +8273 +8234 +8631 +8838 +3977 +6417 +4004 +7013 +1013 +7276 +5767 +7496 +3281 +7526 +3946 +5360 +9948 +6175 +3413 +1500 +7530 +2157 +1399 +6236 +1341 +8130 +1583 +5874 +3564 +9592 +3367 +2766 +5389 +3503 +7608 +6069 +4210 +3935 +6948 +6028 +2635 +5023 +9841 +4998 +6555 +8639 +7155 +9567 +1629 +3068 +9025 +1063 +1782 +1853 +8250 +9506 +6650 +8427 +2080 +4496 +9097 +7307 +9110 +7393 +1680 +4921 +6369 +7522 +6699 +1129 +9267 +7769 +7908 +8936 +1915 +6453 +1426 +6885 +8404 +8766 +2243 +4240 +8543 +4582 +5374 +9931 +8716 +6860 +4891 +7034 +8648 +2918 +5224 +1708 +5244 +2779 +8868 +4843 +5202 +4625 +6981 +7012 +6739 +8710 +9382 +3246 +3357 +8136 +2831 +9263 +1297 +3569 +3728 +6316 +9342 +8445 +3594 +4141 +3908 +8569 +3353 +2935 +2221 +3125 +7895 +2025 +4562 +4557 +7721 +6883 +7563 +7723 +1946 +7360 +8669 +1714 +1349 +8145 +1011 +4713 +6924 +3231 +5970 +1958 +8588 +7775 +3023 +4515 +9533 +7845 +9969 +4750 +7314 +5152 +8054 +7784 +8172 +3422 +7693 +7952 +1048 +4978 +9510 +8284 +8355 +2988 +9793 +2637 +6584 +3656 +2675 +9121 +1065 +7823 +8605 +4848 +6121 +5583 +9130 +6861 +2886 +5457 +8159 +1737 +5291 +8788 +6744 +5380 +1317 +6791 +2660 +9529 +3406 +6303 +3614 +6438 +2976 +2796 +2366 +1777 +4291 +9465 +7096 +4280 +1931 +9543 +3810 +1835 +5894 +9002 +7861 +2713 +9327 +2689 +1836 +1505 +7418 +1439 +7800 +7831 +6848 +9926 +4045 +4908 +5599 +1134 +3530 +8638 +1290 +1923 +5904 +5673 +1486 +1565 +7062 +5474 +9734 +4737 +2398 +2930 +4635 +7752 +3897 +2474 +8748 +8310 +1871 +9689 +3351 +2139 +7045 +7069 +5350 +1904 +7818 +5147 +3110 +3756 +3501 +7783 +4899 +1032 +3878 +2255 +4698 +2370 +7884 +8773 +3598 +6392 +5947 +8104 +7841 +2884 +8089 +4497 +8338 +9972 +6983 +2505 +6813 +5049 +3499 +1327 +5571 +3940 +1993 +6413 +1406 +3852 +9792 +6475 +1080 +2678 +6792 +3915 +2906 +3943 +8441 +3318 +5188 +3088 +7519 +9599 +7645 +6452 +3230 +3063 +1137 +6703 +5019 +8415 +1125 +2464 +9027 +4850 +5843 +7187 +6297 +1833 +6012 +6892 +9944 +1666 +6273 +7535 +7502 +4630 +4074 +2446 +7868 +4556 +3835 +8369 +5434 +1084 +2018 +4867 +7169 +8509 +9383 +9541 +8678 +5512 +4125 +5196 +2415 +9487 +6736 +1811 +1164 +5166 +3221 +1250 +2670 +4589 +5715 +4460 +5521 +4840 +5948 +2659 +6059 +3688 +2560 +8217 +6632 +3333 +3666 +4700 +4578 +5234 +8248 +6351 +3742 +1477 +8094 +7052 +8758 +8291 +1750 +8477 +1513 +6939 +2029 +9906 +8804 +6912 +3272 +8372 +6517 +4751 +5958 +6018 +4304 +7933 +7767 +8556 +2846 +6172 +8416 +9919 +6355 +3872 +9907 +7583 +7388 +6384 +2835 +4970 +3736 +1289 +1653 +1638 +9040 +4332 +9411 +8740 +2316 +7751 +1033 +8695 +4183 +4208 +7068 +6025 +6627 +6872 +7123 +7284 +4309 +9284 +2854 +4140 +5020 +3851 +3939 +9505 +9199 +3883 +1324 +5382 +4098 +9986 +1938 +6242 +7812 +4882 +9932 +3259 +9878 +1517 +7690 +2079 +8069 +4652 +2204 +1469 +8919 +8178 +4394 +5300 +2982 +1489 +8407 +4807 +4522 +9083 +3809 +4671 +1709 +4244 +9716 +1716 +6255 +2293 +5478 +7011 +7561 +8175 +1848 +5386 +5303 +7509 +8583 +9730 +9545 +6928 +9659 +5435 +5098 +5842 +7380 +9976 +3889 +8558 +4142 +4590 +5163 +1893 +9322 +7907 +1692 +1531 +3257 +5365 +1959 +5989 +7642 +4379 +9563 +2428 +8573 +9312 +9060 +5240 +8951 +3956 +7184 +2962 +2574 +4857 +7891 +8443 +1725 +5547 +3672 +2343 +7788 +2924 +6932 +8783 +6772 +9933 +5550 +6322 +8396 +5358 +7578 +1802 +9875 +8111 +5469 +1463 +8224 +4628 +8937 +9229 +6694 +2648 +1183 +1220 +4982 +8163 +6778 +7279 +4126 +7121 +7277 +4801 +6665 +5991 +6993 +5094 +8742 +4738 +5014 +3341 +5132 +7195 +4661 +9616 +3996 +6911 +4176 +7525 +5250 +4232 +6530 +5944 +2282 +8442 +3721 +4378 +4803 +2687 +2682 +8722 +8072 +6234 +8735 +7285 +2419 +4414 +2207 +1157 +5661 +6925 +5847 +4348 +4477 +7848 +1374 +5134 +2538 +4657 +2091 +6140 +4926 +1375 +2755 +3250 +5556 +7203 +7477 +1800 +1275 +5867 +2783 +6922 +6262 +6731 +3212 +5390 +1831 +9418 +1532 +7134 +2082 +5170 +9856 +2768 +2753 +6266 +3663 +9491 +1650 +2287 +3647 +8805 +8777 +8182 +5264 +4076 +9732 +2492 +1140 +2791 +9681 +3377 +3953 +6196 +5024 +9674 +6919 +9885 +9146 +4521 +2186 +3283 +2570 +3568 +1786 +7974 +9094 +1561 +2187 +1903 +8480 +6836 +7255 +7215 +3169 +1070 +8739 +9915 +1029 +9759 +7472 +8305 +8791 +1874 +1459 +1166 +1602 +5640 +8077 +3619 +1975 +3092 +1630 +1954 +1392 +2590 +6509 +4968 +9134 +5267 +7893 +8927 +7829 +6815 +9064 +7591 +6385 +2717 +4364 +7256 +1847 +9764 +7007 +8731 +3611 +8006 +9191 +2867 +3964 +4567 +6680 +7088 +8316 +9683 +3181 +2077 +8183 +8202 +2917 +3340 +2475 +7385 +3832 +7777 +1933 +2816 +5579 +7476 +3755 +1720 +7120 +4610 +9939 +8261 +7396 +9133 +8364 +5923 +5157 +9824 +5074 +4127 +5002 +2013 +2266 +2328 +9265 +6817 +1875 +6035 +3307 +1987 +5793 +2230 +6152 +5888 +7178 +5239 +3476 +8747 +1113 +9901 +3020 +9672 +5119 +1303 +3505 +6189 +5658 +7234 +8765 +4519 +9601 +7883 +3849 +6773 +4871 +6587 +8070 +7209 +1541 +7116 +8033 +4736 +8433 +3907 +3302 +6277 +9223 +5938 +9311 +1345 +9283 +5931 +7827 +7055 +6049 +3847 +6201 +4864 +1837 +2228 +4091 +5015 +5779 +6340 +5311 +4110 +3801 +8717 +4081 +2640 +5915 +5381 +8989 +9834 +7732 +1877 +6891 +7539 +3681 +7471 +9787 +7463 +7853 +6795 +2961 +2502 +6806 +4423 +2254 +7881 +8051 +6585 +6532 +2561 +1832 +5191 +3630 +5270 +5564 +9953 +5492 +3815 +7977 +3966 +3916 +9084 +3138 +5245 +4963 +6195 +8123 +5332 +8283 +5671 +6042 +3793 +9994 +9376 +2162 +9508 +3048 +6514 +6233 +4236 +1319 +6926 +6165 +6560 +5960 +1438 +2290 +8824 +3557 +4492 +9828 +8992 +3408 +5034 +4128 +6692 +3597 +3708 +7341 +5605 +6187 +8239 +7949 +9299 +1413 +6613 +9675 +8676 +9584 +6027 +8358 +5789 +8174 +6092 +2264 +8081 +5261 +6592 +1116 +3222 +4940 +9401 +5540 +8294 +4830 +5219 +7356 +3267 +6158 +3803 +7102 +5403 +9519 +1000 +6915 +7549 +4523 +3509 +9790 +9291 +4782 +5086 +3312 +7219 +2220 +6771 +7687 +9679 +2797 +7381 +3987 +8238 +3056 +5868 +7612 +9710 +7451 +5889 +6879 +5089 +5907 +1506 +7159 +8521 +5482 +1398 +3211 +2995 +9779 +5997 +8235 +8362 +1571 +9029 +9092 +5376 +9118 +2749 +6393 +3677 +7321 +9914 +6315 +8585 +3558 +3893 +3289 +3003 +3700 +4711 +1851 +1225 +2573 +9717 +1981 +3572 +1994 +7227 +6800 +8281 +4247 +9745 +8620 +8345 +7006 +6645 +5316 +7587 +6832 +1788 +1574 +1414 +6366 +3843 +5664 +2444 +1508 +1860 +7461 +4560 +9439 +3484 +4387 +6985 +5792 +3249 +6553 +3604 +9645 +2188 +9153 +6599 +5531 +7150 +3637 +9305 +3315 +8684 +5911 +6327 +7512 +1461 +5309 +8794 +1636 +8759 +1304 +1989 +3200 +7108 +2624 +8610 +7603 +5608 +1861 +9038 +1006 +6991 +1936 +2113 +3510 +2595 +4608 +7708 +1236 +4430 +8116 +3821 +9555 +2008 +2178 +6403 +9797 +9204 +5957 +7754 +7166 +2108 +3025 +8658 +5964 +1076 +2593 +4863 +6232 +8993 +7029 +8863 +1597 +8530 +3574 +1088 +7197 +2876 +4285 +3438 +4229 +9128 +7411 +3527 +7547 +3498 +3085 +9913 +5825 +7200 +1980 +2571 +3193 +3722 +5464 +1087 +3354 +9539 +7312 +9835 +1026 +7354 +1280 +5335 +3999 +3280 +2794 +6167 +8734 +7546 +7517 +7269 +6054 +7698 +9126 +2824 +7452 +1784 +6022 +5657 +8548 +2531 +9767 +9466 +4199 +1241 +2959 +7626 +7296 +3447 +6188 +1604 +3696 +9705 +6160 +9920 +3470 +9654 +9996 +2306 +9090 +8276 +2966 +5489 +9215 +9727 +1332 +8197 +6700 +6128 +9328 +8047 +9062 +2841 +8677 +9544 +8189 +5490 +4402 +6802 +4611 +9087 +1226 +8921 +7585 +5740 +5371 +4600 +9456 +5866 +7310 +1783 +8666 +2543 +1570 +1891 +3575 +5901 +4026 +9624 +9234 +4719 +2324 +8184 +3154 +4572 +9335 +2655 +5430 +7596 +3958 +8990 +4027 +3853 +7475 +7926 +1334 +3336 +2596 +7503 +4584 +4510 +5174 +4331 +3196 +5306 +3640 +3465 +5208 +3653 +4798 +9295 +8055 +3955 +6130 +9330 +4367 +9352 +2946 +4064 +4415 +9507 +8161 +7969 +9140 +1404 +4566 +9434 +2369 +2529 +7313 +5130 +4892 +1425 +7735 +6499 +5669 +7160 +4258 +6461 +5509 +5041 +8453 +7090 +3086 +9302 +7398 +9731 +9900 +3586 +1745 +4959 +6390 +1371 +2257 +1514 +8861 +6769 +5631 +9122 +5826 +3580 +8141 +5577 +4794 +1468 +1927 +4937 +2394 +9405 +1381 +2972 +1926 +7613 +9661 +7189 +2694 +5439 +8278 +5518 +3592 +5378 +4175 +4033 +5679 +6086 +7316 +3971 +8738 +2790 +2851 +7614 +7971 +6741 +6111 +9613 +9736 +8590 +5288 +9521 +7665 +6930 +2172 +9073 +7362 +5006 +5817 +5054 +6775 +7498 +5420 +6625 +7378 +7043 +6895 +4250 +4555 +8633 +5187 +5189 +5553 +5081 +7138 +3204 +2567 +6579 +5307 +7376 +1686 +5226 +7420 +2503 +7684 +8979 +5831 +2714 +3655 +8827 +7621 +8975 +3034 +4717 +7764 +4530 +6768 +7403 +9739 +2341 +3128 +4531 +7551 +4180 +8181 +3492 +3441 +9712 +2424 +7822 +6561 +1342 +4645 +3273 +1809 +1165 +5473 +7686 +9241 +5732 +8891 +8943 +6353 +9648 +2473 +7995 +8977 +8664 +1902 +6871 +5069 +8899 +7107 +1591 +7332 +4844 +7617 +8518 +6756 +2275 +7988 +2460 +5339 +7113 +5215 +6052 +1263 +8533 +2031 +1914 +9836 +7142 +3682 +4658 +8896 +2607 +4403 +5634 +3720 +4368 +7787 +1195 +8464 +9697 +5101 +4563 +7038 +6964 +1642 +3711 +5230 +9427 +5974 +4553 +3100 +3317 +1126 +2420 +7961 +3152 +2147 +5782 +3188 +1932 +1053 +4025 +6987 +2546 +6364 +7529 +6119 +9501 +3593 +2327 +7928 +7079 +2630 +2407 +6903 +4395 +5965 +2812 +5176 +3844 +5097 +3854 +2175 +7913 +9039 +6122 +8750 +8510 +1211 +4393 +8663 +2431 +1154 +7373 +3559 +6550 +1911 +1308 +1828 +1601 +9655 +3174 +9850 +5878 +1083 +7261 +5701 +9081 +5475 +9269 +6988 +7575 +9568 +9184 +3796 +4668 +2101 +5185 +5412 +7141 +8683 +1652 +8383 +9578 +8519 +3455 +2612 +8439 +7719 +7505 +7119 +9480 +6328 +6714 +2385 +4783 +5091 +8560 +3691 +4712 +8132 +7836 +9865 +1522 +5537 +7973 +4575 +3435 +6363 +1731 +4960 +8263 +1663 +9696 +6935 +6058 +7906 +8967 +9164 +5499 +3867 +2708 +8140 +4880 +8837 +5892 +4676 +1773 +1410 +5630 +7204 +9054 +6011 +4919 +2601 +9304 +8686 +8973 +1204 +7960 +2356 +6630 +8825 +4012 +5465 +1957 +7909 +6942 +3355 +8274 +4470 +4190 +2860 +5729 +3467 +1130 +4865 +2194 +4401 +2836 +8635 +2347 +3834 +6223 +7540 +4318 +2383 +1724 +6401 +6100 +1359 +3553 +9381 +3084 +2039 +8280 +8591 +7514 +2295 +6628 +6308 +6043 +3817 +1408 +3658 +3081 +6890 +5644 +2514 +3789 diff --git a/res/datasets/fake/pheno/cad_extended_tr.h5ad.var.csv b/res/datasets/fake/pheno/cad_extended_tr.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/pheno/cad_extended_tr.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/pheno/cad_test.h5ad.X.npy b/res/datasets/fake/pheno/cad_test.h5ad.X.npy new file mode 100644 index 0000000..263cbeb Binary files /dev/null and b/res/datasets/fake/pheno/cad_test.h5ad.X.npy differ diff --git a/res/datasets/fake/pheno/cad_test.h5ad.obs.csv b/res/datasets/fake/pheno/cad_test.h5ad.obs.csv new file mode 100644 index 0000000..691e980 --- /dev/null +++ b/res/datasets/fake/pheno/cad_test.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5043 +6379 +5236 +3107 +7782 +1581 +9723 +5282 +4339 +4450 +1573 +2453 +9756 +7054 +3737 +4325 +1437 +7122 +1203 +4381 +5093 +9375 +8944 +8066 +1951 +8879 +5117 +1863 +4994 +9909 +1637 +4082 +2086 +1882 +1014 +5241 +9224 +4138 +7300 +2939 +6358 +6563 +3047 +1816 +4896 +6214 +5595 +3489 +8400 +4980 +7662 +6294 +5829 +6093 +2663 +8849 +1843 +8653 +6183 +7966 +3358 +5522 +6168 +3410 +1944 +3768 +8807 +4904 +8892 +5175 +6673 +4464 +6908 +3980 +4820 +9303 +7383 +3374 +2569 +7231 +3925 +4102 +3115 +5396 +3466 +4117 +5472 +9743 +7647 +7586 +6136 +7065 +6305 +7366 +6252 +3797 +8422 +6677 +7677 +4693 +5470 +2878 +3828 +4670 +8043 +5516 +7020 +7930 +8703 +4131 +8474 +7146 +2826 +2798 +3027 +8322 +1064 +7620 +1224 +9807 +4823 +1457 +5900 +4854 +3541 +9339 +6447 +9580 +8675 +7593 +7957 +7390 +3439 +4594 +4638 +4255 +4220 +4037 +9468 +7636 +3639 +1866 +4699 +6231 +3044 +1107 +5003 +8302 +9525 +7865 +3134 +7553 +6274 +6021 +4132 +8381 +6812 +8577 +5123 +5601 +4952 +4166 +4613 +3393 +9161 +2772 +2629 +2580 +4916 +2072 +7849 +2742 +8772 +2646 +8428 +3901 +6647 +8856 +5953 +7168 +7911 +3362 +6436 +2782 +4235 +4434 +4991 +3176 +9350 +2410 +6839 +3727 +8139 +8907 +8103 +2047 +9761 +4893 +5937 +7483 +4356 +9248 +1411 +1873 +7733 +8962 +9886 +6194 +6085 +2103 +2978 +3836 +3001 +8568 +6567 +4340 +7929 +9445 +7510 +8823 +5508 +3891 +8659 +1133 +9334 +3205 +5121 +5336 +6907 +3942 +1313 +8272 +9950 +5063 +5794 +7263 +6620 +5281 +1415 +9400 +9918 +4660 +6863 +5018 +6410 +5491 +3833 +5302 +9517 +3931 +3287 +4438 +1898 +4879 +4139 +8501 +9530 +2789 +3683 +3214 +8221 +6934 +1732 +7729 +6581 +9853 +1216 +2457 +2150 +8046 +7758 +4643 +3338 +7436 +4121 +5952 +3675 +6125 +3629 +2459 +4146 +4080 +8244 +6659 +4955 +1806 +9808 +7318 +5840 +3120 +4502 +2169 +1117 +3985 +9826 +2430 +4639 +6975 +3898 +7602 +5666 +6216 +1266 +7572 +1908 +5930 +8500 +9477 +8922 +4949 +5535 +4421 +2705 +8303 +8317 +2425 +2696 +3199 +9132 +1969 +5565 +6811 +7364 +1778 +8542 +5567 +5144 +5161 +1780 +7248 +7669 +4245 +2344 +6501 +6494 +9139 +7047 +5037 +5480 +7628 +3724 +6835 +3517 +7283 +5552 +1018 +6096 +7811 +1693 +1302 +1507 +8468 +6770 +6224 +7459 +5192 +1964 +6335 +8451 +5148 +2499 +2980 +3960 +4031 +9833 +4547 +9014 +7770 +4541 +5479 +2577 +4690 +7468 +4933 +9640 +2137 +6318 +6427 +8914 +7399 +9801 +5487 +2427 +9703 +2565 +3753 +2089 +7826 +2226 +6824 +3494 +6182 +3347 +2198 +3487 +1552 +2027 +8091 +8318 +6535 +1174 +8004 +7832 +2075 +2885 +6173 +7579 +4136 +7950 +1147 +1868 +6040 +8636 +8455 +4815 +3396 +9071 +2639 +8778 +6765 +3862 +2131 +5780 +6430 +9478 +3419 +4922 +8120 +1184 +2785 +3483 +2211 +2210 +2743 +9207 +8687 +1131 +4696 +6547 +7162 +8040 +1069 +1961 +7798 +7979 +7956 +5876 +3468 +4306 +8721 +7454 +6945 +1128 +4383 +1467 +6200 +3669 +9968 +3864 +4889 +7147 +1862 +7445 +8905 +9372 +6537 +9472 +8050 +2201 +2197 +1329 +7637 +1615 +9242 +8173 +8874 +5067 +7780 +7132 +5011 +5349 +6264 +5330 +1370 +5588 +5171 +7370 +4198 +2090 +9526 +3903 +7232 +8380 +5027 +9169 +2377 +6180 +4485 +9542 +2094 +6198 +6478 +9810 +2100 +2510 +1968 +2627 +7101 +9937 +9019 +5511 +9531 +6459 +5066 +5771 +5647 +6098 +3239 +5271 +7339 +7555 +7467 +6378 +6990 +7876 +3982 +1590 +5660 +4648 +2331 +5845 +1567 +6177 +5548 +5126 +3936 +2052 +3278 +9497 +3969 +1252 +7010 +8657 +5242 +9159 +3194 +5456 +6880 +5030 +8618 +9771 +8017 +5417 +1916 +4837 +5290 +8448 +2491 +6359 +5946 +8913 +6896 +3645 +9412 +1264 +9371 +3401 +3183 +6343 +7046 +5983 +3747 +4503 +2970 +5775 +8694 +5757 +4475 +7840 +5777 +6434 +4467 +7717 +9923 +8522 +5129 +2620 +5710 +8068 +9050 +2880 +5167 +8307 +8517 +8315 +9684 +9859 +7002 +9230 +1684 +1299 +3264 +7384 +6095 +8910 +7224 +3402 +1073 +8575 +5050 +7592 +8644 +2568 +7428 +3386 +4327 +5557 +7183 +3282 +4626 +3425 +1819 +1427 +9880 +3167 +5118 +7193 +1608 +6480 +9991 +3298 +6230 +8300 +2617 +1474 +6442 +2060 +5217 +6477 +7559 +6249 +1239 +1424 +5044 +5151 +8592 +1429 +3561 +4791 +7607 +6507 +5761 +4413 +1930 +3566 +2218 +3900 +3119 +7749 +4371 +9652 +1690 +1661 +7127 +9462 +7447 +8621 +1043 +5734 +5621 +5591 +7458 +4380 +9625 +8326 +5650 +2881 +7993 +6362 +8435 +4218 +2093 +6717 +2840 +5334 +9151 +4868 +8226 +8393 +9587 +2898 +8333 +5038 +5984 +2318 +1641 +2722 +9182 +6746 +2616 +9830 +7958 +5863 +6546 +2196 +7330 +9125 +5951 +6992 +4708 +8860 +5031 +6554 +1186 +3146 +6763 +9876 +7521 +7478 +4486 +1952 +1549 +7343 +7701 +1568 +3623 +3132 +5455 +1285 +1373 +9358 +4462 +8013 +7851 +5623 +3288 +7017 +3750 +2650 +8240 +3313 +3162 +6368 +7487 +3965 +6003 +9244 +4795 +8323 +5544 +4634 +5978 +1909 +4517 +2479 +5539 +9036 +4120 +3624 +5554 +1441 +3684 +6334 +9218 +8640 +7654 +3585 +2241 +3158 +8600 +8520 +8789 +9700 +1112 +2350 +3089 +5762 +4917 +4032 +8449 +4997 +6246 +7455 +2632 +9181 +9437 +4788 +8397 +8893 +7981 +5285 +6986 +5905 +4622 +4999 +5328 +7100 +9238 +4784 +7357 +4644 +7342 +8016 +8359 +9454 +1970 +7673 +9252 +9155 +6518 +4097 +9280 +5262 +2690 +6498 +4225 +7023 +8744 +1101 +4923 +5656 +4705 +2877 +5697 +9145 +3673 +2439 +8254 +7443 +5153 +7124 +9565 +4346 +8912 +4063 +2411 +6709 +7839 +4726 +1402 +2339 +8651 +6222 +8599 +9678 +4192 +2762 +4909 +1540 +2649 +4478 +6038 +9142 +1850 +7041 +2236 +3555 +3254 +3240 +1010 +4109 +9368 +4213 +8030 +4253 +1857 +4688 +9538 +3909 +5077 +5885 +3321 +1685 +7796 +6397 +8924 +2819 +2368 +5610 +1316 +9105 +4322 +6720 +2934 +2647 +8802 +8534 +8915 +4910 +5532 +3538 +7442 +3019 +7216 +4409 +7660 +6116 +7317 +9799 +8401 +7543 +8928 +5737 +1515 +3270 +6312 +5772 +3164 +7302 +4597 +6304 +3424 +1400 +2087 +1759 +2996 +3927 +5158 +3049 +2788 +4435 +2476 +5690 +8496 +4206 +9827 +1956 +9264 +5929 +4762 +8436 +6036 +3404 +7850 +7580 +5114 +8124 +6540 +3251 +9091 +8890 +5881 +1389 +8167 +8627 +9012 +4087 +7934 +7681 +5297 +5750 +1717 +6729 +1420 +7402 +9829 +3831 +7427 +2987 +9341 +2057 +8399 +3850 +2929 +6394 +2374 +4722 +2562 +1144 +3922 +7935 +9612 +5973 +5678 +2400 +5668 +7176 +3326 +8476 +3012 +4313 +7527 +3418 +3565 +3894 +4411 +9315 +7311 +7927 +1386 +8757 +2112 +3165 +9789 +4034 +3589 +9208 +9639 +3430 +5251 +4239 +4914 +3129 +2303 +1056 +7672 +4663 +7323 +5133 +1852 +1058 +9180 +2336 +8883 +3631 +2891 +6559 +7230 +4003 +8164 +5993 +3054 +5345 +2355 diff --git a/res/datasets/fake/pheno/cad_test.h5ad.var.csv b/res/datasets/fake/pheno/cad_test.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/pheno/cad_test.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/pheno/cad_valid.h5ad.X.npy b/res/datasets/fake/pheno/cad_valid.h5ad.X.npy new file mode 100644 index 0000000..83d64a5 Binary files /dev/null and b/res/datasets/fake/pheno/cad_valid.h5ad.X.npy differ diff --git a/res/datasets/fake/pheno/cad_valid.h5ad.obs.csv b/res/datasets/fake/pheno/cad_valid.h5ad.obs.csv new file mode 100644 index 0000000..4bb8ba4 --- /dev/null +++ b/res/datasets/fake/pheno/cad_valid.h5ad.obs.csv @@ -0,0 +1,1025 @@ +eid +5745 +2599 +9855 +8508 +4833 +7426 +3620 +9959 +5519 +8405 +8382 +6742 +8997 +8958 +4400 +2180 +2839 +1476 +5600 +8053 +9066 +2685 +5705 +9410 +7349 +7616 +2134 +6071 +5055 +6073 +6617 +8227 +2733 +3378 +3863 +4353 +8138 +2391 +3217 +2380 +2955 +8736 +5124 +3144 +3794 +6361 +9657 +7898 +9256 +7226 +3269 +3643 +5102 +1697 +2115 +3202 +7944 +1383 +3384 +1821 +9504 +3607 +8403 +2334 +4797 +5079 +8557 +5597 +1182 +8857 +4787 +6612 +8375 +3870 +5352 +6425 +8481 +1822 +9172 +3507 +1658 +8223 +6259 +9331 +4973 +7268 +9116 +3050 +3350 +6982 +4927 +1558 +7609 +3388 +4158 +2323 +8696 +9783 +8085 +8622 +2875 +8842 +6846 +4838 +2443 +5026 +1447 +4835 +9560 +4766 +6962 +4197 +2208 +1208 +2507 +5199 +1814 +8356 +7903 +6399 +3885 +7915 +9281 +2706 +3751 +6586 +1306 +7945 +9564 +8854 +2055 +1830 +4800 +7456 +8613 +6752 +5818 +4165 +8242 +7778 +4744 +2515 +8462 +1028 +6191 +9154 +5436 +2272 +5708 +5461 +2345 +7606 +1464 +7149 +8437 +2263 +5323 +3170 +6354 +2653 +8674 +6570 +2827 +8870 +5520 +7724 +8966 +4930 +9034 +6179 +3491 +1219 +4494 +9879 +1214 +7309 +1634 +9638 +3236 +1793 +5806 +2893 +5962 +1645 +6284 +2042 +4038 +9433 +1624 +9174 +7501 +7207 +3554 +6669 +1356 +3304 +7786 +1628 +2184 +6428 +4789 +8114 +7336 +5127 +2416 +5955 +2850 +8208 +3150 +2862 +1470 +8532 +1609 +3306 +4763 +6637 +8233 +6565 +1412 +5427 +4653 +7599 +8343 +9010 +6822 +8699 +1355 +4984 +4971 +9673 +8203 +3062 +9713 +6074 +4310 +8498 +9325 +1805 +1929 +9364 +8918 +2423 +3526 +1539 +6331 +4533 +2870 +3543 +3433 +9821 +6938 +5452 +3710 +5471 +8930 +1273 +3785 +5542 +7714 +3041 +1569 +4754 +6024 +3819 +5689 +9822 +2809 +8406 +5524 +8876 +8105 +1169 +2143 +4422 +6862 +8320 +4040 +2335 +5830 +8491 +6015 +6750 +8488 +1797 +2965 +3522 +1235 +1578 +5279 +6798 +8452 +7919 +7699 +3016 +7328 +6661 +6062 +6441 +2872 +2294 +7392 +1701 +3209 +1948 +9917 +4975 +9117 +7305 +7464 +5643 +4758 +8867 +8008 +2904 +8582 +7789 +9023 +8594 +4357 +7727 +9085 +9255 +1906 +1533 +4261 +8270 +9398 +8243 +8949 +5563 +4769 +8886 +6295 +9065 +6634 +1017 +5515 +6373 +7843 +2144 +8956 +8661 +6678 +2004 +9871 +3732 +3421 +9297 +1587 +3245 +6418 +6497 +4248 +9642 +1812 +3596 +9707 +6753 +2262 +6740 +5712 +9057 +7953 +4524 +8007 +9359 +2811 +2414 +9561 +6387 +3301 +4849 +9386 +7757 +1676 +1378 +5022 +5592 +5029 +2984 +2712 +6904 +7001 +3045 +6837 +3151 +7499 +8395 +1935 +5820 +6820 +6810 +4749 +5648 +8602 +2591 +5624 +5419 +2174 +7615 +1071 +3009 +5855 +7772 +7855 +2361 +7534 +1143 +8478 +6668 +4052 +2863 +2702 +1354 +6157 +7137 +4113 +6876 +9791 +6275 +2253 +1596 +8755 +9893 +6061 +7258 +1998 +5617 +2149 +7419 +2964 +6470 +8885 +7287 +4405 +5433 +1367 +3253 +2365 +4821 +9338 +2081 +2481 +6956 +5370 +2747 +1119 +2286 +8361 +4088 +4605 +5760 +8337 +9840 +5193 +3376 +4549 +3011 +6640 +5071 +9458 +8321 +3991 +7657 +8001 +2843 +2501 +6208 +1659 +1768 +4612 +7834 +4748 +4100 +2672 +4895 +2219 +8934 +4145 +2938 +1928 +9905 +1431 +1152 +6272 +5263 +4068 +5016 +4543 +5438 +9211 +1767 +7879 +7683 +8880 +6288 +6404 +7828 +5405 +9055 +3349 +4778 +3177 +8616 +5622 +8071 +3532 +6178 +5035 +5140 +1478 +7753 +3026 +4939 +1307 +1451 +2136 +4535 +4727 +8597 +1244 +6745 +2737 +8538 +3305 +9420 +7801 +7397 +5105 +2273 +1480 +3345 +8275 +4831 +7151 +1009 +3091 +1736 +4915 +4443 +7885 +5752 +1667 +4624 +7595 +5278 +4832 +2485 +6996 +7875 +8929 +1395 +8012 +3830 +3256 +9047 +6082 +5805 +8947 +5665 +9641 +6722 +5996 +1323 +4513 +1556 +6838 +1156 +6747 +5421 +7192 +2528 +2203 +9925 +9687 +1350 +6845 +9106 +9056 +4785 +9333 +7389 +4089 +7139 +3734 +5359 +8746 +3583 +2261 +5675 +4659 +9414 +3473 +7588 +7408 +8412 +8229 +1897 +9814 +9247 +8186 +7288 +9351 +7756 +7922 +5372 +1255 +6881 +8301 +7582 +8423 +1238 +8786 +9413 +4551 +8236 +5344 +3621 +2830 +9574 +9863 +2269 +9496 +4619 +7260 +1475 +5699 +2142 +3389 +7711 +6718 +7762 +3442 +1081 +2656 +4406 +9198 +4707 +5646 +3480 +2894 +4620 +6199 +8339 +6814 +8598 +3443 +9765 +5990 +6132 +4715 +7372 +2033 +5959 +7656 +5787 +4046 +6734 +7760 +5210 +2806 +3539 +9940 +6473 +7685 +5173 +2661 +3364 +2731 +6558 +2936 +4683 +6352 +6755 +5429 +9390 +2710 +5891 +2066 +2644 +8681 +2227 +1314 +6323 +6088 +9597 +1550 +5748 +7179 +7937 +8365 +6972 +4096 +1564 +7610 +8165 +7414 +7441 +7236 +8490 +7794 +7270 +2342 +6844 +3968 +6154 +2919 +9421 +1760 +5321 +4621 +6283 +5017 +9520 +1588 +1444 +2040 +9175 +1869 +2859 +3685 +9861 +2579 +5116 +3069 +6067 +1798 +5687 +5800 +1967 +1502 +2105 +5366 +2486 +3141 +1600 +3071 +6611 +6853 +6796 +2242 +7438 +2135 +3148 +7165 +8566 +7697 +9943 +8038 +1246 +5609 +4014 +7355 +9594 +4649 +3046 +3548 +9536 +9361 +7722 +1281 +3848 +3628 +9823 +6311 +8100 +8137 +2971 +7663 +7866 +8289 +8752 +3948 +9656 +8456 +7338 +4056 +5383 +4416 +1592 +4209 +6575 +2384 +1818 +6887 +7158 +2206 +6804 +4656 +4986 +2088 +3546 +3371 +6472 +4333 +5233 +2065 +1521 +7425 +5275 +8075 +9762 +9550 +7299 +2270 +8319 +7678 +7573 +2657 +8484 +8009 +7666 +6847 +3219 +3590 +5919 +1221 +1577 +8810 +6396 +3741 +5272 +6917 +8467 +1804 +4149 +9551 +1041 +7474 +8188 +9476 +9467 +8816 +2412 +4956 +9964 +4729 +1526 +9319 +1066 +9069 +5727 +7191 +9292 +4201 +3788 +5249 +3678 +9189 +6333 +6525 +4315 +6251 +3644 +6225 +9891 +9637 +3542 +5736 +8831 +5087 +7346 +1606 +4667 +6151 +1027 +6221 +4360 +6037 +5593 +3225 +4931 +6253 +9453 +2233 +3292 +3208 +9562 +1340 +6398 +7421 +4001 +3859 +9212 +9844 +2360 +3657 +2775 +7278 +3201 +5950 +2246 +7873 +2931 +2695 +7718 +8552 +7241 +6084 +5607 +5294 +4241 +9693 +9977 +7518 +6348 +4267 +5898 +8578 +7251 +6713 +1829 +3661 +2447 +4679 +5463 +6146 +9449 +6087 +9329 +6381 +8194 +1230 +6508 +7218 +9883 +9107 +2363 +2314 +1792 +5400 +9653 +6819 +8698 +9794 +8450 +8835 +9524 +9634 +8127 +2299 +7700 +8614 +4185 +8576 +6220 +1227 +9213 +4685 +2445 +3405 +3407 +3887 +7163 +1035 +4642 +5912 +8776 +9326 +3735 +5743 +6013 +1111 +3244 +3073 +5414 +8643 +1537 +8251 +6577 +5884 +4301 +8187 +3079 +3029 +9044 +3824 +9287 +1649 +2896 +6491 +5395 +2021 +9582 +4539 +5411 +6217 +9185 +3458 +1741 +2429 +7577 +4862 +3890 +1291 +3097 +2140 +7897 +5143 +5432 +1743 +4161 +3103 +9714 +7847 +4057 +2715 +2494 +9088 +2807 +6016 +2422 +2532 +7878 +8444 +2353 +8801 +4335 +6350 +2012 +1161 +8117 +5466 +4536 diff --git a/res/datasets/fake/pheno/cad_valid.h5ad.var.csv b/res/datasets/fake/pheno/cad_valid.h5ad.var.csv new file mode 100644 index 0000000..8b822d3 --- /dev/null +++ b/res/datasets/fake/pheno/cad_valid.h5ad.var.csv @@ -0,0 +1,2 @@ +Field,BasedOn,ValueType,Explanation,Transformation +CAD,"40001-0.0, 40001-1.0, 41271-0.0, 41271-0.1, 41271-0.2, 41271-0.3, 41271-0.4, 41271-0.5, 41271-0.6, 41271-0.7, 41271-0.8, 41271-0.9, 41271-0.10, 41271-0.11, 41271-0.12, 41271-0.13, 41271-0.14, 41271-0.15, 41271-0.16, 41271-0.17, 41271-0.18, 41271-0.19, 41271-0.20, 41271-0.21, 41271-0.22, 41271-0.23, 41271-0.24, 41271-0.25, 41271-0.26, 41271-0.27, 41271-0.28, 41271-0.29, 41271-0.30, 41271-0.31, 41271-0.32, 41271-0.33, 41271-0.34, 41271-0.35, 41271-0.36, 41271-0.37, 41271-0.38, 41271-0.39, 41271-0.40, 41271-0.41, 41271-0.42, 41271-0.43, 41271-0.44, 41271-0.45, 41271-0.46, 41270-0.0, 41270-0.1, 41270-0.2, 41270-0.3, 41270-0.4, 41270-0.5, 41270-0.6, 41270-0.7, 41270-0.8, 41270-0.9, 41270-0.10, 41270-0.11, 41270-0.12, 41270-0.13, 41270-0.14, 41270-0.15, 41270-0.16, 41270-0.17, 41270-0.18, 41270-0.19, 41270-0.20, 41270-0.21, 41270-0.22, 41270-0.23, 41270-0.24, 41270-0.25, 41270-0.26, 41270-0.27, 41270-0.28, 41270-0.29, 41270-0.30, 41270-0.31, 41270-0.32, 41270-0.33, 41270-0.34, 41270-0.35, 41270-0.36, 41270-0.37, 41270-0.38, 41270-0.39, 41270-0.40, 41270-0.41, 41270-0.42, 41270-0.43, 41270-0.44, 41270-0.45, 41270-0.46, 41270-0.47, 41270-0.48, 41270-0.49, 41270-0.50, 41270-0.51, 41270-0.52, 41270-0.53, 41270-0.54, 41270-0.55, 41270-0.56, 41270-0.57, 41270-0.58, 41270-0.59, 41270-0.60, 41270-0.61, 41270-0.62, 41270-0.63, 41270-0.64, 41270-0.65, 41270-0.66, 41270-0.67, 41270-0.68, 41270-0.69, 41270-0.70, 41270-0.71, 41270-0.72, 41270-0.73, 41270-0.74, 41270-0.75, 41270-0.76, 41270-0.77, 41270-0.78, 41270-0.79, 41270-0.80, 41270-0.81, 41270-0.82, 41270-0.83, 41270-0.84, 41270-0.85, 41270-0.86, 41270-0.87, 41270-0.88, 41270-0.89, 41270-0.90, 41270-0.91, 41270-0.92, 41270-0.93, 41270-0.94, 41270-0.95, 41270-0.96, 41270-0.97, 41270-0.98, 41270-0.99, 41270-0.100, 41270-0.101, 41270-0.102, 41270-0.103, 41270-0.104, 41270-0.105, 41270-0.106, 41270-0.107, 41270-0.108, 41270-0.109, 41270-0.110, 41270-0.111, 41270-0.112, 41270-0.113, 41270-0.114, 41270-0.115, 41270-0.116, 41270-0.117, 41270-0.118, 41270-0.119, 41270-0.120, 41270-0.121, 41270-0.122, 41270-0.123, 41270-0.124, 41270-0.125, 41270-0.126, 41270-0.127, 41270-0.128, 41270-0.129, 41270-0.130, 41270-0.131, 41270-0.132, 41270-0.133, 41270-0.134, 41270-0.135, 41270-0.136, 41270-0.137, 41270-0.138, 41270-0.139, 41270-0.140, 41270-0.141, 41270-0.142, 41270-0.143, 41270-0.144, 41270-0.145, 41270-0.146, 41270-0.147, 41270-0.148, 41270-0.149, 41270-0.150, 41270-0.151, 41270-0.152, 41270-0.153, 41270-0.154, 41270-0.155, 41270-0.156, 41270-0.157, 41270-0.158, 41270-0.159, 41270-0.160, 41270-0.161, 41270-0.162, 41270-0.163, 41270-0.164, 41270-0.165, 41270-0.166, 41270-0.167, 41270-0.168, 41270-0.169, 41270-0.170, 41270-0.171, 41270-0.172, 41270-0.173, 41270-0.174, 41270-0.175, 41270-0.176, 41270-0.177, 41270-0.178, 41270-0.179, 41270-0.180, 41270-0.181, 41270-0.182, 41270-0.183, 41270-0.184, 41270-0.185, 41270-0.186, 41270-0.187, 41270-0.188, 41270-0.189, 41270-0.190, 41270-0.191, 41270-0.192, 41270-0.193, 41270-0.194, 41270-0.195, 41270-0.196, 41270-0.197, 41270-0.198, 41270-0.199, 41270-0.200, 41270-0.201, 41270-0.202, 41270-0.203, 41270-0.204, 41270-0.205, 41270-0.206, 41270-0.207, 41270-0.208, 41270-0.209, 41270-0.210, 41270-0.211, 41270-0.212, 41272-0.0, 41272-0.1, 41272-0.2, 41272-0.3, 41272-0.4, 41272-0.5, 41272-0.6, 41272-0.7, 41272-0.8, 41272-0.9, 41272-0.10, 41272-0.11, 41272-0.12, 41272-0.13, 41272-0.14, 41272-0.15, 41272-0.16, 41272-0.17, 41272-0.18, 41272-0.19, 41272-0.20, 41272-0.21, 41272-0.22, 41272-0.23, 41272-0.24, 41272-0.25, 41272-0.26, 41272-0.27, 41272-0.28, 41272-0.29, 41272-0.30, 41272-0.31, 41272-0.32, 41272-0.33, 41272-0.34, 41272-0.35, 41272-0.36, 41272-0.37, 41272-0.38, 41272-0.39, 41272-0.40, 41272-0.41, 41272-0.42, 41272-0.43, 41272-0.44, 41272-0.45, 41272-0.46, 41272-0.47, 41272-0.48, 41272-0.49, 41272-0.50, 41272-0.51, 41272-0.52, 41272-0.53, 41272-0.54, 41272-0.55, 41272-0.56, 41272-0.57, 41272-0.58, 41272-0.59, 41272-0.60, 41272-0.61, 41272-0.62, 41272-0.63, 41272-0.64, 41272-0.65, 41272-0.66, 41272-0.67, 41272-0.68, 41272-0.69, 41272-0.70, 41272-0.71, 41272-0.72, 41272-0.73, 41272-0.74, 41272-0.75, 41272-0.76, 41272-0.77, 41272-0.78, 41272-0.79, 41272-0.80, 41272-0.81, 41272-0.82, 41272-0.83, 41272-0.84, 41272-0.85, 41272-0.86, 41272-0.87, 41272-0.88, 41272-0.89, 41272-0.90, 41272-0.91, 41272-0.92, 41272-0.93, 41272-0.94, 41272-0.95, 41272-0.96, 41272-0.97, 41272-0.98, 41272-0.99, 41272-0.100, 41272-0.101, 41272-0.102, 41272-0.103, 41272-0.104, 41272-0.105, 41272-0.106, 41272-0.107, 41272-0.108, 41272-0.109, 41272-0.110, 41272-0.111, 41272-0.112, 41272-0.113, 41272-0.114, 41272-0.115, 41272-0.116, 20002-0.0, 20002-0.1, 20002-0.2, 20002-0.3, 20002-0.4, 20002-0.5, 20002-0.6, 20002-0.7, 20002-0.8, 20002-0.9, 20002-0.10, 20002-0.11, 20002-0.12, 20002-0.13, 20002-0.14, 20002-0.15, 20002-0.16, 20002-0.17, 20002-0.18, 20002-0.19, 20002-0.20, 20002-0.21, 20002-0.22, 20002-0.23, 20002-0.24, 20002-0.25, 20002-0.26, 20002-0.27, 20002-0.28, 20002-0.29, 20002-0.30, 20002-0.31, 20002-0.32, 20002-0.33, 20002-1.0, 20002-1.1, 20002-1.2, 20002-1.3, 20002-1.4, 20002-1.5, 20002-1.6, 20002-1.7, 20002-1.8, 20002-1.9, 20002-1.10, 20002-1.11, 20002-1.12, 20002-1.13, 20002-1.14, 20002-1.15, 20002-1.16, 20002-1.17, 20002-1.18, 20002-1.19, 20002-1.20, 20002-1.21, 20002-1.22, 20002-1.23, 20002-1.24, 20002-1.25, 20002-1.26, 20002-1.27, 20002-1.28, 20002-1.29, 20002-1.30, 20002-1.31, 20002-1.32, 20002-1.33, 20002-2.0, 20002-2.1, 20002-2.2, 20002-2.3, 20002-2.4, 20002-2.5, 20002-2.6, 20002-2.7, 20002-2.8, 20002-2.9, 20002-2.10, 20002-2.11, 20002-2.12, 20002-2.13, 20002-2.14, 20002-2.15, 20002-2.16, 20002-2.17, 20002-2.18, 20002-2.19, 20002-2.20, 20002-2.21, 20002-2.22, 20002-2.23, 20002-2.24, 20002-2.25, 20002-2.26, 20002-2.27, 20002-2.28, 20002-2.29, 20002-2.30, 20002-2.31, 20002-2.32, 20002-2.33, 20002-3.0, 20002-3.1, 20002-3.2, 20002-3.3, 20002-3.4, 20002-3.5, 20002-3.6, 20002-3.7, 20002-3.8, 20002-3.9, 20002-3.10, 20002-3.11, 20002-3.12, 20002-3.13, 20002-3.14, 20002-3.15, 20002-3.16, 20002-3.17, 20002-3.18, 20002-3.19, 20002-3.20, 20002-3.21, 20002-3.22, 20002-3.23, 20002-3.24, 20002-3.25, 20002-3.26, 20002-3.27, 20002-3.28, 20002-3.29, 20002-3.30, 20002-3.31, 20002-3.32, 20002-3.33, 20001-0.0, 20001-0.1, 20001-0.2, 20001-0.3, 20001-0.4, 20001-0.5, 20001-1.0, 20001-1.1, 20001-1.2, 20001-1.3, 20001-1.4, 20001-1.5, 20001-2.0, 20001-2.1, 20001-2.2, 20001-2.3, 20001-2.4, 20001-2.5, 20001-3.0, 20001-3.1, 20001-3.2, 20001-3.3, 20001-3.4, 20001-3.5",uint8,None, diff --git a/res/datasets/fake/tf_target_motif.csv b/res/datasets/fake/tf_target_motif.csv new file mode 100644 index 0000000..13f73de --- /dev/null +++ b/res/datasets/fake/tf_target_motif.csv @@ -0,0 +1,12 @@ +,Unnamed: 0,TF_name,TF_id,target_id,target_is_TF,motif_in_target,motif_in_tf,TF_is_TF,target_is_not_TF +0,0,TFAP2A,ENSG00000137203,ENSG00000001460,0,0,0,1,1 +1,1,TFAP2A,ENSG00000137203,ENSG00000001461,0,0,0,1,1 +2,2,TFAP2A,ENSG00000137203,ENSG00000001617,0,0,0,1,1 +3,3,TFAP2A,ENSG00000137203,ENSG00000002016,0,0,0,1,1 +4,4,TFAP2A,ENSG00000137203,ENSG00000002330,0,0,0,1,1 +5,5,TFAP2A,ENSG00000137203,ENSG00000002834,0,0,0,1,1 +6,6,TFAP2A,ENSG00000137203,ENSG00000003249,0,0,0,1,1 +7,7,TFAP2A,ENSG00000137203,ENSG00000003402,0,0,0,1,1 +8,8,TFAP2A,ENSG00000137203,ENSG00000003756,0,0,0,1,1 +9,9,TFAP2A,ENSG00000137203,ENSG00000004399,0,0,0,1,1 +10,10,TFAFAKE1,ENSG00000230368,ENSG00000162571,0,0,0,1,1 \ No newline at end of file diff --git a/scripts/verify_no_eids_in_history.py b/scripts/verify_no_eids_in_history.py new file mode 100644 index 0000000..463d144 --- /dev/null +++ b/scripts/verify_no_eids_in_history.py @@ -0,0 +1,346 @@ +"""Scan git history for eid-shaped numeric literals (6-8 digit numbers). + +Meant to be run before/after a history rewrite to confirm no UKB-eid-shaped +literals remain reachable from any ref. This is a heuristic scan for review, +not a formal guarantee: it filters out common false positives (dates, float +fragments, hex/hash strings) but surviving candidates should still be checked +manually. + +Every path is scanned by default, regardless of extension - use --exclude to +opt specific files/folders back out (e.g. known-noisy vendored assets). + +Two scan modes (--mode): + diff (default) walks `git log -p`, i.e. every commit's diff against its + parent(s). Fast, but inherently tied to git's diff-generation logic + (merge commits, rename detection, etc). + blobs reads every distinct blob object reachable from the given refs + directly via `git cat-file`, bypassing diff/log generation + entirely. Slower, but independent of any diff-engine blind spots - + useful as a cross-check against the diff mode. Binary (non-UTF-8) + blobs are skipped in this mode too; it only text-scans decodable + content. + +Usage: + python scripts/verify_no_eids_in_history.py + python scripts/verify_no_eids_in_history.py -- test/ res/datasets/fake/ + python scripts/verify_no_eids_in_history.py --known-fake fake_eids.txt + python scripts/verify_no_eids_in_history.py --exclude "poetry.lock" "docs/build/*" "*.svg" + python scripts/verify_no_eids_in_history.py --mode blobs +""" +import argparse +import fnmatch +import re +import subprocess +import sys +from collections import defaultdict + +DIGIT_RE = re.compile(r"(? bool: + if len(value) == 8: + mm, dd = value[4:6], value[6:8] + elif len(value) == 6: + mm, dd = value[2:4], value[4:6] + else: + return False + try: + m, d = int(mm), int(dd) + except ValueError: + return False + return 1 <= m <= 12 and 1 <= d <= 31 + + +def is_float_fragment(line: str, start: int, end: int) -> bool: + before = line[start - 1] if start > 0 else "" + after = line[end] if end < len(line) else "" + return before == "." or after == "." + + +def is_hex_fragment(line: str, start: int, end: int) -> bool: + before = line[start - 1] if start > 0 else "" + after = line[end] if end < len(line) else "" + if before in HEX_CHARS or after in HEX_CHARS: + return True + if start > 0 and line[start - 1] == "#": + return True + return False + + +def should_scan_path(path: str, exclude_patterns) -> bool: + basename = path.rsplit("/", 1)[-1] + return not any( + fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(basename, pattern) + for pattern in exclude_patterns + ) + + +def iter_diff_blocks(refs, pathspecs): + cmd = ["git", "log", *refs, "-p", "--"] + cmd += pathspecs if pathspecs else ["."] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return re.split(r"^diff --git ", result.stdout, flags=re.M)[1:] + + +# Ordered pipeline: each stage is (name, predicate). A candidate is dropped +# at the first stage whose predicate returns True (i.e. "this stage rejects it"). +STAGES = [ + ("known_fake", lambda value, line, start, end, known_fake: value in known_fake), + ("date", lambda value, line, start, end, known_fake: looks_like_date(value)), + ( + "float_fragment", + lambda value, line, start, end, known_fake: is_float_fragment(line, start, end), + ), + ( + "hex_fragment", + lambda value, line, start, end, known_fake: is_hex_fragment(line, start, end), + ), +] + + +HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@") + + +def new_stage_hits(): + # stage_hits[stage_name][value][path] = (line_no, line_content) of the first + # occurrence of `value` in `path` that was still alive *after* that stage ran. + # stage_hits["raw"] holds every match before any filtering; stage_hits[] holds the final survivors. + stage_names = ["raw"] + [name for name, _ in STAGES] + return {name: defaultdict(dict) for name in stage_names} + + +def record_match(stage_hits, path, line_no, line, value, start, end, known_fake): + record = (line_no, line) + + # Always record at the "raw" (unfiltered) stage first... + if path not in stage_hits["raw"][value]: + stage_hits["raw"][value][path] = record + + # ...then run the filter pipeline. Each stage's predicate returns True to + # REJECT the candidate; on rejection we stop immediately (so it's absent + # from this and all later stages). Otherwise we record it as a survivor + # of this stage and move on to the next filter. + for name, predicate in STAGES: + if predicate(value, line, start, end, known_fake): + break + if path not in stage_hits[name][value]: + stage_hits[name][value][path] = record + + +def scan(refs, pathspecs, known_fake, exclude_patterns): + stage_hits = new_stage_hits() + + # `git log --all -p` emits one unified diff per commit. Splitting on + # "diff --git " gives us one block per touched file per commit, each + # starting with its own "a/ b/" header. + for block in iter_diff_blocks(refs, pathspecs): + header = block[:200] + m = re.search(r"a/(\S+) b/(\S+)", header) + if not m: + continue + path = m.group(2) + if not should_scan_path(path, exclude_patterns): + continue # user-supplied --exclude pattern matched this path/basename + + # Track the line number in the *post-change* file by walking the diff's + # hunk headers ("@@ -a,b +c,d @@") and advancing new_line_no as we go: + # "+" lines exist only in the new file (count, then advance), "-" lines + # exist only in the old file (don't advance new_line_no), and unprefixed + # context lines exist in both (advance without recording a match). + new_line_no = None + for line in block.splitlines(): + hunk_m = HUNK_RE.match(line) + if hunk_m: + new_line_no = int(hunk_m.group(1)) + continue + if new_line_no is None: + continue # haven't hit a hunk header yet (still in the diff preamble) + + if line.startswith("+") and not line.startswith("+++"): + content = line[1:] # strip the diff "+" marker + for dm in DIGIT_RE.finditer(content): + record_match( + stage_hits, + path, + new_line_no, + content, + dm.group(), + dm.start(), + dm.end(), + known_fake, + ) + new_line_no += 1 + elif line.startswith("-") and not line.startswith("---"): + pass # removed line, doesn't exist in the new file, no line number to advance + else: + new_line_no += 1 # context line, present in both old and new + + return stage_hits + + +def blob_paths(refs, pathspecs): + # `git rev-list --objects` walks every reachable commit's *full tree* + # (not just diffs) and prints " " for every blob/tree found, + # deduplicated by sha. This is how we get one representative path per + # distinct blob content, independent of any diff/log machinery. + cmd = ["git", "rev-list", "--objects", *refs, "--"] + cmd += pathspecs if pathspecs else ["."] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + paths = {} + for line in result.stdout.splitlines(): + parts = line.split(" ", 1) + if len(parts) == 2: + sha, path = parts + paths.setdefault(sha, path) # keep first-seen path per blob + return paths + + +def filter_blob_shas(shas): + # rev-list --objects also lists commit/tree objects; keep blobs only. + if not shas: + return [] + result = subprocess.run( + ["git", "cat-file", "--batch-check=%(objectname) %(objecttype)"], + input="\n".join(shas) + "\n", + capture_output=True, + text=True, + check=True, + ) + return [ + line.split()[0] for line in result.stdout.splitlines() if line.endswith(" blob") + ] + + +def iter_blob_contents(shas): + # `git cat-file --batch` streams every requested object in one process: + # each is " \n\n". Reading this + # ourselves (rather than one `cat-file -p` subprocess per blob) is what + # makes scanning ~1000 blobs fast. + if not shas: + return + proc = subprocess.run( + ["git", "cat-file", "--batch"], + input=("\n".join(shas) + "\n").encode(), + capture_output=True, + check=True, + ) + data = proc.stdout + pos, n = 0, len(data) + while pos < n: + header_end = data.index(b"\n", pos) + parts = data[pos:header_end].decode().split() + if len(parts) != 3: + break + sha, _objtype, size = parts[0], parts[1], int(parts[2]) + content_start = header_end + 1 + yield sha, data[content_start : content_start + size] + pos = content_start + size + 1 + + +def scan_blobs(refs, pathspecs, known_fake, exclude_patterns): + stage_hits = new_stage_hits() + + paths = blob_paths(refs, pathspecs) + blob_shas = filter_blob_shas(sorted(paths)) + + for sha, content in iter_blob_contents(blob_shas): + path = paths.get(sha, sha) + if not should_scan_path(path, exclude_patterns): + continue + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + continue # binary blob; out of scope for text-based scanning + + for line_no, line in enumerate(text.splitlines(), start=1): + for dm in DIGIT_RE.finditer(line): + record_match( + stage_hits, + path, + line_no, + line, + dm.group(), + dm.start(), + dm.end(), + known_fake, + ) + + return stage_hits + + +def load_known_fake(path): + if path is None: + return set() + with open(path) as f: + return {line.strip() for line in f if line.strip()} + + +def print_candidates(hits): + for value in sorted(hits): + print(f" {value}") + for path in sorted(hits[value]): + line_no, content = hits[value][path] + print(f" {path}:{line_no}: {content.strip()}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("pathspecs", nargs="*", help="restrict scan to these paths") + parser.add_argument( + "--refs", nargs="*", default=["--all"], help="git log refs (default: --all)" + ) + parser.add_argument( + "--known-fake", help="file of newline-separated values to ignore" + ) + parser.add_argument( + "--exclude", + nargs="*", + default=[], + help="glob patterns (fnmatch-style, '*' allowed) matched against the full " + "path or basename; matching files are skipped, e.g. 'poetry.lock' " + "'docs/build/*' '*.svg'", + ) + parser.add_argument( + "--mode", + choices=["diff", "blobs"], + default="diff", + help="'diff' (default) walks git log -p; 'blobs' reads every reachable " + "blob object directly via git cat-file, bypassing diff/log " + "generation entirely - slower, but independent of diff-engine " + "blind spots (merge commits, rename detection, etc)", + ) + args = parser.parse_args() + + known_fake = load_known_fake(args.known_fake) + scan_fn = scan if args.mode == "diff" else scan_blobs + stage_hits = scan_fn(args.refs, args.pathspecs, known_fake, args.exclude) + final_stage = STAGES[-1][0] + hits = stage_hits[final_stage] + + print("Candidates remaining after each filter stage:") + print(f" raw (shape match, excludes applied): {len(stage_hits['raw'])}") + for name, _ in STAGES: + print(f" after {name}: {len(stage_hits[name])}") + print() + + print( + f"=== Stage 0: raw ({len(stage_hits['raw'])} distinct candidates, unfiltered) ===\n" + ) + print_candidates(stage_hits["raw"]) + print() + + if not hits: + print("No eid-shaped literals survive all filter stages.") + return 0 + + print( + f"=== Final stage: after {final_stage} ({len(hits)} distinct candidates) ===\n" + ) + print_candidates(hits) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/filters/__init__.py b/test/filters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/filters/composed_test.py b/test/filters/composed_test.py new file mode 100644 index 0000000..a28279c --- /dev/null +++ b/test/filters/composed_test.py @@ -0,0 +1,344 @@ +import pytest +import numpy as np +import torch + +from udm.filters.simple import IsIn, AnyIn, HasNan +from udm.filters.composed import And, Or, Not + + +class TestAnd(object): + @pytest.mark.parametrize( + "filters,error", + [ + ([], Exception), + ([IsIn([0, 1, 2])], None), + ([IsIn([0, 1, 2]), IsIn([3, 4, 5])], None), + ([IsIn([0, 1, 2]), IsIn([3, 4, 5]), IsIn([6, 7, 8])], None), + ([IsIn([0, 1, 2]), HasNan()], None), + ], + ) + def test__init__(self, filters, error): + if error: + with pytest.raises(error): + and_filter = And(filters) + else: + and_filter = And(filters) + + @pytest.mark.parametrize( + "filters,val,exp_result", + [ + ([IsIn([0, 1, 2])], [0, 3], torch.tensor([True, False], dtype=bool)), + ( + [IsIn([0, 1]), IsIn([0, 2])], + [0, 1, 2, 3], + torch.tensor([True, False, False, False], dtype=bool), + ), + ( + [AnyIn(torch.tensor([0, 1])), Not(HasNan())], + torch.tensor([0.0, np.nan, 2.0]).reshape(3, 1), + torch.tensor([True, False, False], dtype=bool), + ), + ], + ) + def test__call__(self, filters, val, exp_result): + and_filter = And(filters) + result = and_filter(val) + assert torch.all(result == exp_result) + + @pytest.mark.parametrize( + "filters,vals,exp_result", + [ + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + ], + torch.tensor([False, True, False, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([0.0, 0.0, 0.0, 0.0]).reshape(-1, 1), + ], + torch.tensor([False, True, True, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([1.0, 1.0, 1.0, 1.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, True]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([2.0, 2.0, 2.0, 2.0]).reshape(-1, 1), + ], + torch.tensor([True, True, False, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([3.0, 4.0, 5.0, 6.0]).reshape(-1, 1), + ], + torch.tensor([False, True, False, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), Not(HasNan())], + [ + torch.tensor( + [0.0, 0.0, 0.0, 2.0, 2.0, 2.0, np.nan, np.nan, np.nan] + ).reshape(-1, 1), + torch.tensor( + [1.0, 3.0, np.nan, 1.0, 3.0, np.nan, 1.0, 3.0, np.nan] + ).reshape(-1, 1), + ], + torch.tensor( + [True, True, False, True, False, False, False, False, False], + dtype=bool, + ), + ), + ], + ) + def test_compute(self, filters, vals, exp_result): + and_filter = And(filters) + for v in vals: + and_filter.update(v) + result = and_filter.compute() + assert torch.all(result == exp_result) + + +class TestOr(object): + @pytest.mark.parametrize( + "filters,error", + [ + ([], Exception), + ([IsIn([0, 1, 2])], None), + ([IsIn([0, 1, 2]), IsIn([3, 4, 5])], None), + ([IsIn([0, 1, 2]), IsIn([3, 4, 5]), IsIn([6, 7, 8])], None), + ([IsIn([0, 1, 2]), HasNan()], None), + ], + ) + def test__init__(self, filters, error): + if error: + with pytest.raises(error): + and_filter = Or(filters) + else: + and_filter = Or(filters) + + @pytest.mark.parametrize( + "filters,val,exp_result", + [ + ([IsIn([0, 1, 2])], [0, 3], torch.tensor([True, False], dtype=bool)), + ( + [IsIn([0, 1]), IsIn([0, 2])], + [0, 1, 2, 3], + torch.tensor([True, True, True, False], dtype=bool), + ), + ( + [AnyIn(torch.tensor([0, 1])), HasNan()], + torch.tensor([0.0, 2.0, np.nan]).reshape(3, 1), + torch.tensor([True, False, True], dtype=bool), + ), + ], + ) + def test__call__(self, filters, val, exp_result): + or_filter = Or(filters) + result = or_filter(val) + assert torch.all(result == exp_result) + + @pytest.mark.parametrize( + "filters,vals,exp_result", + [ + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([0.0, 0.0, 0.0, 0.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, True]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([1.0, 1.0, 1.0, 1.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, True]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([2.0, 2.0, 2.0, 2.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, True]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), AnyIn(torch.tensor([1.0, 2.0]))], + [ + torch.tensor([0.0, 1.0, 2.0, 3.0]).reshape(-1, 1), + torch.tensor([3.0, 4.0, 5.0, 6.0]).reshape(-1, 1), + ], + torch.tensor([True, True, True, False]), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), Not(HasNan())], + [ + torch.tensor([0.0, 0.0, 0.0]).reshape(-1, 1), + torch.tensor([1.0, 3.0, np.nan]).reshape(-1, 1), + ], + torch.tensor([True, True, True], dtype=bool), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), Not(HasNan())], + [ + torch.tensor([2.0, 2.0, 2.0]).reshape(-1, 1), + torch.tensor([1.0, 3.0, np.nan]).reshape(-1, 1), + ], + torch.tensor([True, True, False], dtype=bool), + ), + ( + [AnyIn(torch.tensor([0.0, 1.0])), Not(HasNan())], + [ + torch.tensor([np.nan, np.nan, np.nan]).reshape(-1, 1), + torch.tensor([1.0, 3.0, np.nan]).reshape(-1, 1), + ], + torch.tensor([True, False, False], dtype=bool), + ), + ], + ) + def test_compute(self, filters, vals, exp_result): + or_filter = Or(filters) + for v in vals: + or_filter.update(v) + result = or_filter.compute() + assert torch.all(result == exp_result) + + +class TestNot(object): + @pytest.mark.parametrize( + "filters,error", + [ + (None, Exception), + ([], Exception), + ([IsIn([0, 1, 2]), IsIn([0, 1, 2])], Exception), + (IsIn([0, 1, 2]), None), + (And([IsIn([0, 1, 2]), IsIn([3, 4, 5])]), None), + ], + ) + def test__init__(self, filters, error): + if error: + with pytest.raises(error): + not_filter = Not(filters) + else: + not_filter = Not(filters) + + @pytest.mark.parametrize( + "filters,val,exp_result", + [ + (IsIn([0, 1, 2]), [0, 3], torch.tensor([False, True], dtype=bool)), + ( + Not(IsIn([0, 1, 2])), + [0, 3], + torch.tensor([True, False], dtype=bool), + ), + ( + HasNan(), + torch.tensor([[0.0, 1.0], [np.nan, 2.0]]), + ~torch.tensor([False, True], dtype=bool), + ), + ( + And([IsIn([0, 1]), IsIn([0, 2])]), + [0, 1, 2, 3], + ~torch.tensor([True, False, False, False], dtype=bool), + ), + ], + ) + def test__call__(self, filters, val, exp_result): + not_filter = Not(filters) + result = not_filter(val) + assert torch.all(result == exp_result) + + @pytest.mark.parametrize( + "filters, vals, exp_result", + [ + ( + HasNan(), + [torch.tensor([[0.0, 1.0], [np.nan, 2.0]])], + ~torch.tensor([False, True], dtype=bool), + ), + ( + Not(HasNan()), + [torch.tensor([[0.0, 1.0], [np.nan, 2.0]])], + ~torch.tensor([True, False], dtype=bool), + ), + ( + HasNan(), + [ + torch.tensor( + [[0.0, 1.0], [0.0, 1.0], [np.nan, 2.0], [np.nan, 2.0]] + ), + torch.tensor( + [[1.0, 2.0], [3.0, np.nan], [1.0, 2.0], [3.0, np.nan]] + ), + ], + ~torch.tensor([False, True, True, True], dtype=bool), + ), + ( + Not(HasNan()), + [ + torch.tensor( + [[0.0, 1.0], [0.0, 1.0], [np.nan, 2.0], [np.nan, 2.0]] + ), + torch.tensor( + [[1.0, 2.0], [3.0, np.nan], [1.0, 2.0], [3.0, np.nan]] + ), + ], + ~torch.tensor([True, False, False, False], dtype=bool), + ), + ( + AnyIn(torch.tensor([1.0])), + [torch.tensor([[0.0], [1.0]])], + ~torch.tensor([False, True], dtype=bool), + ), + ( + Not(AnyIn(torch.tensor([1.0]))), + [torch.tensor([[0.0], [1.0]])], + ~torch.tensor([True, False], dtype=bool), + ), + ( + AnyIn(torch.tensor([1.0])), + [ + torch.tensor([0.0, 0.0, 1.0, 1.0]).reshape(-1, 1), + torch.tensor([2.0, 1.0, 2.0, 1.0]).reshape(-1, 1), + ], + ~torch.tensor([False, True, True, True]), + ), + ( + Not(AnyIn(torch.tensor([1.0]))), + [ + torch.tensor([0.0, 0.0, 1.0, 1.0]).reshape(-1, 1), + torch.tensor([2.0, 1.0, 2.0, 1.0]).reshape(-1, 1), + ], + ~torch.tensor([True, False, False, False]), + ), + ], + ) + def test_compute(self, filters, vals, exp_result): + not_filter = Not(filters) + for v in vals: + not_filter.update(v) + result = not_filter.compute() + assert torch.all(result == exp_result) diff --git a/test/filters/factory_test.py b/test/filters/factory_test.py new file mode 100644 index 0000000..e35e79e --- /dev/null +++ b/test/filters/factory_test.py @@ -0,0 +1,72 @@ +import pytest +import numpy as np +import torch +from omegaconf import DictConfig + +from udm.filters import get_filter + + +@pytest.fixture +def IsInCfg(): + return DictConfig({"name": "IsIn", "__init__": "__init__", "whitelist": [0, 1]}) + + +@pytest.fixture +def AnyInCfg(): + return DictConfig( + {"name": "AnyIn", "__init__": "from_sequence", "whitelist": [0, 1]} + ) + + +@pytest.fixture +def HasNanCfg(): + return DictConfig({"name": "HasNan", "__init__": "__init__"}) + + +@pytest.fixture +def AndCfg(): + return DictConfig( + { + "name": "And", + "__init__": "__init__", + "filters": [ + {"name": "IsIn", "__init__": "__init__", "whitelist": [0, 1]}, + {"name": "IsIn", "__init__": "__init__", "whitelist": [0, 2]}, + ], + } + ) + + +@pytest.mark.parametrize( + "config,custom_filters,val,exp_result", + [ + ( + pytest.lazy_fixture("IsInCfg"), + {}, + [0, 3], + torch.tensor([True, False], dtype=bool), + ), + ( + pytest.lazy_fixture("AnyInCfg"), + {}, + torch.tensor([[0.0, 1.0], [2.0, 3.0]]), + torch.tensor([True, False], dtype=bool), + ), + ( + pytest.lazy_fixture("HasNanCfg"), + {}, + torch.tensor([[0.0, 1.0], [np.nan, 2.0]]), + torch.tensor([False, True], dtype=bool), + ), + ( + pytest.lazy_fixture("AndCfg"), + {}, + [0, 1, 2, 3], + torch.tensor([True, False, False, False], dtype=bool), + ), + ], +) +def test_get_filter(config, custom_filters, val, exp_result): + filter = get_filter(config, custom=custom_filters) + result = filter(val) + assert torch.all(result == exp_result) diff --git a/test/filters/plugin_filters_test.py b/test/filters/plugin_filters_test.py new file mode 100644 index 0000000..94f0d53 --- /dev/null +++ b/test/filters/plugin_filters_test.py @@ -0,0 +1,532 @@ +import pytest +import numpy as np +import torch +from omegaconf import DictConfig, ListConfig + +from udm.filters.simple import IsIn, HasNan +from udm.filters.composed import Not +from udm.filters.plugin_filters import PluginRowFilter, PluginColFilter + + +@pytest.fixture +def PluginRowFilterCfg_key_val(): + return DictConfig( + { + "key_filter": {"name": "IsIn", "__init__": "__init__", "whitelist": [0, 1]}, + "val_filters": { + "data": { + "name": "Not", + "__init__": "__init__", + "filters": [{"name": "HasNan", "__init__": "__init__"}], + } + }, + } + ) + + +@pytest.fixture +def PluginRowFilterCfg_val(): + return DictConfig( + { + "key_filter": None, + "val_filters": { + "x": {"name": "HasNan", "__init__": "__init__"}, + "y": {"name": "HasNan", "__init__": "__init__"}, + }, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_key(): + return DictConfig( + { + "key_filters": { + "x": { + "name": "Not", + "filters": [ + {"name": "IsIn", "__init__": "__init__", "whitelist": [1]} + ], + }, + "y": { + "name": "Not", + "filters": [ + {"name": "IsIn", "__init__": "__init__", "whitelist": [2]}, + ], + }, + }, + "val_filters": {}, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_2d_key(PluginColFilterCfg_key): + PluginColFilterCfg_key.key_filters.x = ListConfig( + [ + { + "name": "Not", + "filters": [{"name": "IsIn", "__init__": "__init__", "whitelist": [1]}], + }, + None, + ] + ) + return PluginColFilterCfg_key + + +@pytest.fixture +def PluginColFilterCfg_val(): + return DictConfig( + { + "key_filters": {}, + "val_filters": { + "x": {"name": "HasNan", "__init__": "__init__"}, + "y": {"name": "HasNan", "__init__": "__init__"}, + }, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_2d_val(PluginColFilterCfg_val): + PluginColFilterCfg_val.val_filters.x = ListConfig( + [{"name": "HasNan", "__init__": "__init__"}, None] + ) + return PluginColFilterCfg_val + + +@pytest.fixture +def PluginColFilterCfg_key_val(): + return DictConfig( + { + "key_filters": { + "data": {"name": "IsIn", "__init__": "__init__", "whitelist": [0, 1]} + }, + "val_filters": { + "data": { + "name": "Not", + "__init__": "__init__", + "filters": [{"name": "HasNan", "__init__": "__init__"}], + } + }, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_key_multidim(): + return DictConfig( + { + "key_filters": { + "x": (None, {"name": "IsIn", "__init__": "__init__", "whitelist": [1]}), + "y": ( + { + "name": "Not", + "filters": [ + {"name": "IsIn", "__init__": "__init__", "whitelist": [0]} + ], + }, + { + "name": "Not", + "filters": [ + {"name": "IsIn", "__init__": "__init__", "whitelist": [2]} + ], + }, + ), + }, + "val_filters": {}, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_val_multidim(): + return DictConfig( + { + "key_filters": {}, + "val_filters": { + "x": ( + {"name": "HasNan", "__init__": "__init__"}, + { + "name": "Not", + "filters": [ + { + "name": "AnyIn", + "__init__": "from_sequence", + "whitelist": [1.0], + } + ], + }, + ), + "y": (None, {"name": "HasNan", "__init__": "__init__"}), + }, + } + ) + + +@pytest.fixture +def PluginColFilterCfg_key_val_multidim( + PluginColFilterCfg_key_multidim, PluginColFilterCfg_val_multidim +): + return DictConfig( + { + "key_filters": PluginColFilterCfg_key_multidim.key_filters, + "val_filters": PluginColFilterCfg_val_multidim.val_filters, + } + ) + + +class TestPluginRowFilter(object): + @pytest.mark.parametrize( + "key_filter,val_filters,error", + [ + (None, {}, Exception), + (IsIn([0, 1, 2]), {}, None), + (None, {"y": Not(HasNan())}, None), + (None, {"x": HasNan(), "y": Not(HasNan())}, None), + (IsIn([0, 1, 2]), {"x": HasNan(), "y": Not(HasNan())}, None), + ], + ) + def test__init__(self, key_filter, val_filters, error): + if error: + with pytest.raises(error): + plugin_filter = PluginRowFilter( + key_filter=key_filter, val_filters=val_filters + ) + else: + plugin_filter = PluginRowFilter( + key_filter=key_filter, val_filters=val_filters + ) + + @pytest.mark.parametrize( + "config,error", + [ + (pytest.lazy_fixture("PluginRowFilterCfg_key_val"), None), + (pytest.lazy_fixture("PluginRowFilterCfg_val"), None), + ], + ) + def test_from_config(self, config, error): + if error: + with pytest.raises(error): + plugin_filter = PluginRowFilter.from_config(config) + else: + plugin_filter = PluginRowFilter.from_config(config) + + @pytest.mark.parametrize( + "cfg_or_filters,eid,val,exp_result", + [ + ( + (IsIn([0, 1, 2]), {}), + [0, 3], + {"data": torch.tensor([0.0, 1.0])}, + torch.tensor([True, False], dtype=bool), + ), + ( + (None, {"data": Not(HasNan())}), + [0, 1], + {"data": torch.tensor([0.0, np.nan])}, + torch.tensor([True, False], dtype=bool), + ), + ( + (IsIn([0, 1]), {"data": Not(HasNan())}), + [0, 1, 2, 3], + {"data": torch.tensor([0.0, np.nan, 0.0, np.nan])}, + torch.tensor([True, False, False, False], dtype=bool), + ), + ( + (None, {"x": HasNan(), "y": HasNan()}), + [0, 1, 2, 3], + { + "x": torch.tensor([np.nan, np.nan, 0.0, 0.0]), + "y": torch.tensor([np.nan, 0.0, np.nan, 0.0]), + }, + torch.tensor([True, False, False, False], dtype=bool), + ), + ( + pytest.lazy_fixture("PluginRowFilterCfg_key_val"), + [0, 1, 2, 3], + {"data": torch.tensor([0.0, np.nan, 0.0, np.nan])}, + torch.tensor([True, False, False, False], dtype=bool), + ), + ( + pytest.lazy_fixture("PluginRowFilterCfg_val"), + [0, 1, 2, 3], + { + "x": torch.tensor([np.nan, np.nan, 0.0, 0.0]), + "y": torch.tensor([np.nan, 0.0, np.nan, 0.0]), + }, + torch.tensor([True, False, False, False], dtype=bool), + ), + ], + ) + def test__call__(self, cfg_or_filters, eid, val, exp_result): + if isinstance(cfg_or_filters, DictConfig): + plugin_filter = PluginRowFilter.from_config(cfg_or_filters) + else: + key_filter, val_filters = cfg_or_filters + plugin_filter = PluginRowFilter( + key_filter=key_filter, val_filters=val_filters + ) + result = plugin_filter(eid, val) + assert torch.all(result == exp_result) + + +class TestPluginColFilter(object): + # @pytest.mark.parametrize( + # "key_filter,val_filters,error", + # [ + # (None, {}, Exception), + # (IsIn([0, 1, 2]), {}, None), + # (None, {"y": Not(HasNan())}, None), + # (None, {"x": HasNan(), "y": Not(HasNan())}, None), + # (IsIn([0, 1, 2]), {"x": HasNan(), "y": Not(HasNan())}, None), + # ], + # ) + # def test__init__(self, key_filter, val_filters, error): + # if error: + # with pytest.raises(error): + # plugin_filter = PluginRowFilter( + # key_filter=key_filter, val_filters=val_filters + # ) + # else: + # plugin_filter = PluginRowFilter(key_filter=key_filter, val_filters=val_filters) + + # @pytest.mark.parametrize( + # "config,error", + # [ + # (pytest.lazy_fixture("PluginRowFilterCfg_key_val"), None), + # (pytest.lazy_fixture("PluginRowFilterCfg_val"), None), + # ], + # ) + # def test_from_config(self, config, error): + # if error: + # with pytest.raises(error): + # plugin_filter = PluginRowFilter.from_config(config) + # else: + # plugin_filter = PluginRowFilter.from_config(config) + + @pytest.mark.parametrize( + "cfg_or_filters,eid,vals,exp_result", + [ + ( + ({"data": IsIn([0, 1, 2])}, {}), + {"data": [0, 3]}, + [ + {"data": torch.tensor([[0.0, 1.0]])}, + {"data": torch.tensor([[3.0, 4.0]])}, + ], + {"data": (torch.tensor([True, False], dtype=bool),)}, + ), + ( + ({}, {"data": Not(HasNan())}), + {"data": [0, 1, 2, 3]}, + [ + {"data": torch.tensor([[0.0, 0.0, np.nan, np.nan]])}, + {"data": torch.tensor([[1.0, np.nan, 1.0, np.nan]])}, + ], + {"data": (torch.tensor([True, False, False, False], dtype=bool),)}, + ), + ( + ({"data": IsIn([0, 1])}, {"data": Not(HasNan())}), + {"data": [0, 1, 2, 3]}, + [ + {"data": torch.tensor([[0.0, 1.0, 2.0, 3.0]])}, + {"data": torch.tensor([[1.0, np.nan, 1.0, np.nan]])}, + ], + {"data": (torch.tensor([True, False, False, False], dtype=bool),)}, + ), + ( + ({"data": IsIn([0, 1])}, {"data": Not(HasNan())}), + {"data": [0, 1, 2, 3]}, + [ + {"data": torch.tensor([[np.nan, 1.0, np.nan, 3.0]])}, + {"data": torch.tensor([[1.0, np.nan, 1.0, np.nan]])}, + ], + {"data": (torch.tensor([False, False, False, False], dtype=bool),)}, + ), + ( + ({}, {"x": HasNan(), "y": HasNan()}), + { + "x": ["col0", "col1", "col2", "col3"], + "y": ["col4", "col5", "col6", "col7"], + }, + [ + { + "x": torch.tensor([[0.0, 1.0, 2.0, 3.0]]), + "y": torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + }, + { + "x": torch.tensor([[np.nan, np.nan, 0.0, 0.0]]), + "y": torch.tensor([[np.nan, 0.0, np.nan, 0.0]]), + }, + ], + { + "x": (torch.tensor([True, True, False, False], dtype=bool),), + "y": (torch.tensor([True, False, True, False], dtype=bool),), + }, + ), + ( + ({}, {"x": HasNan(), "y": HasNan()}), + { + "x": ["col0", "col1", "col2", "col3"], + "y": ["col4", "col5", "col6", "col7"], + }, + [ + { + "x": torch.tensor([[0.0, 1.0, np.nan, np.nan]]), + "y": torch.tensor([[1.0, np.nan, 3.0, np.nan]]), + }, + { + "x": torch.tensor([[np.nan, np.nan, 0.0, 0.0]]), + "y": torch.tensor([[np.nan, 0.0, np.nan, 0.0]]), + }, + ], + { + "x": (torch.tensor([True, True, True, True], dtype=bool),), + "y": (torch.tensor([True, True, True, True], dtype=bool),), + }, + ), + ( + pytest.lazy_fixture("PluginColFilterCfg_key_multidim"), + {"x": [[0, 1, 2], [0, 1]], "y": [[0, 1, 2], [2, 3]]}, + [ + { + "x": torch.tensor( + [ + [[0.0, np.nan], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [np.nan, 3.0], [4.0, 5.0]], + ] + ), + }, + { + "x": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + }, + ], + { + "x": ( + torch.tensor([True, True, True], dtype=bool), + torch.tensor([False, True], dtype=bool), + ), + "y": ( + torch.tensor([False, True, True], dtype=bool), + torch.tensor([False, True], dtype=bool), + ), + }, + ), + ( + pytest.lazy_fixture("PluginColFilterCfg_val_multidim"), + {"x": [[0, 1, 2], [0, 1]], "y": [[0, 1, 2], [2, 3]]}, + [ + { + "x": torch.tensor( + [ + [[0.0, np.nan], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [np.nan, 3.0], [4.0, 5.0]], + ] + ), + }, + { + "x": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + }, + ], + { + "x": ( + torch.tensor([True, False, False], dtype=bool), + torch.tensor([True, False], dtype=bool), + ), + "y": ( + torch.tensor([True, True, True], dtype=bool), + torch.tensor([True, False], dtype=bool), + ), + }, + ), + ( + pytest.lazy_fixture("PluginColFilterCfg_key_val_multidim"), + {"x": [[0, 1, 2], [0, 1]], "y": [[0, 1, 2], [2, 3]]}, + [ + { + "x": torch.tensor( + [ + [[0.0, np.nan], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [np.nan, 3.0], [4.0, 5.0]], + ] + ), + }, + { + "x": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + "y": torch.tensor( + [ + [[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]], + ] + ), + }, + ], + { + "x": ( + # torch.tensor([True, True, True], dtype=bool) & torch.tensor([True, False, False], dtype=bool) + torch.tensor([True, False, False], dtype=bool), + # torch.tensor([False, True], dtype=bool) & torch.tensor([True, False], dtype=bool) + torch.tensor([False, False], dtype=bool), + ), + "y": ( + # torch.tensor([False, True, True], dtype=bool) & torch.tensor([True, True, True], dtype=bool) + torch.tensor([False, True, True], dtype=bool), + # torch.tensor([False, True], dtype=bool) & torch.tensor([True, False], dtype=bool) + torch.tensor([False, False], dtype=bool), + ), + }, + ), + ], + ) + def test_compute(self, cfg_or_filters, eid, vals, exp_result): + if isinstance(cfg_or_filters, DictConfig): + plugin_filter = PluginColFilter.from_config(cfg_or_filters) + else: + key_filters, val_filters = cfg_or_filters + plugin_filter = PluginColFilter( + key_filters=key_filters, val_filters=val_filters + ) + plugin_filter.init_state(eid) + for v in vals: + plugin_filter.update(v) + result = plugin_filter.compute() + assert set(result.keys()) == set(exp_result.keys()) + for k in result.keys(): + assert isinstance(result[k], tuple) + assert len(result[k]) == len(exp_result[k]) + for dim_result, dim_exp_result in zip(result[k], exp_result[k]): + assert torch.all(dim_result == dim_exp_result) diff --git a/test/filters/simple_test.py b/test/filters/simple_test.py new file mode 100644 index 0000000..dba23e2 --- /dev/null +++ b/test/filters/simple_test.py @@ -0,0 +1,278 @@ +import pytest +import numpy as np +import torch +from typing import ( + Any, + Dict, + Iterable, +) + +from udm.filters.simple import Filter, StatefulFilter, IsIn, AnyIn, HasNan + + +class AnyOne(Filter, StatefulFilter): + def __init__(self): + super().__init__() + + def __call__(self, value): + return torch.any(value == 1, dim=1) + + def _update(self, input: Iterable[Any], state: Dict[str, Any]) -> Dict[str, Any]: + """Updates the filter state""" + mask = self.__call__(input) + if state["mask"] is None: + state["mask"] = mask + else: + state["mask"] = torch.logical_or(state["mask"], mask) + return state + + def _compute(self, state: Dict[str, Any]) -> torch.BoolTensor: + """Computes the mask using the state.""" + return state["mask"] + + def _init_state(self) -> Dict[str, Any]: + """Initializes the filter state""" + state = {"mask": None} + return state + + +class AnyTwo(Filter, StatefulFilter): + def __init__(self): + super().__init__() + + def __call__(self, value): + return torch.any(value == 2, dim=1) + + def _update(self, input: Iterable[Any], state: Dict[str, Any]) -> Dict[str, Any]: + """Updates the filter state""" + mask = self.__call__(input) + if state["mask"] is None: + state["mask"] = mask + else: + state["mask"] = torch.logical_or(state["mask"], mask) + return state + + def _compute(self, state: Dict[str, Any]) -> torch.BoolTensor: + """Computes the mask using the state.""" + return state["mask"] + + def _init_state(self) -> Dict[str, Any]: + """Initializes the filter state""" + state = {"mask": None} + return state + + +class TestIsIn(object): + @pytest.mark.parametrize("whitelist", [[], [0, 1, 2], ["a", "b", "c"]]) + def test__init__(self, whitelist): + isin_filter = IsIn(whitelist) + + @pytest.mark.parametrize( + "whitelist,eid,exp_result", + [ + ([], [0], torch.tensor([False], dtype=bool)), + ([0, 1, 2], [0], torch.tensor([True], dtype=bool)), + ([0, 1, 2], [3], torch.tensor([False], dtype=bool)), + ([0, 1, 2], [0, 3], torch.tensor([True, False], dtype=bool)), + ( + [0, 1, 2], + [0, 3, 1, 4, 2, 5], + torch.tensor([True, False, True, False, True, False], dtype=bool), + ), + ([], ["a"], torch.tensor([False], dtype=bool)), + (["a", "b", "c"], ["a"], torch.tensor([True], dtype=bool)), + (["a", "b", "c"], ["d"], torch.tensor([False], dtype=bool)), + (["a", "b", "c"], ["a", "d"], torch.tensor([True, False], dtype=bool)), + ( + ["a", "b", "c"], + ["a", "d", "b", "e", "c", "f"], + torch.tensor([True, False, True, False, True, False], dtype=bool), + ), + ], + ) + def test__call__(self, whitelist, eid, exp_result): + isin_filter = IsIn(whitelist) + result = isin_filter(eid) + assert torch.all(result == exp_result) + + +class TestAnyIn(object): + @pytest.mark.parametrize( + "whitelist,error", + [ + (torch.tensor([]), AssertionError), + (torch.tensor([0.0]), None), + (torch.tensor([0.0, 1.0, 2.0, 3.0]), None), + (torch.tensor([[0.0, 1.0], [2.0, 3.0]]), None), + ], + ) + def test__init__(self, whitelist, error): + if error: + with pytest.raises(error): + anyin_filter = AnyIn(whitelist) + else: + anyin_filter = AnyIn(whitelist) + + @pytest.mark.parametrize( + "whitelist,val,exp_result", + [ + ( + torch.tensor([0.0]), + torch.tensor([0.0]), + torch.tensor([True], dtype=bool), + ), + ( + torch.tensor([0.0]), + torch.tensor([1.0]), + torch.tensor([False], dtype=bool), + ), + ( + torch.tensor([0.0, 1.0]), + torch.tensor([0.0, 1.0, 2.0, 3.0]), + torch.tensor([True, True, False, False], dtype=bool), + ), + ( + torch.tensor([0.0, 1.0]), + torch.tensor([[0.0, 1.0], [2.0, 3.0]]), + torch.tensor([True, False], dtype=bool), + ), + ], + ) + def test__call__(self, whitelist, val, exp_result): + anyin_filter = AnyIn(whitelist) + result = anyin_filter(val) + assert torch.all(result == exp_result) + + @pytest.mark.parametrize( + "whitelist,vals,exp_result", + [ + ( + torch.tensor([0.0]), + [torch.tensor([0.0])], + torch.tensor([True], dtype=bool), + ), + ( + torch.tensor([0.0]), + [torch.tensor([1.0])], + torch.tensor([False], dtype=bool), + ), + ( + torch.tensor([0.0]), + [ + torch.tensor([0.0]), + torch.tensor([1.0]), + ], + torch.tensor([True], dtype=bool), + ), + ( + torch.tensor([0.0]), + [ + torch.tensor([1.0]), + torch.tensor([0.0]), + ], + torch.tensor([True], dtype=bool), + ), + ( + torch.tensor([0.0, 1.0]), + [torch.tensor([0.0, 1.0, 2.0, 3.0])], + torch.tensor([True, True, False, False], dtype=bool), + ), + ( + torch.tensor([0.0, 1.0]), + [torch.tensor([[0.0, 1.0], [2.0, 3.0]])], + torch.tensor([True, False], dtype=bool), + ), + ( + torch.tensor([0.0, 1.0]), + [ + torch.tensor( + [ + [1.0, 2.0], # True + [1.0, 2.0], # True + [2.0, 3.0], # False + [2.0, 3.0], # False + ] + ), + torch.tensor( + [ + [1.0, 2.0], # True + [3.0, 4.0], # False + [1.0, 2.0], # True + [3.0, 4.0], # False + ] + ), + ], + torch.tensor([True, True, True, False], dtype=bool), + ), + ], + ) + def test_compute(self, whitelist, vals, exp_result): + anyin_filter = AnyIn(whitelist) + for v in vals: + anyin_filter.update(v) + result = anyin_filter.compute() + assert torch.all(result == exp_result) + + +class TestHasNan(object): + def test__init__(self): + hasnan_filter = HasNan() + + @pytest.mark.parametrize( + "val,exp_result", + [ + # 1D tensor + (torch.tensor([0.0]), torch.tensor([False], dtype=bool)), + (torch.tensor([np.nan]), torch.tensor([True], dtype=bool)), + ( + torch.tensor([0.0, np.nan]), + torch.tensor([False, True], dtype=bool), + ), + # 2D tensor + (torch.tensor([[0.0]]), torch.tensor([False], dtype=bool)), + (torch.tensor([[np.nan]]), torch.tensor([True], dtype=bool)), + (torch.tensor([[0.0, np.nan]]), torch.tensor([True], dtype=bool)), + ( + torch.tensor([[0.0, 1.0], [np.nan, 2.0]]), + torch.tensor([False, True], dtype=bool), + ), + ], + ) + def test__call__(self, val, exp_result): + hasnan_filter = HasNan() + if isinstance(exp_result, Exception): + with pytest.raises(exp_result): + hasnan_filter(val) + else: + result = hasnan_filter(val) + assert torch.all(result == exp_result) + + @pytest.mark.parametrize( + "vals,exp_result", + [ + ([torch.tensor([[0.0]])], torch.tensor([False], dtype=bool)), + ([torch.tensor([[np.nan]])], torch.tensor([True], dtype=bool)), + ([torch.tensor([[0.0, np.nan]])], torch.tensor([True], dtype=bool)), + ( + [torch.tensor([[0.0, 1.0], [np.nan, 2.0]])], + torch.tensor([False, True], dtype=bool), + ), + ( + [ + torch.tensor( + [[0.0, 1.0], [0.0, 1.0], [np.nan, 2.0], [np.nan, 2.0]] + ), + torch.tensor( + [[2.0, 3.0], [3.0, np.nan], [2.0, 3.0], [3.0, np.nan]] + ), + ], + torch.tensor([False, True, True, True], dtype=bool), + ), + ], + ) + def test_compute(self, vals, exp_result): + hasnan_filter = HasNan() + for v in vals: + hasnan_filter.update(v) + result = hasnan_filter.compute() + assert torch.all(result == exp_result) diff --git a/test/general_datamodule_test.py b/test/general_datamodule_test.py new file mode 100644 index 0000000..0041382 --- /dev/null +++ b/test/general_datamodule_test.py @@ -0,0 +1,1342 @@ +import os +import pathlib +from copy import deepcopy +from inspect import isclass +from math import ceil +from os.path import abspath, dirname +from typing import Any +import yaml +from tempfile import TemporaryDirectory +import hydra +import numpy as np +import pandas as pd +import pytest +from pytest_lazyfixture import lazy_fixture +import torch +from omegaconf import DictConfig, OmegaConf + +from udm.general_datamodule import GeneralDatamodule +from udm.plugins.serialized.serialized_plugin import SerializedPlugin +from test.transforms.tensor_transforms.factory_test import ScaleTrafo +from test.general_dataset_test import assertBatchEqual +from test.utils import SKIPIF_NOT_BIH_HPC, try_clear_hydra +from test.plugins.dataplugin_test import MinimalPlugin +from test.plugins.genetic.genetic_plugin_test import ( + geno_debug_config, + mini_train_data, + prepare_mini_train as geno_prepare_mini_train, + prepare_mini_valid as geno_prepare_mini_valid, +) # noqa +from test.plugins.ehr.ehr_plugin_test import ehr_debug_config # noqa +from test.plugins.tabular.tabular_plugin_test import tabular_debug_config # noqa +from test.plugins.covariates.covariates_plugin_test import ( + covariates_debug_config, +) # noqa +from test.plugins.h5ad.h5ad_plugin_test import pheno_config # noqa + + +UDM_HOME = ".." +UDM_HOME_ABS = abspath(os.path.join(dirname(__file__), UDM_HOME)) + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def debug_eids(): + DEBUG_EIDS_PATH = os.path.join( + UDM_HOME, + "data/cvd/data/2_datasets_pre/211110_anewbeginning/artifacts/debug/not_anonymized_debug/eids.yaml", + ) + eids = yaml.load(DEBUG_EIDS_PATH, Loader=yaml.CSafeLoader) + return list(eids) + + +@pytest.fixture +def debug_eids_head(debug_eids): + return debug_eids[:100] + + +def _eid_split(eids, ratio): + n = int(ratio * len(eids)) + return eids[:n], eids[n:] + + +class CustomCollatePlugin(MinimalPlugin): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def collate_fn(self, batch) -> Any: + # Double every sample in the batch + x = batch[0]["data"] + doubled_batch = torch.concat([x, x], dim=1) + return {"data": doubled_batch} + + +@pytest.fixture +def scale_trafo_cfg(): + return DictConfig( + { + "name": "PluginTransform", + "transforms": { + "data": {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2} + }, + } + ) + + +@pytest.fixture +def composed_trafo_cfg(): + return DictConfig( + { + "name": "PluginTransform", + "transforms": { + "data": { + "name": "ListTransform", + "transforms": [ + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + ], + } + }, + } + ) + + +@pytest.fixture +def from_clsmethod_config(): + config = OmegaConf.create( + { + "plugins": { + "minimal": { + "name": "MinimalPlugin", + "__init__": "from_list", + "data": [[0], [1], [2]], + } + }, + "splits": {"train": [0], "valid": [1], "test": [2]}, + "batch_size": 1, + "shuffle": False, + } + ) + yield config + + +@pytest.fixture +def custom_collate_config(): + config = OmegaConf.create( + { + "plugins": { + "minimal": {"name": "MinimalPlugin"}, + "custom_collate": {"name": "CustomCollatePlugin"}, + }, + "splits": {"train": [0, 1], "valid": [2, 3], "test": [4, 5]}, + "batch_size": 2, + "shuffle": False, + } + ) + yield config + + +@pytest.fixture +def minimal_config(): + config = OmegaConf.create( + { + "plugins": { + "minimal": {"name": "MinimalPlugin", "transforms": None}, + }, + "splits": {"train": [0, 1], "valid": [2, 3], "test": [4, 5]}, + "transforms": {}, + "batch_size": 2, + "shuffle": False, + } + ) + yield config + + +@pytest.fixture +def minimal_str_config(): + config = OmegaConf.create( + { + "plugins": { + "minimal": { + "name": "MinimalPlugin", + "eids_type": "str", + "transforms": None, + }, + }, + "splits": { + "train": ["a1", "b2"], + "valid": ["c3", "d4"], + "test": ["e5", "f6"], + }, + "transforms": {}, + "batch_size": 2, + "shuffle": False, + } + ) + yield config + + +@pytest.fixture +def minimal_plg_trafo_config(minimal_config): + config = minimal_config + overrides_cfg = DictConfig( + { + "plugins": { + "minimal": { + # ... + "transforms": { + "name": "PluginTransform", + "transforms": { + "data": { + "name": "ScaleTrafo", + "__init__": "__init__", + "alpha": 2, + } + }, + } + } + } + } + ) + config = OmegaConf.merge(config, overrides_cfg) + yield config + + +def _minimal_ds_trafo_config(dataset_trafo="train"): + return DictConfig( + { + # ... + "transforms": { + # ... other splits/datasets + dataset_trafo: { + "name": "DatasetTransform", + "transforms": { + "minimal": { + "name": "PluginTransform", + "transforms": { + "data": { + "name": "ScaleTrafo", + "__init__": "__init__", + "alpha": 2, + } + }, + } + }, + } + } + } + ) + + +@pytest.fixture +def minimal_train_ds_trafo_config(minimal_config): + overrides_config = _minimal_ds_trafo_config(dataset_trafo="train") + config = OmegaConf.merge(minimal_config, overrides_config) + yield config + + +@pytest.fixture +def minimal_valid_ds_trafo_config(minimal_config): + overrides_config = _minimal_ds_trafo_config(dataset_trafo="valid") + config = OmegaConf.merge(minimal_config, overrides_config) + yield config + + +@pytest.fixture +def minimal_test_ds_trafo_config(minimal_config): + overrides_config = _minimal_ds_trafo_config(dataset_trafo="test") + config = OmegaConf.merge(minimal_config, overrides_config) + yield config + + +@pytest.fixture +def cov_config(covariates_debug_config): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[]) + config.plugins = {"cov": covariates_debug_config} + yield config + + +@pytest.fixture +def geno_config(geno_debug_config): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[]) + config.plugins = {"geno": geno_debug_config} + yield config + + +@pytest.fixture +def ehr_config(ehr_debug_config): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[]) + config.plugins = {"ehr": ehr_debug_config} + yield config + + +@pytest.fixture +def tabular_config(tabular_debug_config): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[]) + config.plugins = {"tabular": tabular_debug_config} + yield config + + +@pytest.fixture +def h5ad_config(pheno_config): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[]) + config.plugins = {"pheno": pheno_config} + yield config + + +@pytest.fixture +def geno_geno_mini_config(geno_prepare_mini_train, geno_prepare_mini_valid): + try_clear_hydra() + TMP_PATH1, TMP_PATH2 = geno_prepare_mini_train, geno_prepare_mini_valid + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + "default.yaml", + overrides=[ + "+plugins/genetic@plugins.geno1=default", + f"plugins.geno1.src=[{TMP_PATH1}]", + "plugins.geno1.src_names=[geno]", + "plugins.geno1.eid_map_path=null", + "+plugins/genetic@plugins.geno2=default", + f"plugins.geno2.src=[{TMP_PATH2}]", + "plugins.geno2.src_names=[geno]", + "plugins.geno2.eid_map_path=null", + "+splits={train: [0], valid: [1], test: [2]}", + "combine_eids_as=intersect", + "batch_size=1", + "keyerr=True", + ], + ) + yield config + + +@pytest.fixture +def geno_geno_mini_map_config(temp_dir, geno_geno_mini_config): + config = geno_geno_mini_config + with TemporaryDirectory(dir=temp_dir) as temp_dir: + config.plugins.geno2.eid_map_path = f"{temp_dir}/eid_map.csv" + config.plugins.geno2.eid_map_from = "eid_map_from" + config.plugins.geno2.eid_map_to = "eid_map_to" + eid_map = pd.DataFrame( + {"eid_map_from": [3, 4, 5, 6], "eid_map_to": [0, 1, 2, 6]} + ) + eid_map_path = f"{temp_dir}/eid_map.csv" + eid_map.to_csv(eid_map_path, sep="\t") + yield config + + +@pytest.fixture +def cov_cov_config(debug_eids_head): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=["plugins=cov_cov"]) + data_paths = os.path.join( + "data", + "cvd", + "data", + "2_datasets_pre", + "211110_anewbeginning", + "artifacts", + "debug", + "not_anonymized_debug", + "baseline_covariates_minimal_debug_20220711.feather", + ) + config.plugins.cov1.data_paths = data_paths + config.plugins.cov2.data_paths = data_paths + eids_train_valid, eids_test = _eid_split(debug_eids_head, 0.9) + eids_train, eids_valid = _eid_split(eids_train_valid, 0.9) + config.splits = {"train": eids_train, "valid": eids_valid, "test": eids_test} + yield config + + +@pytest.fixture +def cov_geno_config(debug_eids_head): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + "default.yaml", + overrides=[ + "+plugins/covariates@plugins.cov=default", + "+plugins/genetic@plugins.geno=debug", + ], + ) + # Fix covariate paths + config.plugins.cov.data_paths = os.path.join( + "data", + "cvd", + "data", + "2_datasets_pre", + "211110_anewbeginning", + "artifacts", + "debug", + "not_anonymized_debug", + "baseline_covariates_minimal_debug_20220711.feather", + ) + # Fix geno paths + FAKE_GENO_PATH = os.path.join(UDM_HOME_ABS, "res/datasets/fake/geno/") + config.plugins.geno.dataset_path = FAKE_GENO_PATH + eids_train_valid, eids_test = _eid_split(debug_eids_head, 0.9) + eids_train, eids_valid = _eid_split(eids_train_valid, 0.9) + config.splits = {"train": eids_train, "valid": eids_valid, "test": eids_test} + yield config + + +class TestGeneralDataModule(object): + @pytest.mark.parametrize( + "split1,split2,warning", + [ + ([], [], None), + ([0], [], None), + ([], [0], None), + ([0], [1], None), + ([0], [0], UserWarning), + ([0, 1], [1, 2], UserWarning), + ], + ) + def test_check_two_splits(self, split1, split2, warning): + split1 = np.array(split1, dtype=int) + split2 = np.array(split2, dtype=int) + if warning is None: + GeneralDatamodule._check_two_splits( + "split1", "split2", "train", "valid", split1, split2 + ) + else: + with pytest.warns(warning): + GeneralDatamodule._check_two_splits( + "split1", "split2", "train", "valid", split1, split2 + ) + + @pytest.mark.parametrize( + "config, overrides, custom_plugins, custom_transforms", + [ + ( + lazy_fixture("from_clsmethod_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + ), + (lazy_fixture("minimal_config"), {}, {"MinimalPlugin": MinimalPlugin}, {}), + ( + lazy_fixture("minimal_str_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + ), + ( + lazy_fixture("minimal_plg_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + ), + ( + lazy_fixture("minimal_train_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + ), + ( + lazy_fixture("minimal_valid_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + ), + ( + lazy_fixture("minimal_test_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + ), + pytest.param( + lazy_fixture("cov_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("geno_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("ehr_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("tabular_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("h5ad_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param(lazy_fixture("geno_geno_mini_config"), {}, {}, {}), + pytest.param(lazy_fixture("geno_geno_mini_map_config"), {}, {}, {}), + pytest.param( + lazy_fixture("geno_geno_mini_map_config"), + {"combine_eids_as": "union"}, + {}, + {}, + ), + pytest.param( + lazy_fixture("cov_cov_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("cov_geno_config"), {}, {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + ], + ) + def test__init__(self, config, overrides, custom_plugins, custom_transforms): + config.update(overrides) + gdm = GeneralDatamodule( + **config, + custom_plugins=custom_plugins, + custom_transforms=custom_transforms, + ) + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, overrides", + [ + (lazy_fixture("cov_config"), {}), + (lazy_fixture("geno_config"), {}), + (lazy_fixture("ehr_config"), {}), + (lazy_fixture("tabular_config"), {}), + (lazy_fixture("h5ad_config"), {}), + (lazy_fixture("geno_geno_mini_config"), {}), + (lazy_fixture("geno_geno_mini_map_config"), {}), + (lazy_fixture("geno_geno_mini_map_config"), {"combine_eids_as": "union"}), + (lazy_fixture("cov_cov_config"), {}), + (lazy_fixture("cov_geno_config"), {}), + ], + ) + def test_prepare_data(self, config, overrides): + config.update(overrides) + gdm = GeneralDatamodule(**config) + gdm.prepare_data() + + @staticmethod + def _run_setup( + config, custom_plugins={}, custom_transforms={}, update_splits=False + ): + gdm = GeneralDatamodule( + **config, custom_plugins=custom_plugins, custom_transforms=custom_transforms + ) + gdm.prepare_data() + gdm.setup() + if update_splits: + eids = gdm.get_eids() + train_valid_eids, test_eids = _eid_split(eids, 0.9) + train_eids, valid_eids = _eid_split(train_valid_eids, 0.9) + splits = {"train": train_eids, "valid": valid_eids, "test": test_eids} + gdm.update_splits(splits) + # NOTE: This requires pytorch_lightning >= 1.6 + gdm.prepare_data() + gdm.setup() + return gdm + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, overrides, update_splits, warning", + [ + (lazy_fixture("cov_config"), {}, True, None), + (lazy_fixture("geno_config"), {}, True, None), + (lazy_fixture("ehr_config"), {}, True, None), + (lazy_fixture("tabular_config"), {}, True, None), + (lazy_fixture("h5ad_config"), {}, True, None), + (lazy_fixture("geno_geno_mini_config"), {}, False, UserWarning), + (lazy_fixture("geno_geno_mini_map_config"), {}, False, UserWarning), + ( + lazy_fixture("geno_geno_mini_map_config"), + {"combine_eids_as": "union"}, + False, + None, + ), + (lazy_fixture("cov_cov_config"), {}, False, None), + (lazy_fixture("cov_geno_config"), {}, False, None), + ], + ) + def test_setup(self, config, overrides, update_splits, warning): + config.update(overrides) + gdm = TestGeneralDataModule._run_setup(config, update_splits=update_splits) + gdm.prepare_data() + if warning: + with pytest.warns(warning): + gdm.setup() + else: + gdm.setup() + del gdm + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, overrides, exp_eids", + [ + (lazy_fixture("minimal_config"), {}, [0, 1, 2, 3, 4, 5]), + ( + lazy_fixture("minimal_str_config"), + {}, + ["a1", "b2", "c3", "d4", "e5", "f6"], + ), + (lazy_fixture("geno_geno_mini_config"), {}, []), + (lazy_fixture("geno_geno_mini_map_config"), {}, [0, 1]), + ( + lazy_fixture("geno_geno_mini_map_config"), + {"combine_eids_as": "union"}, + [0, 1, 2, 6], + ), + ( + lazy_fixture("geno_geno_mini_config"), + {"combine_eids_as": "union"}, + [0, 1, 2, 3, 4, 6], + ), + ], + ) + def test_get_eids(self, config, overrides, exp_eids): + config.update(overrides) + gdm = GeneralDatamodule(**config) + gdm.prepare_data() + gdm.setup() + eids = gdm.get_eids() + assert eids == exp_eids + del gdm + + @pytest.mark.parametrize( + "config, overrides, custom_plugins, custom_transforms, shapes_only, update_splits, exp_len, exp_batch", + [ + ( + lazy_fixture("custom_collate_config"), + {}, + { + "MinimalPlugin": MinimalPlugin, + "CustomCollatePlugin": CustomCollatePlugin, + }, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 1]], dtype=torch.uint8) + }, + "custom_collate": { + "data": torch.tensor( + [[0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 1]], dtype=torch.uint8 + ) + }, + }, + ), + ( + lazy_fixture("minimal_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 1]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_str_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 1]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_plg_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 2]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_train_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 2]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_valid_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 1]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_test_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 0, 0], [0, 0, 1]], dtype=torch.uint8) + } + }, + ), + pytest.param( + lazy_fixture("cov_config"), + {}, + {}, + {}, + True, + True, + ceil(81 / 16), + {"cov": {"covariates": [16, 31]}}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.9**2 / 16), + {"geno": {"genetic": [16, 10]}}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("ehr_config"), + {}, + {}, + {}, + True, + True, + ceil(81 / 16), + { + "ehr": { + "records": [16, 6370], + "censorings": [16, 1], + "exclusions": [16, 1192], + "label_events": [16, 1192], + "label_times": [16, 1192], + "label_auxiliary_times": [16, 5178], + "label_auxiliary_events": [16, 5178], + } + }, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("tabular_config"), + {}, + {}, + {}, + True, + True, + ceil(81 / 16), + {"tabular": {"tabular_data": [16, 9]}}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("h5ad_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.9**2 / 16), + {"pheno": {"tabular_data": [16, 1]}}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_geno_mini_map_config"), + {}, + {}, + {}, + False, + False, + 1, + { + "geno1": { + "genetic": torch.tensor([0, 1, 0], dtype=torch.float16) / 127 + }, + "geno2": { + "genetic": torch.tensor([1, 0, 0], dtype=torch.float16) / 127 + }, + }, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_geno_mini_config"), + {"combine_eids_as": "union"}, + {}, + {}, + False, + False, + 1, + KeyError, + marks=[pytest.mark.xfail], + ), + pytest.param( + lazy_fixture("geno_geno_mini_config"), + {"combine_eids_as": "union", "keyerr": False}, + {}, + {}, + False, + False, + 1, + { + "geno1": { + "genetic": torch.tensor([0, 1, 0], dtype=torch.float16) / 127 + }, + "geno2": {}, + }, + ), + pytest.param( + lazy_fixture("cov_cov_config"), + {}, + {}, + {}, + True, + False, + ceil(81 / 16), + { + "cov1": {"covariates": [16, 31]}, + "cov2": {"covariates": [16, 31]}, + }, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("cov_geno_config"), + {}, + {}, + {}, + True, + False, + ceil(81 / 16), + { + "cov": {"covariates": [16, 31]}, + "geno": {"genetic": [16, 10]}, + }, + marks=SKIPIF_NOT_BIH_HPC, + ), + ], + ) + def test_train_dataloader( + self, + config, + overrides, + custom_plugins, + custom_transforms, + shapes_only, + update_splits, + exp_len, + exp_batch, + ): + config.update(overrides) + gdm = TestGeneralDataModule._run_setup( + config, + custom_plugins=custom_plugins, + custom_transforms=custom_transforms, + update_splits=update_splits, + ) + dataloader = gdm.train_dataloader(drop_last=False) + if isclass(exp_batch) and issubclass(exp_batch, Exception): + with pytest.raises(exp_batch): + iterator = iter(dataloader) + next(iterator) + else: + batch = deepcopy(next(iter(dataloader))) + assert len(dataloader) == exp_len + assertBatchEqual(batch, exp_batch, shapes_only=shapes_only) + del gdm, dataloader + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, overrides, custom_plugins, custom_transforms, shapes_only, update_splits, exp_len, exp_batch", + [ + ( + lazy_fixture("minimal_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 1, 0], [1, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_str_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 1, 0], [1, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_plg_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 2, 0], [2, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_train_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 1, 0], [1, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_valid_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 2, 0], [2, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_test_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[0, 1, 0], [1, 0, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("cov_config"), + {}, + {}, + {}, + True, + True, + 1, + {"cov": {"covariates": [9, 31]}}, + ), + ( + lazy_fixture("geno_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.9 * 0.1 / 16), + {"geno": {"genetic": [16, 10]}}, + ), + ( + lazy_fixture("ehr_config"), + {}, + {}, + {}, + True, + True, + 1, + { + "ehr": { + "records": [9, 6370], + "censorings": [9, 1], + "exclusions": [9, 1192], + "label_events": [9, 1192], + "label_times": [9, 1192], + "label_auxiliary_times": [9, 5178], + "label_auxiliary_events": [9, 5178], + } + }, + ), + ( + lazy_fixture("h5ad_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.9 * 0.1 / 16), + {"pheno": {"tabular_data": [16, 1]}}, + ), + ( + lazy_fixture("tabular_config"), + {}, + {}, + {}, + True, + True, + 1, + {"tabular": {"tabular_data": [9, 9]}}, + ), + ( + lazy_fixture("cov_cov_config"), + {}, + {}, + {}, + True, + False, + 1, + { + "cov1": {"covariates": [9, 31]}, + "cov2": {"covariates": [9, 31]}, + }, + ), + ( + lazy_fixture("cov_geno_config"), + {}, + {}, + {}, + True, + False, + 1, + { + "cov": {"covariates": [9, 31]}, + "geno": {"genetic": [9, 10]}, + }, + ), + ], + ) + def test_val_dataloader( + self, + config, + overrides, + custom_plugins, + custom_transforms, + shapes_only, + update_splits, + exp_len, + exp_batch, + ): + config.update(overrides) + gdm = TestGeneralDataModule._run_setup( + config, + custom_plugins=custom_plugins, + custom_transforms=custom_transforms, + update_splits=update_splits, + ) + dataloader = gdm.val_dataloader(drop_last=False) + batch = deepcopy(next(iter(dataloader))) + assert len(dataloader) == exp_len + assertBatchEqual(batch, exp_batch, shapes_only=shapes_only) + del gdm, dataloader + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, overrides, custom_plugins, custom_transforms, shapes_only, update_splits, exp_len, exp_batch", + [ + ( + lazy_fixture("minimal_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[1, 0, 1], [1, 1, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_str_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[1, 0, 1], [1, 1, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_plg_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[2, 0, 2], [2, 2, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_train_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[1, 0, 1], [1, 1, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_valid_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[1, 0, 1], [1, 1, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("minimal_test_ds_trafo_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + {"ScaleTrafo": ScaleTrafo}, + False, + False, + 1, + { + "minimal": { + "data": torch.tensor([[2, 0, 2], [2, 2, 0]], dtype=torch.uint8) + } + }, + ), + ( + lazy_fixture("cov_config"), + {}, + {}, + {}, + True, + True, + 1, + {"cov": {"covariates": [10, 31]}}, + ), + ( + lazy_fixture("geno_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.1 / 16), + {"geno": {"genetic": [16, 10]}}, + ), + ( + lazy_fixture("ehr_config"), + {}, + {}, + {}, + True, + True, + 1, + { + "ehr": { + "records": [10, 6370], + "censorings": [10, 1], + "exclusions": [10, 1192], + "label_events": [10, 1192], + "label_times": [10, 1192], + "label_auxiliary_times": [10, 5178], + "label_auxiliary_events": [10, 5178], + } + }, + ), + ( + lazy_fixture("tabular_config"), + {}, + {}, + {}, + True, + True, + 1, + {"tabular": {"tabular_data": [10, 9]}}, + ), + ( + lazy_fixture("h5ad_config"), + {}, + {}, + {}, + True, + True, + ceil(4096 * 0.1 / 16), + {"pheno": {"tabular_data": [16, 1]}}, + ), + ( + lazy_fixture("cov_cov_config"), + {}, + {}, + {}, + True, + False, + 1, + { + "cov1": {"covariates": [10, 31]}, + "cov2": {"covariates": [10, 31]}, + }, + ), + ( + lazy_fixture("cov_geno_config"), + {}, + {}, + {}, + True, + False, + 1, + { + "cov": {"covariates": [10, 31]}, + "geno": {"genetic": [10, 10]}, + }, + ), + ], + ) + def test_test_dataloader( + self, + config, + overrides, + custom_plugins, + custom_transforms, + shapes_only, + update_splits, + exp_len, + exp_batch, + ): + config.update(overrides) + gdm = TestGeneralDataModule._run_setup( + config, + custom_plugins=custom_plugins, + custom_transforms=custom_transforms, + update_splits=update_splits, + ) + dataloader = gdm.test_dataloader(drop_last=False) + batch = deepcopy(next(iter(dataloader))) + assert len(dataloader) == exp_len + assertBatchEqual(batch, exp_batch, shapes_only=shapes_only) + del gdm, dataloader + + @pytest.mark.parametrize( + "config, overrides, custom_plugins", + [ + ( + lazy_fixture("from_clsmethod_config"), + {}, + {"MinimalPlugin": MinimalPlugin}, + ), + pytest.param(lazy_fixture("cov_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC), + pytest.param(lazy_fixture("geno_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC), + pytest.param(lazy_fixture("ehr_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC), + pytest.param( + lazy_fixture("tabular_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param(lazy_fixture("h5ad_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC), + pytest.param( + lazy_fixture("geno_geno_mini_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("geno_geno_mini_map_config"), + {}, + {}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_geno_mini_map_config"), + {"combine_eids_as": "union"}, + {}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("cov_cov_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param( + lazy_fixture("cov_geno_config"), {}, {}, marks=SKIPIF_NOT_BIH_HPC + ), + ], + ) + def test_serialize(self, config, overrides, custom_plugins): + config.update(overrides) + DIR = abspath(os.path.join(dirname(__file__), UDM_HOME)) + with TemporaryDirectory(dir=DIR) as module_dir: + plugin_name = list(config["plugins"].keys())[0] + module_path = ( + pathlib.Path(module_dir) / f"{plugin_name}_plugin_serialized.pkl" + ) + gdm = GeneralDatamodule(**config, custom_plugins=custom_plugins) + gdm.plugins[plugin_name].serialize(module_path) + + plugin = SerializedPlugin.from_compressed_pickle(module_path) diff --git a/test/general_dataset_test.py b/test/general_dataset_test.py new file mode 100644 index 0000000..309d4f0 --- /dev/null +++ b/test/general_dataset_test.py @@ -0,0 +1,679 @@ +import os +from inspect import isclass +from os.path import abspath, dirname +from typing import Callable +import itertools as it +import hydra +import numpy as np +import pandas as pd +import pytest +from pytest_lazyfixture import lazy_fixture +import torch +from omegaconf import OmegaConf +from tempfile import TemporaryDirectory + +from udm.general_dataset import GeneralDataset +from udm.plugins.covariates.covariates_plugin import CovariatesPlugin +from udm.plugins.genetic.genetic_plugin import GeneticPlugin +from udm.transforms.dict_transforms import PluginTransform, DatasetTransform +from test.utils import assertBatchEqual, try_clear_hydra +from test.transforms.tensor_transforms.factory_test import ScaleTrafo +from test.plugins.dataplugin_test import MinimalPlugin +from test.plugins.genetic.genetic_plugin_test import geno_debug_config # noqa +from test.plugins.covariates.covariates_plugin_test import ( + covariates_debug_config, + mini1_df, + prepare_mini1, + covariates_mini1_config, +) # noqa + + +UDM_HOME = ".." +UDM_HOME_ABSPATH = abspath(os.path.join(dirname(__file__), UDM_HOME)) + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +def prepare_plugin(plugin_cls, config): + plugin = plugin_cls.from_config(config) + plugin.prepare_data() + plugin.setup() + return plugin + + +@pytest.fixture +def minimal_plugin(): + return prepare_plugin(MinimalPlugin, OmegaConf.create({})) + + +@pytest.fixture +def minimal_str_plugin(): + return prepare_plugin(MinimalPlugin, OmegaConf.create({"eids_type": "str"})) + + +@pytest.fixture +def covariates_mini1_plugin(covariates_mini1_config): + return prepare_plugin(CovariatesPlugin, covariates_mini1_config) + + +@pytest.fixture +def prepare_genetic_mini1(temp_dir): + try_clear_hydra() + with TemporaryDirectory(dir=temp_dir) as temp_dir: + DS_NAME = "genetic1" + genetic_data = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0]], dtype=np.uint8) + data_mmap = np.memmap( + f"{temp_dir}/{DS_NAME}.X.npy", + dtype=np.uint8, + mode="w+", + shape=(3, 3), + ) + data_mmap[:, :] = genetic_data[:, :] + data_mmap.flush() + obs = pd.DataFrame({"index": [0, 1, 2]}) + var = pd.DataFrame( + { + "Name": ["snp1", "snp2", "snp3"], + "ID": ["rs1", "rs2", "rs3"], + "GeneticAlt": ["G", "A", "G"], + "Field": ["ENSG001", "ENSG002", "ENSG003"], + } + ) + obs.to_csv(f"{temp_dir}/{DS_NAME}.obs.csv", index=False) + var.to_csv(f"{temp_dir}/{DS_NAME}.var.csv", index=False) + yield temp_dir, DS_NAME + + +@pytest.fixture +def genetic_mini1_plugin(prepare_genetic_mini1): + temp_dir, ds_name = prepare_genetic_mini1 + hydra.initialize( + version_base=None, + config_path=os.path.join(UDM_HOME, "config", "plugins", "genetic"), + ) + base_config = hydra.compose(config_name="default.yaml", overrides=[]) + overrides_config = OmegaConf.create( + { + "src": ds_name, + "src_names": "data", + "dataset_path": temp_dir, + "eid_map_path": None, + } + ) + config = OmegaConf.merge(base_config, overrides_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def prepare_genetic_mini2(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + DS_NAME = "genetic2" + genetic_data = np.array( + [[1, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]], dtype=np.uint8 + ) + data_mmap = np.memmap( + f"{temp_dir}/{DS_NAME}.X.npy", + dtype=np.uint8, + mode="w+", + shape=(4, 3), + ) + data_mmap[:, :] = genetic_data[:, :] + data_mmap.flush() + obs = pd.DataFrame({"index": [0, 1, 2, 3]}) + var = pd.DataFrame( + { + "Name": ["snp4", "snp5", "snp6"], + "ID": ["rs1", "rs2", "rs3"], + "GeneticAlt": ["C", "T", "G"], + "Field": ["ENSG004", "ENSG005", "ENSG006"], + } + ) + obs.to_csv(f"{temp_dir}/{DS_NAME}.obs.csv", index=False) + var.to_csv(f"{temp_dir}/{DS_NAME}.var.csv", index=False) + yield temp_dir, DS_NAME + + +@pytest.fixture +def genetic_mini2_plugin(prepare_genetic_mini2): + try_clear_hydra() + ds_path, ds_name = prepare_genetic_mini2 + hydra.initialize( + version_base=None, + config_path=os.path.join(UDM_HOME, "config", "plugins", "genetic"), + ) + base_config = hydra.compose(config_name="default.yaml", overrides=[]) + overrides_config = OmegaConf.create( + { + "src": ds_name, + "src_names": "data", + "dataset_path": ds_path, + "eid_map_path": None, + } + ) + config = OmegaConf.merge(base_config, overrides_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def genetic_mini1_mini2_plugin(genetic_mini1_plugin, genetic_mini2_plugin): + yield [genetic_mini1_plugin, genetic_mini2_plugin] + + +@pytest.fixture +def genetic_base_config(): + try_clear_hydra() + hydra.initialize( + version_base=None, + config_path=os.path.join(UDM_HOME, "config", "plugins", "genetic"), + ) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + config = hydra.compose( + config_name="default.yaml", + overrides=[ + f"dataset_path={FAKE_DATA_PATH}", + "eid_map_path=null", + ], + ) + return config + + +@pytest.fixture +def genetic_plugin(genetic_base_config): + config = genetic_base_config + split_config = OmegaConf.create( + { + "src": [ + "genetic_extended_tr", + "genetic_valid", + "genetic_test", + ], + "src_names": ["train", "valid", "test"], + } + ) + config = OmegaConf.merge(config, split_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def genetic_train_plugin(genetic_base_config): + config = genetic_base_config + split_config = OmegaConf.create( + { + "src": "genetic_extended_tr", + "src_names": "train", + } + ) + config = OmegaConf.merge(config, split_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def genetic_train_eids(): + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + src = "genetic_extended_tr" + obs_path = os.path.join(FAKE_DATA_PATH, f"{src}.obs.csv") + with open(obs_path, "r") as f: + eids = pd.read_csv(f)["index"].tolist() + return eids + + +@pytest.fixture +def genetic_valid_plugin(genetic_base_config): + config = genetic_base_config + split_config = OmegaConf.create( + { + "src": "genetic_valid", + "src_names": "valid", + } + ) + config = OmegaConf.merge(config, split_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def genetic_valid_eids(): + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + src = "genetic_valid" + obs_path = os.path.join(FAKE_DATA_PATH, f"{src}.obs.csv") + with open(obs_path, "r") as f: + eids = pd.read_csv(f)["index"].tolist() + return eids + + +@pytest.fixture +def genetic_test_plugin(genetic_base_config): + config = genetic_base_config + split_config = OmegaConf.create( + { + "src": "genetic_test", + "src_names": "test", + } + ) + config = OmegaConf.merge(config, split_config) + plugin = prepare_plugin(GeneticPlugin, config) + yield plugin + + +@pytest.fixture +def genetic_test_eids(): + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + src = "genetic_test" + obs_path = os.path.join(FAKE_DATA_PATH, f"{src}.obs.csv") + with open(obs_path, "r") as f: + eids = pd.read_csv(f)["index"].tolist() + return eids + + +@pytest.fixture +def genetic_eids(genetic_train_eids, genetic_valid_eids, genetic_test_eids): + return [*genetic_train_eids, *genetic_valid_eids, *genetic_test_eids] + + +@pytest.fixture +def genetic_eids_head(genetic_eids): + return genetic_eids[:10] + + +@pytest.fixture +def geno_cov_debug_plugin(geno_debug_config, covariates_debug_config): + geno_debug_plugin = prepare_plugin(GeneticPlugin, geno_debug_config) + covariates_debug_plugin = prepare_plugin(CovariatesPlugin, covariates_debug_config) + yield [geno_debug_plugin, covariates_debug_plugin] + + +class TestGeneralDataset(object): + @pytest.mark.parametrize( + "plugins, names, custom, transforms", + [ + (lazy_fixture("minimal_plugin"), "minimal", {}, None), + (lazy_fixture("minimal_str_plugin"), "minimal", {}, None), + pytest.param( + lazy_fixture("genetic_plugin"), + "genetic", + {}, + None, + ), + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + "genetic", + {}, + None, + ), + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + "genetic", + {}, + DatasetTransform( + {"geno": PluginTransform({"genetic": ScaleTrafo(2.0)})} + ), + ), + ], + ) + def test__init__(self, plugins, names, custom, transforms): + if isinstance(plugins, list): + plugin_eids = [p.get_eids() for p in plugins] + else: + plugin_eids = [plugins.get_eids()] + eids_all = list(set(it.chain(*plugin_eids))) + eids = eids_all[:10] + general_ds = GeneralDataset( + plugins, names, eids, transforms=transforms, keyerr=False + ) + + @pytest.mark.parametrize( + "plugins, eids, exp_len", + [ + (lazy_fixture("minimal_plugin"), None, 6), + (lazy_fixture("minimal_plugin"), [0, 2, 3], 3), + (lazy_fixture("minimal_str_plugin"), ["a1", "b2", "f6"], 3), + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + None, + 3, + ), + pytest.param( + lazy_fixture("genetic_mini2_plugin"), + None, + 4, + ), + pytest.param( + lazy_fixture("genetic_mini1_mini2_plugin"), + [0, 1, 2, 3], + 4, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + lazy_fixture("genetic_eids_head"), + 10, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + lazy_fixture("genetic_train_eids"), + 2048, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + lazy_fixture("genetic_valid_eids"), + 1024, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + lazy_fixture("genetic_test_eids"), + 1024, + ), + pytest.param( + lazy_fixture("genetic_train_plugin"), + lazy_fixture("genetic_train_eids"), + 2048, + ), + pytest.param( + lazy_fixture("genetic_valid_plugin"), + lazy_fixture("genetic_valid_eids"), + 1024, + ), + pytest.param( + lazy_fixture("genetic_test_plugin"), + lazy_fixture("genetic_test_eids"), + 1024, + ), + ], + ) + def test__len__(self, plugins, eids, exp_len): + if isinstance(eids, Callable): + eids = eids(plugins) + general_ds = GeneralDataset(plugins, eids=eids, keyerr=False) + assert len(general_ds) == exp_len + + FEMALE = [1.0, 0.0] + MALE = [0.0, 1.0] + ELF = [1.0, 0.0, 0.0] + HOBBIT = [0.0, 1.0, 0.0] + HUMAN = [0.0, 0.0, 1.0] + AGES = np.array([200.0, 60.0, 57.0]) + AGES = (AGES - np.mean(AGES)) / np.std(AGES) + + @pytest.mark.parametrize( + "plugins, names, custom, transforms, eids, ids, shapes_only, exp_item", + [ + *[ + ( + lazy_fixture("minimal_plugin"), + ["minimal"], + {}, + None, + None, + idx, + False, + { + "minimal": { + "data": torch.tensor( + [ + [0, 0, 0], + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ], + dtype=torch.uint8, + )[idx, :] + } + }, + ) + for idx in range(6) + ], + ( + lazy_fixture("minimal_plugin"), + ["minimal"], + {}, + None, + None, + [0, 2, 3], + False, + { + "minimal": { + "data": torch.tensor( + [[0, 0, 0], [0, 1, 0], [1, 0, 0]], dtype=torch.uint8 + ) + } + }, + ), + *[ + ( + lazy_fixture("minimal_str_plugin"), + ["minimal"], + {}, + None, + None, + idx, + False, + { + "minimal": { + "data": torch.tensor( + [ + [0, 0, 0], + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ], + dtype=torch.uint8, + )[idx, :] + } + }, + ) + for idx in range(6) + ], + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + ["genetic_mini1"], + {}, + None, + None, + 0, + False, + { + "genetic_mini1": { + "genetic": torch.tensor([0, 1, 0], dtype=torch.float16) / 127 + }, + }, + ), + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + ["genetic_mini1"], + {}, + DatasetTransform( + {"genetic_mini1": PluginTransform({"genetic": ScaleTrafo(2.0)})} + ), + None, + 0, + False, + { + "genetic_mini1": { + "genetic": torch.tensor([0, 2, 0], dtype=torch.float16) / 127 + }, + }, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + ["genetic"], + {}, + None, + lazy_fixture("genetic_eids_head"), + 5797, + False, + IndexError, + ), + pytest.param( + lazy_fixture("genetic_plugin"), + ["genetic"], + {}, + None, + lazy_fixture("genetic_eids_head"), + 11, + False, + IndexError, + ), + pytest.param( + lazy_fixture("genetic_mini1_mini2_plugin"), + ["genetic_mini1", "genetic_mini2"], + {}, + None, + None, + 0, + False, + { + "genetic_mini1": { + "genetic": torch.tensor([0, 1, 0], dtype=torch.float16) / 127 + }, + "genetic_mini2": { + "genetic": torch.tensor([1, 0, 0], dtype=torch.float16) / 127 + }, + }, + ), + ( + lazy_fixture("covariates_mini1_plugin"), + ["cov_mini1"], + {}, + None, + None, + 0, + False, + { + "cov_mini1": { + "covariates": torch.tensor( + [AGES[0], *FEMALE, *ELF, 1.0], dtype=torch.float64 + ) + }, + }, + ), + ( + lazy_fixture("covariates_mini1_plugin"), + ["cov_mini1"], + {}, + None, + None, + 0, + False, + { + "cov_mini1": { + "covariates": torch.tensor( + [AGES[0], *FEMALE, *ELF, 1.0], dtype=torch.float64 + ) + }, + }, + ), + ], + ) + def test__getitem__( + self, + plugins, + names, + custom, + transforms, + eids, + ids, + shapes_only, + exp_item, + ): + if isinstance(eids, Callable): + eids = eids(plugins) + general_ds = GeneralDataset( + plugins, names=names, eids=eids, transforms=transforms, keyerr=False + ) + if isclass(exp_item) and issubclass(exp_item, Exception): + with pytest.raises(exp_item): + item = general_ds[ids] + else: + item = general_ds[ids] + assertBatchEqual(item, exp_item, shapes_only=shapes_only) + + @pytest.mark.parametrize( + "plugins, names, eids, exp_metadata", + [ + pytest.param( + lazy_fixture("genetic_mini1_plugin"), + ["geno"], + None, + { + "geno": { + "tags": [], + "eids": [0, 1, 2], + "features": {"genetic": (["rs1", "rs2", "rs3"],)}, + "feature_types": {"genetic": np.uint8}, + "variables": { + "0": { + "Name": "snp1", + "ID": "rs1", + "GeneticAlt": "G", + "Field": "ENSG001", + }, + "1": { + "Name": "snp2", + "ID": "rs2", + "GeneticAlt": "A", + "Field": "ENSG002", + }, + "2": { + "Name": "snp3", + "ID": "rs3", + "GeneticAlt": "G", + "Field": "ENSG003", + }, + }, + } + }, + ), + pytest.param( + lazy_fixture("genetic_mini2_plugin"), + ["geno"], + None, + { + "geno": { + "tags": [], + "eids": [0, 1, 2, 3], + "features": {"genetic": (["rs1", "rs2", "rs3"],)}, + "feature_types": {"genetic": np.uint8}, + "variables": { + "0": { + "Name": "snp4", + "ID": "rs1", + "GeneticAlt": "C", + "Field": "ENSG004", + }, + "1": { + "Name": "snp5", + "ID": "rs2", + "GeneticAlt": "T", + "Field": "ENSG005", + }, + "2": { + "Name": "snp6", + "ID": "rs3", + "GeneticAlt": "G", + "Field": "ENSG006", + }, + }, + } + }, + ), + ], + ) + def test_get_metadata(self, plugins, names, eids, exp_metadata): + if isinstance(eids, Callable): + eids = eids(plugins) + general_ds = GeneralDataset(plugins, names=names, eids=eids, keyerr=False) + metadata = general_ds.get_metadata() + KEYS = set(["eids", "features", "feature_types"]) + assert isinstance(metadata, dict) + assert metadata == exp_metadata diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/integration/test_dummy.py b/test/integration/test_dummy.py new file mode 100644 index 0000000..0e518e0 --- /dev/null +++ b/test/integration/test_dummy.py @@ -0,0 +1,3 @@ +class TestDummy: + def test_dummy(self): + assert True diff --git a/test/plugins/__init__.py b/test/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/covariates/__init__.py b/test/plugins/covariates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/covariates/covariates_plugin_test.py b/test/plugins/covariates/covariates_plugin_test.py new file mode 100644 index 0000000..be5e081 --- /dev/null +++ b/test/plugins/covariates/covariates_plugin_test.py @@ -0,0 +1,511 @@ +import os +import numpy as np +import hydra +import yaml +import pyarrow as pa +import pyarrow.feather as feather +import pytest +import torch +from tempfile import TemporaryDirectory +from pytest_lazyfixture import lazy_fixture + +from udm.plugins.covariates.covariates_plugin import CovariatesPlugin +from test.utils import Head, SKIPIF_NOT_BIH_HPC, try_clear_hydra + +UDM_HOME = "../../.." + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def mini1_df(): + return { + "eid": [0, 1, 2], + "age": [200.0, 60.0, 57.0], + "sex": ["Female", "Male", "Female"], + "ancestry": ["Elf", "Human", "Hobbit"], + "recruitment_center": ["MiddleEarth", "MiddleEarth", "MiddleEarth"], + } + + +@pytest.fixture +def mini1_str_df(mini1_df): + mini1_df.update({"eid": ["a1", "b2", "c3"]}) + return mini1_df + + +def prepare_temp_file(temp_dir, name, df): + TMP_PATH = os.path.join(temp_dir, name) + covariates_pa = pa.Table.from_pydict(df) + feather.write_feather(covariates_pa, TMP_PATH) + return TMP_PATH + + +@pytest.fixture +def prepare_mini1(temp_dir, mini1_df): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_temp_file(temp_dir, "covariates1.feather", mini1_df) + + +@pytest.fixture +def prepare_mini1_str(temp_dir, mini1_str_df): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_temp_file(temp_dir, "covariates1.feather", mini1_str_df) + + +@pytest.fixture +def mini2_df(): + return { + "eid": [3, 4, 5], + "age": [60.0, 70.0, 1000.0], + "sex": ["Male", "Male", "Male"], + "ancestry": ["Human", "Human", "Dragon"], + "recruitment_center": ["MiddleEarth", "MiddleEarth", "MiddleEarth"], + } + + +@pytest.fixture +def mini2_str_df(mini2_df): + mini2_df.update({"eid": ["d4", "e5", "f6"]}) + return mini2_df + + +@pytest.fixture +def prepare_mini2(temp_dir, mini2_df): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_temp_file(temp_dir, "covariates2.feather", mini2_df) + + +@pytest.fixture +def prepare_mini2_str(temp_dir, mini2_str_df): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_temp_file(temp_dir, "covariates2.feather", mini2_str_df) + + +def covariates_config(config_name, overrides=[]): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "covariates") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose(config_name=config_name, overrides=overrides) + return config + + +@pytest.fixture +def covariates_debug_config(): + yield covariates_config(config_name="debug", overrides=[]) + + +@pytest.fixture +def covariates_mini_config(prepare_mini1, prepare_mini2): + TMP_PATH1, TMP_PATH2 = prepare_mini1, prepare_mini2 + yield covariates_config( + "default.yaml", + overrides=[ + f"data_paths=[{TMP_PATH1}, {TMP_PATH2}]", + "names=[]", + ], + ) + + +@pytest.fixture +def covariates_mini1_config(prepare_mini1): + TMP_PATH = prepare_mini1 + yield covariates_config("default.yaml", overrides=[f"data_paths={TMP_PATH}"]) + + +@pytest.fixture +def covariates_mini1_str_config(prepare_mini1_str): + TMP_PATH = prepare_mini1_str + yield covariates_config("default.yaml", overrides=[f"data_paths={TMP_PATH}"]) + + +@pytest.fixture +def covariates_mini2_config(prepare_mini2): + TMP_PATH = prepare_mini2 + yield covariates_config("default.yaml", overrides=[f"data_paths={TMP_PATH}"]) + + +@pytest.fixture +def covariates_mini2_str_config(prepare_mini2_str): + TMP_PATH = prepare_mini2_str + yield covariates_config("default.yaml", overrides=[f"data_paths={TMP_PATH}"]) + + +@pytest.fixture +def covariates_full_wandb_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "covariates") + hydra.initialize(version_base=None, config_path=config_path) + path_wandb = "data/cvd/data/2_datasets_pre/211110_anewbeginning/artifacts/" + config = hydra.compose( + config_name="from_wandb_default.yaml", + overrides=[f"data_paths=[{path_wandb}]"], + ) + yield config + + +@pytest.fixture +def covariates_debug_eids(): + COVARIATES_DEBUG_EIDS_PATH = os.path.join( + UDM_HOME, + "data/cvd/data/2_datasets_pre/211110_anewbeginning/artifacts/debug/not_anonymized_debug/eids.yaml", + ) + eids = yaml.load(COVARIATES_DEBUG_EIDS_PATH, Loader=yaml.CSafeLoader) + return list(eids) + + +@pytest.fixture +def covariates_debug_eids_head(covariates_debug_eids): + return Head(covariates_debug_eids) + + +@pytest.fixture +def covariates_debug_eid(covariates_debug_eids): + return covariates_debug_eids[101] + + +class TestCovariatesPlugin(object): + @pytest.mark.parametrize( + "config", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), marks=SKIPIF_NOT_BIH_HPC + ), + lazy_fixture("covariates_mini_config"), + lazy_fixture("covariates_mini1_config"), + lazy_fixture("covariates_mini1_str_config"), + lazy_fixture("covariates_mini2_config"), + lazy_fixture("covariates_mini2_str_config"), + pytest.param( + lazy_fixture("covariates_full_wandb_config"), marks=SKIPIF_NOT_BIH_HPC + ), + ], + ) + def test__init__(self, config): + if config["__init__"] == "from_wandb": + covariates_plugin = CovariatesPlugin.from_wandb(**config) + else: + covariates_plugin = CovariatesPlugin.from_config(config) + + FEMALE = [1.0, 0.0] + MALE = [0.0, 1.0] + ELF = [1.0, 0.0, 0.0] + HOBBIT = [0.0, 1.0, 0.0] + HUMAN = [0.0, 0.0, 1.0] + AGES = np.array([200.0, 60.0, 57.0]) + AGES = (AGES - np.mean(AGES)) / np.std(AGES) + + @pytest.mark.parametrize( + "config, eid, features, exp_item", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), + lazy_fixture("covariates_debug_eid"), + None, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + ( + lazy_fixture("covariates_mini1_config"), + 0, + None, + [AGES[0], *FEMALE, *ELF, 1.0], + ), + ( + lazy_fixture("covariates_mini1_config"), + 1, + None, + [AGES[1], *MALE, *HUMAN, 1.0], + ), + ( + lazy_fixture("covariates_mini1_config"), + 2, + None, + [AGES[2], *FEMALE, *HOBBIT, 1.0], + ), + ( + lazy_fixture("covariates_mini1_str_config"), + "a1", + None, + [AGES[0], *FEMALE, *ELF, 1.0], + ), + ( + lazy_fixture("covariates_mini1_str_config"), + "b2", + None, + [AGES[1], *MALE, *HUMAN, 1.0], + ), + ( + lazy_fixture("covariates_mini1_str_config"), + "c3", + None, + [AGES[2], *FEMALE, *HOBBIT, 1.0], + ), + ( + lazy_fixture("covariates_mini1_config"), + 0, + { + "covariates": ( + torch.tensor( + [True, True, True, False, False, False, False], + dtype=torch.bool, + ), + ) + }, + [AGES[0], *FEMALE], + ), + ( + lazy_fixture("covariates_mini1_config"), + 0, + {"covariates": (["num__age", "cat__sex_Female", "cat__sex_Male"],)}, + [AGES[0], *FEMALE], + ), + ( + lazy_fixture("covariates_mini1_config"), + 1, + { + "covariates": ( + torch.tensor( + [False, False, False, True, True, True, True], + dtype=torch.bool, + ), + ) + }, + [*HUMAN, 1.0], + ), + ( + lazy_fixture("covariates_mini1_config"), + 2, + { + "covariates": ( + torch.tensor( + [True, False, False, False, False, False, True], + dtype=torch.bool, + ), + ) + }, + [AGES[2], 1.0], + ), + ( + lazy_fixture("covariates_mini1_config"), + [0, 2], + None, + [[AGES[0], *FEMALE, *ELF, 1.0], [AGES[2], *FEMALE, *HOBBIT, 1.0]], + ), + ( + lazy_fixture("covariates_mini1_config"), + [0, 2], + {"covariates": ([True, True, True, False, False, False, False],)}, + [[AGES[0], *FEMALE], [AGES[2], *FEMALE]], + ), + ( + lazy_fixture("covariates_mini1_config"), + slice(0, 2), + None, + [[AGES[0], *FEMALE, *ELF, 1.0], [AGES[1], *MALE, *HUMAN, 1.0]], + ), + ], + ) + def test__getitem__(self, config, eid, features, exp_item): + covariates_plugin = CovariatesPlugin.from_config(config) + covariates_plugin.prepare_data() + covariates_plugin.setup() + item = covariates_plugin[eid, features] + assert isinstance(item, dict) + assert list(item.keys()) == ["covariates"] + # One-hot encoding of [Female, Male] + One-hot encoding of [Elf, Hobbit, Human] + if exp_item is not None: + expected = torch.tensor(exp_item, dtype=torch.float64) + assert torch.allclose(item["covariates"], expected) + + @pytest.mark.parametrize( + "config", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), marks=SKIPIF_NOT_BIH_HPC + ), + lazy_fixture("covariates_mini_config"), + lazy_fixture("covariates_mini1_config"), + lazy_fixture("covariates_mini1_str_config"), + lazy_fixture("covariates_mini2_config"), + lazy_fixture("covariates_mini2_str_config"), + pytest.param( + lazy_fixture("covariates_full_wandb_config"), marks=SKIPIF_NOT_BIH_HPC + ), + ], + ) + def test_prepare_data(self, config): + if config["__init__"] == "from_wandb": + covariates_plugin = CovariatesPlugin.from_wandb(**config) + else: + covariates_plugin = CovariatesPlugin.from_config(config) + covariates_plugin.prepare_data() + + @pytest.mark.parametrize( + "config", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), marks=SKIPIF_NOT_BIH_HPC + ), + lazy_fixture("covariates_mini_config"), + lazy_fixture("covariates_mini1_config"), + lazy_fixture("covariates_mini1_str_config"), + lazy_fixture("covariates_mini2_config"), + lazy_fixture("covariates_mini2_str_config"), + pytest.param( + lazy_fixture("covariates_full_wandb_config"), marks=SKIPIF_NOT_BIH_HPC + ), + ], + ) + def test_setup(self, config): + if config["__init__"] == "from_wandb": + covariates_plugin = CovariatesPlugin.from_wandb(**config) + else: + covariates_plugin = CovariatesPlugin.from_config(config) + covariates_plugin.prepare_data() + covariates_plugin.setup() + + @pytest.mark.parametrize( + "config, overrides, exp_columns", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), + {}, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + (lazy_fixture("covariates_mini1_config"), {}, None), + (lazy_fixture("covariates_mini2_config"), {}, None), + pytest.param( + lazy_fixture("covariates_full_wandb_config"), + {}, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("covariates_debug_config"), + { + "usecols": [], + "dropcols": [], + }, + set( + [ + "age_at_recruitment_f21022_0_0", + "sex_f31_0_0", + "ethnic_background_f21000_0_0", + "uk_biobank_assessment_centre_f54_0_0", + ] + ), + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("covariates_debug_config"), + { + "usecols": ["age_at_recruitment_f21022_0_0", "sex_f31_0_0"], + "dropcols": [], + }, + set(["age_at_recruitment_f21022_0_0", "sex_f31_0_0"]), + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("covariates_debug_config"), + { + "usecols": [], + "dropcols": ["age_at_recruitment_f21022_0_0", "sex_f31_0_0"], + }, + set( + [ + "ethnic_background_f21000_0_0", + "uk_biobank_assessment_centre_f54_0_0", + ] + ), + marks=SKIPIF_NOT_BIH_HPC, + ), + ], + ) + def test_get_metadata(self, config, overrides, exp_columns): + if overrides: + config.update(overrides) + if config["__init__"] == "from_wandb": + covariates_plugin = CovariatesPlugin.from_wandb(**config) + else: + covariates_plugin = CovariatesPlugin.from_config(config) + covariates_plugin.prepare_data() + covariates_plugin.setup() + metadata = covariates_plugin.get_metadata() + assert isinstance(metadata, dict) + KEYS_METADATA = set(["eids", "features", "feature_types"]) + assert set(metadata.keys()).intersection(KEYS_METADATA) == KEYS_METADATA + if exp_columns: + assert set(metadata["features"]) == exp_columns + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_eids", + [ + pytest.param( + lazy_fixture("covariates_debug_config"), + False, + False, + [], + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("covariates_debug_config"), + True, + False, + lazy_fixture("covariates_debug_eids_head"), + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("covariates_debug_config"), + True, + True, + lazy_fixture("covariates_debug_eids_head"), + marks=SKIPIF_NOT_BIH_HPC, + ), + (lazy_fixture("covariates_mini1_config"), False, False, []), + (lazy_fixture("covariates_mini1_config"), True, False, [0, 1, 2]), + (lazy_fixture("covariates_mini1_config"), True, True, [0, 1, 2]), + ( + lazy_fixture("covariates_mini1_str_config"), + True, + True, + ["a1", "b2", "c3"], + ), + (lazy_fixture("covariates_mini2_config"), True, True, [3, 4, 5]), + ( + lazy_fixture("covariates_mini2_str_config"), + True, + True, + ["d4", "e5", "f6"], + ), + pytest.param( + lazy_fixture("covariates_full_wandb_config"), + True, + True, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + ], + ) + def test_get_eids(self, config, prepare_data, setup, exp_eids, request): + if config["__init__"] == "from_wandb": + covariates_plugin = CovariatesPlugin.from_wandb(**config) + else: + covariates_plugin = CovariatesPlugin.from_config(config) + if prepare_data: + covariates_plugin.prepare_data() + if setup: + covariates_plugin.setup() + eids = covariates_plugin.get_eids() + assert isinstance(eids, list) + if exp_eids is not None: + # No need to compare ALL eids! + if isinstance(exp_eids, Head): + assert Head(eids) == exp_eids + else: + assert eids == exp_eids diff --git a/test/plugins/dataplugin_test.py b/test/plugins/dataplugin_test.py new file mode 100644 index 0000000..509667c --- /dev/null +++ b/test/plugins/dataplugin_test.py @@ -0,0 +1,669 @@ +import pytest +from pytest_lazyfixture import lazy_fixture +import pandas as pd +import tempfile +import numpy as np +import torch +from inspect import isclass +from collections import OrderedDict +from typing import Any, Dict, Hashable, Iterable, List, Tuple +from omegaconf import OmegaConf, DictConfig + +from udm.plugins.dataplugin import ( + DataPlugin, + PluginFeatureSelection, + mapped, +) +from udm.utils import SliceableDict, numpy_to_torch +from udm.filters import PluginRowFilter, PluginColFilter, IsIn, AnyIn +from udm.transforms.dict_transforms import PluginTransform +from udm.transforms.tensor_transforms import ListTransform +from test.transforms.tensor_transforms.factory_test import ScaleTrafo +from test.filters.simple_test import AnyOne, AnyTwo + + +class MinimalPlugin(DataPlugin): + def __init__(self, eids_type="int", **kwargs): + super().__init__(**kwargs) + self.data = torch.tensor( + [[0, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 1], [1, 1, 0]], + dtype=torch.uint8, + ) + self.eids = ( + ["a1", "b2", "c3", "d4", "e5", "f6"] + if eids_type == "str" + else [0, 1, 2, 3, 4, 5] + ) + self.eid_to_idx = SliceableDict({eid: idx for idx, eid in enumerate(self.eids)}) + + def __len__(self): + return len(self.data) + + @mapped + def __getitem__( + self, eid: int, features: PluginFeatureSelection = None + ) -> Dict[str, torch.Tensor]: + return super().__getitem__(eid, features=features) + + def _getitem(self, eid: int, features: PluginFeatureSelection = None): + idx = self.eid_to_idx[eid] + features = features["data"][0] if features else slice(None) + if isinstance(idx, Iterable) and isinstance(features, Iterable): + return {"data": self.data[np.ix_(idx, features)]} + else: + return {"data": self.data[idx, features]} + + def get_feature_types(self) -> Dict[str, torch.dtype]: + return {"data": torch.uint8} + + def _get_features(self) -> Dict[str, Tuple[List[Hashable]]]: + return {"data": ([0, 1, 2],)} + + @classmethod + def from_list(cls, data: list, **kwargs): + instance = cls() + data = np.array(data) + instance.data = torch.from_numpy(data) + return instance + + def get_eids(self) -> List[int]: + return self.get_native_eids() + + def get_native_eids(self) -> List[int]: + return self.eids + + +class GenericPlugin(DataPlugin): + def __init__(self, path: str, **kwargs): + super().__init__(**kwargs) + self.path = path + + def __len__(self): + return len(self.df) + + @mapped + def __getitem__( + self, eid: int, features: PluginFeatureSelection = None + ) -> Dict[str, torch.Tensor]: + return super().__getitem__(eid, features=features) + + def _getitem( + self, eid: int, features: PluginFeatureSelection = None + ) -> Dict[str, torch.Tensor]: + features_ = features["data"][0] if features else slice(None, None, None) + if isinstance(features_, torch.Tensor): + features_ = features_.tolist() + tensor = torch.tensor(self.df.loc[eid, features_].values) + return {"data": tensor} + + def get_feature_types(self) -> Dict[str, Any]: + dtype_list = df.dtypes.tolist() + return {"data": numpy_to_torch(dtype_list[0])} + + def _get_features(self) -> Dict[str, Tuple[List[Hashable]]]: + return {"data": (list(self.df.columns),)} + + def _setup(self) -> None: + return None + + def prepare_data(self) -> None: + self.df = pd.read_csv(self.path) + + def _get_native_eids(self) -> List[int]: + """Returns the native eids of all samples in this DataPlugin as list.""" + return list(self.df.index.astype(int)) + + def get_native_eids_as_struct(self) -> "OrderedDict[str, List[int]]": + """Returns the native eids of all samples in this DataPlugin as struct.""" + return {0: self.get_native_eids()} + + +@pytest.fixture +def temp_df_path(): + with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv") as temp_file: + df = pd.DataFrame( + {"C1": [1.0, 2.0, 3.0], "C2": [2.0, 3.0, 4.0], "C3": [3.0, 4.0, 5.0]} + ) + temp_path = temp_file.name + df.to_csv(temp_path, index=False) + yield temp_path + + +@pytest.fixture +@pytest.mark.usefixtures("temp_df_path") +def base_cfg(temp_df_path): + return DictConfig( + { + "path": temp_df_path, + "eid_map_path": None, + "eid_map_from": "col1", + "eid_map_to": "col2", + "transforms": None, + "row_filter": None, + "col_filter": None, + "tags": [], + } + ) + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_eids_cfg(base_cfg): + filters_cfg = DictConfig( + { + "row_filter": { + "key_filter": { + "name": "IsIn", + "__init__": "__init__", + "whitelist": [0, 1], + }, + "val_filters": {}, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_nanrows_cfg(base_cfg): + filters_cfg = DictConfig( + { + "row_filter": { + "key_filter": None, + "val_filters": { + "data": { + "name": "Not", + "__init__": "__init__", + "filters": [{"name": "HasNan", "__init__": "__init__"}], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_ones_cfg(base_cfg): + filters_cfg = DictConfig( + { + "row_filter": { + "key_filter": None, + "val_filters": {"data": {"name": "AnyOne", "__init__": "__init__"}}, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_anyin_cfg(base_cfg): + filters_cfg = DictConfig( + { + "row_filter": { + "key_filter": None, + "val_filters": { + "data": { + "name": "AnyIn", + "__init__": "from_sequence", + "whitelist": [1, 5], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_features_cfg(base_cfg): + filters_cfg = DictConfig( + { + "col_filter": { + "key_filters": { + "data": { + "name": "IsIn", + "__init__": "__init__", + "whitelist": ["C1", "C3"], + } + }, + "val_filters": {}, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_features_val_cfg(base_cfg): + filters_cfg = DictConfig( + { + "col_filter": { + "key_filters": {}, + "val_filters": { + "data": { + "name": "AnyIn", + "__init__": "from_sequence", + "whitelist": [4, 6], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_features_keyval_cfg(base_cfg): + filters_cfg = DictConfig( + { + "col_filter": { + "key_filters": { + "data": { + "name": "IsIn", + "__init__": "__init__", + "whitelist": ["C1", "C3"], + } + }, + "val_filters": { + "data": { + "name": "AnyIn", + "__init__": "from_sequence", + "whitelist": [4, 6], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def filter_features_notones_cfg(base_cfg): + filters_cfg = DictConfig( + { + "col_filter": { + "key_filter": None, + "val_filters": { + "data": { + "name": "Not", + "__init__": "__init__", + "filters": [{"name": "AnyOne", "__init__": "__init__"}], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, filters_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def scale_trafo_cfg(base_cfg): + transforms_cfg = DictConfig( + { + "transforms": { + "name": "PluginTransform", + "transforms": { + "data": {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2.0} + }, + } + } + ) + config = OmegaConf.merge(base_cfg, transforms_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("base_cfg") +def list_trafo_cfg(base_cfg): + transforms_cfg = DictConfig( + { + "transforms": { + "name": "PluginTransform", + "transforms": { + "data": { + "name": "ListTransform", + "transforms": [ + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + ], + } + }, + } + } + ) + config = OmegaConf.merge(base_cfg, transforms_cfg) + return config + + +@pytest.fixture +@pytest.mark.usefixtures("scale_trafo_cfg") +def filter_custom_trafo_filter_cfg(scale_trafo_cfg): + filter_cfg = DictConfig( + { + "col_filter": { + "key_filter": None, + "val_filters": { + "data": { + "name": "Not", + "__init__": "__init__", + "filters": [{"name": "AnyTwo", "__init__": "__init__"}], + } + }, + } + } + ) + config = OmegaConf.merge(scale_trafo_cfg, filter_cfg) + return config + + +class TestDataplugin(object): + @pytest.mark.parametrize( + "eid_map_path,eid_map_from,eid_map_to,transforms,row_filter,col_filter,tags", + [ + (None, "col1", "col2", None, None, None, []), + ( + None, + "col1", + "col2", + PluginTransform({"data": ScaleTrafo(2.0)}), + None, + None, + [], + ), + ( + None, + "col1", + "col2", + PluginTransform( + {"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])} + ), + None, + None, + [], + ), + ( + None, + "col1", + "col2", + None, + PluginRowFilter( + key_filter=IsIn([1, 2]), + val_filters={"data": AnyIn(torch.tensor([2, 3]))}, + ), + None, + [], + ), + ( + None, + "col1", + "col2", + None, + PluginColFilter( + key_filters={"data": IsIn(["C1", "C3"])}, + val_filters={"data": AnyIn(torch.tensor([2, 3]))}, + ), + None, + [], + ), + ], + ) + def test__init__( + self, + eid_map_path, + eid_map_from, + eid_map_to, + transforms, + row_filter, + col_filter, + tags, + ): + plugin = GenericPlugin( + path=temp_df_path, + eid_map_path=eid_map_path, + eid_map_from=eid_map_from, + eid_map_to=eid_map_to, + transforms=transforms, + row_filter=row_filter, + col_filter=col_filter, + tags=tags, + ) + + @pytest.mark.parametrize( + "config,custom_transforms", + [ + (lazy_fixture("base_cfg"), {}), + (lazy_fixture("scale_trafo_cfg"), {"ScaleTrafo": ScaleTrafo}), + (lazy_fixture("list_trafo_cfg"), {"ScaleTrafo": ScaleTrafo}), + (lazy_fixture("filter_nanrows_cfg"), {}), + (lazy_fixture("filter_eids_cfg"), {}), + (lazy_fixture("filter_ones_cfg"), {"AnyOne": AnyOne}), + (lazy_fixture("filter_anyin_cfg"), {}), + ], + ) + def test_from_config(self, config, custom_transforms): + plugin = GenericPlugin.from_config(config, custom=custom_transforms) + + @pytest.mark.parametrize( + "args_or_config,custom_transforms,eid,features,exp_item", + [ + ((None, "col1", "col2", None, []), {}, 0, None, {"data": [1.0, 2.0, 3.0]}), + (lazy_fixture("base_cfg"), {}, 0, None, {"data": [1.0, 2.0, 3.0]}), + ((None, "col1", "col2", None, []), 1, {}, None, {"data": [2.0, 3.0, 4.0]}), + (lazy_fixture("base_cfg"), {}, 1, None, {"data": [2.0, 3.0, 4.0]}), + ( + (None, "col1", "col2", PluginTransform({"data": ScaleTrafo(2.0)}), []), + {}, + 0, + None, + {"data": [2.0, 4.0, 6.0]}, + ), + ( + lazy_fixture("scale_trafo_cfg"), + {"ScaleTrafo": ScaleTrafo}, + 0, + None, + {"data": [2.0, 4.0, 6.0]}, + ), + ( + ( + None, + "col1", + "col2", + PluginTransform( + {"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])} + ), + [], + ), + {}, + 0, + None, + {"data": [4.0, 8.0, 12.0]}, + ), + ( + lazy_fixture("list_trafo_cfg"), + {"ScaleTrafo": ScaleTrafo}, + 0, + None, + {"data": [4.0, 8.0, 12.0]}, + ), + (lazy_fixture("base_cfg"), {}, 1, None, {"data": [2.0, 3.0, 4.0]}), + ( + lazy_fixture("base_cfg"), + {}, + 0, + {}, + {}, + ), + ( + lazy_fixture("filter_ones_cfg"), + {"AnyOne": AnyOne}, + 0, + None, + KeyError, + ), + ( + lazy_fixture("filter_ones_cfg"), + {"AnyOne": AnyOne}, + 1, + None, + {"data": [2.0, 3.0, 4.0]}, + ), + ( + lazy_fixture("filter_ones_cfg"), + {"AnyOne": AnyOne}, + 2, + None, + {"data": [3.0, 4.0, 5.0]}, + ), + (lazy_fixture("filter_anyin_cfg"), {}, 0, None, KeyError), + ( + lazy_fixture("filter_anyin_cfg"), + {}, + 1, + None, + {"data": [2.0, 3.0, 4.0]}, + ), + (lazy_fixture("filter_anyin_cfg"), {}, 2, None, KeyError), + ( + lazy_fixture("filter_features_notones_cfg"), + {"AnyOne": AnyOne}, + 0, + None, + {"data": [2.0, 3.0]}, + ), + ( + lazy_fixture("filter_custom_trafo_filter_cfg"), + {"AnyTwo": AnyTwo, "ScaleTrafo": ScaleTrafo}, + 0, + None, + {"data": [4.0, 6.0]}, + ), + ( + lazy_fixture("filter_features_cfg"), + {}, + 0, + {"data": ["C2"]}, + KeyError, + ), + ( + lazy_fixture("filter_features_cfg"), + {}, + 0, + None, + {"data": [1.0, 3.0]}, + ), + ( + lazy_fixture("filter_features_cfg"), + {}, + 1, + None, + {"data": [2.0, 4.0]}, + ), + ( + lazy_fixture("filter_features_cfg"), + {}, + 2, + None, + {"data": [3.0, 5.0]}, + ), + ( + lazy_fixture("filter_features_val_cfg"), + {}, + 0, + None, + {"data": [2.0, 3.0]}, + ), + ( + lazy_fixture("filter_features_val_cfg"), + {}, + 1, + None, + {"data": [3.0, 4.0]}, + ), + ( + lazy_fixture("filter_features_val_cfg"), + {}, + 2, + None, + {"data": [4.0, 5.0]}, + ), + ( + lazy_fixture("filter_features_keyval_cfg"), + {}, + 0, + None, + {"data": [3.0]}, + ), + ( + lazy_fixture("filter_features_keyval_cfg"), + {}, + 1, + None, + {"data": [4.0]}, + ), + ( + lazy_fixture("filter_features_keyval_cfg"), + {}, + 2, + None, + {"data": [5.0]}, + ), + ], + ) + @pytest.mark.usefixtures("temp_df_path") + def test__getitem__( + self, args_or_config, custom_transforms, eid, features, exp_item, temp_df_path + ): + if isclass(exp_item) and issubclass(exp_item, Exception): + with pytest.raises(exp_item): + self._test__getitem__( + args_or_config, + custom_transforms, + eid, + features, + exp_item, + temp_df_path, + ) + else: + self._test__getitem__( + args_or_config, custom_transforms, eid, features, exp_item, temp_df_path + ) + + def _test__getitem__( + self, args_or_config, custom_transforms, eid, features, exp_item, temp_df_path + ): + if isinstance(args_or_config, DictConfig): + plugin = GenericPlugin.from_config(args_or_config, custom=custom_transforms) + else: + eid_map_path, eid_map_from, eid_map_to, transforms, tags = args_or_config + plugin = GenericPlugin( + path=temp_df_path, + eid_map_path=eid_map_path, + eid_map_from=eid_map_from, + eid_map_to=eid_map_to, + transforms=transforms, + ) + plugin.prepare_data() + plugin.setup() + item = plugin[eid, features] + assert isinstance(item, dict) + assert item.keys() == exp_item.keys() + for key in item.keys(): + assert torch.all(item[key] == torch.tensor(exp_item[key])) diff --git a/test/plugins/ehr/__init__.py b/test/plugins/ehr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/ehr/ehr_plugin_test.py b/test/plugins/ehr/ehr_plugin_test.py new file mode 100644 index 0000000..86829a7 --- /dev/null +++ b/test/plugins/ehr/ehr_plugin_test.py @@ -0,0 +1,1040 @@ +import os +from tempfile import TemporaryDirectory +import hydra +import numpy as np +import pandas as pd +import pytest +from pytest_lazyfixture import lazy_fixture +import torch +import yaml + +from udm.plugins.ehr.ehr_plugin import EHRPlugin +from test.utils import SKIPIF_NOT_BIH_HPC, Head, try_clear_hydra + + +UDM_HOME = "../../.." + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def data_mini1(): + records_dict1 = { + "concept_id": [ + "OMOP_1548196", + "OMOP_4324588", + "OMOP_4060831", + "phecode_557", + "phecode_247.72", + "OMOP_4306655", + ], + "domain_id": [ + "Drug", + "Observation", + "Condition", + "GI", + "Metab", + "Observation", + ], + "vocabulary": ["OMOP", "OMOP", "OMOP", "phecode", "phecode", "OMOP"], + "origin": [ + "gp_scripts_tpp", + "gp_ctv3", + "gp_ctv3", + "hes_icd10", + "hes_icd10", + "gp_ctv3", + ], + "eid": [327, 578, 578, 111, 111, 111], + "birth_date": [ + "1938-05-20", + "1949-08-07", + "1949-08-07", + "1956-08-16", + "1956-08-16", + "1956-08-16", + ], + "recruitment_date": [ + "2002-03-14", + "2005-08-23", + "2005-08-23", + "2010-01-07", + "2010-01-07", + "2010-01-07", + ], + "death_date": ["nat", "nat", "nat", "nat", "nat", "nat"], + "exit_date": [ + "2019-08-19", + "2019-08-19", + "2019-08-19", + "2019-08-19", + "2019-08-19", + "2019-08-19", + ], + "date": [ + "1947-02-03 13:26:04", + "1972-03-10 02:35:27", + "1955-11-25 18:07:14", + "2009-12-04 01:36:37", + "1993-03-18 01:14:02", + "2019-08-19 17:54:08", + ], + } + + dtype_dict = { + "concept_id": "category", + "domain_id": "category", + "vocabulary": "category", + "origin": "category", + "birth_date": np.datetime64, + "recruitment_date": np.datetime64, + "death_date": np.datetime64, + "exit_date": np.datetime64, + "date": np.datetime64, + } + return records_dict1, dtype_dict + + +@pytest.fixture +def data_mini1_str(data_mini1): + records_dict1, dtype_dict = data_mini1 + records_dict1.update({"eid": ["a327", "b578", "b578", "c111", "c111", "c111"]}) + yield records_dict1, dtype_dict + + +def prepare_feather(temp_dir, filename, records_dict, dtype_dict): + TMP_PATH = os.path.join(temp_dir, f"{filename}.feather") + records_df = pd.DataFrame(records_dict) + records_df = records_df.astype(dtype_dict) + records_df.to_feather(TMP_PATH) + return TMP_PATH + + +@pytest.fixture +def prepare_mini1(temp_dir, data_mini1): + records_dict1, dtype_dict = data_mini1 + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "records1", records_dict1, dtype_dict) + + +@pytest.fixture +def prepare_mini1_str(temp_dir, data_mini1_str): + records_dict1, dtype_dict = data_mini1_str + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "records1", records_dict1, dtype_dict) + + +@pytest.fixture +def data_mini2(temp_dir): + records_dict2 = { + "concept_id": [ + "phecode_401", + "OMOP_4306655", + "OMOP_4214579", + "OMOP_19020124", + "phecode_247.72", + ], + "domain_id": ["Cardio", "Observation", "Condition", "Drug", "Metab"], + "vocabulary": ["phecode", "OMOP", "OMOP", "OMOP", "phecode"], + "origin": [ + "hes_icd10", + "gp_ctv3", + "gp_ctv3", + "gp_scripts_emis", + "hes_icd10", + ], + "eid": [181, 181, 996, 733, 733], + "birth_date": [ + "1942-01-07", + "1942-01-07", + "1960-02-23", + "1962-10-25", + "1962-10-25", + ], + # "birth_date":[np.datetime64("1938-05-20"),np.datetime64("1949-08-07")] + "recruitment_date": [ + "1999-02-03", + "1999-02-03", + "2019-12-21", + "2003-08-18", + "2003-08-18", + ], + "death_date": ["2013-03-08", "2013-03-08", "2017-09-01", "nat", "nat"], + "exit_date": [ + "2013-03-08", + "2013-03-08", + "2017-09-01", + "2019-08-19", + "2019-08-19", + ], + "date": [ + "1968-12-30 13:09:34", + "2013-03-08 03:36:19", + "2001-09-05 16:54:37", + "1992-06-21 06:23:06", + "1988-05-20 22:04:36", + ], + } + dtype_dict = { + "concept_id": "category", + "domain_id": "category", + "vocabulary": "category", + "origin": "category", + "birth_date": np.datetime64, + "recruitment_date": np.datetime64, + "death_date": np.datetime64, + "exit_date": np.datetime64, + "date": np.datetime64, + } + return records_dict2, dtype_dict + + +@pytest.fixture +def data_mini2_str(temp_dir, data_mini2): + records_dict2, dtype_dict = data_mini2 + records_dict2.update( + { + "eid": ["d181", "d181", "e996", "f733", "f733"], + } + ) + yield records_dict2, dtype_dict + + +@pytest.fixture +def prepare_mini2(temp_dir, data_mini2): + records_dict2, dtype_dict = data_mini2 + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "records2", records_dict2, dtype_dict) + + +@pytest.fixture +def prepare_mini2_str(temp_dir, data_mini2_str): + records_dict2, dtype_dict = data_mini2_str + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "records2", records_dict2, dtype_dict) + + +@pytest.fixture +def prepare_phecode_def_mini(temp_dir): + phecode_definitions_dict = { + "index": [2367, 3789, 6065], + "phecode": ["247.72", "401", "557"], + "phecode_string": [ + "Streptococcus", + "Lichen planus, nitidus, or striatus", + "Family history of genetic condition", + ], + "phecode_category": ["ID", "ID", "Derm"], + "sex": ["Both", "Both", "Both"], + "ICD10_only": [0, 0, 0], + "phecode_top": [1, 2, 3], + "leaf": [0, 1, 1], + } + + with TemporaryDirectory(dir=temp_dir) as temp_dir: + TMP_PATH = os.path.join(temp_dir, "phecode_def.feather") + phecode_definitions_df = pd.DataFrame(phecode_definitions_dict) + phecode_definitions_df.to_feather(TMP_PATH) + yield TMP_PATH + + +def prepare_eids_yaml(temp_dir, eids): + TMP_PATH = os.path.join(temp_dir, "eids.yaml") + with open(TMP_PATH, "w") as eids_outfile: + yaml.dump(eids, eids_outfile, default_flow_style=False) + return TMP_PATH + + +@pytest.fixture +def prepare_eids_mini1(temp_dir): + eids_array = np.array([327, 578, 111]) + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_eids_yaml(temp_dir, eids_array) + + +@pytest.fixture +def prepare_eids_mini1_str(temp_dir): + eids_array = np.array(["a327", "b578", "c111"]) + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_eids_yaml(temp_dir, eids_array) + + +@pytest.fixture +def prepare_eids_mini2(temp_dir): + eids_array = np.array([181, 996, 733]) + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_eids_yaml(temp_dir, eids_array) + + +@pytest.fixture +def prepare_eids_mini2_str(temp_dir): + eids_array = np.array(["d181", "e996", "f733"]) + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_eids_yaml(temp_dir, eids_array) + + +@pytest.fixture +def prepare_record_freq_mini(temp_dir): + record_freq_dict = { + "record": [ + "OMOP_1548196", + "OMOP_4324588", + "OMOP_4060831", + "phecode_557", + "phecode_401", + "OMOP_4214579", + "OMOP_19020124", + "phecode_247.72", + ], + "n": [1, 1, 1, 1, 1, 1, 1, 1], + } + with TemporaryDirectory(dir=temp_dir) as temp_dir: + TMP_PATH = os.path.join(temp_dir, "record_freq.feather") + record_freq_df = pd.DataFrame(record_freq_dict) + record_freq_df.to_feather(TMP_PATH) + yield TMP_PATH + + +@pytest.fixture +def ehr_debug_eids(): + EHR_DEBUG_EIDS_PATH = os.path.join( + UDM_HOME, + "data/cvd/data/2_datasets_pre/211110_anewbeginning/artifacts/debug/eids.yaml", + ) + eids = yaml.load(EHR_DEBUG_EIDS_PATH, Loader=yaml.CSafeLoader) + return list(eids) + + +@pytest.fixture +def ehr_debug_eids_head(ehr_debug_eids): + return Head(ehr_debug_eids) + + +@pytest.fixture +def ehr_debug_three_eids(ehr_debug_eids): + return [ehr_debug_eids[x] for x in [111, 123, 124]] + + +@pytest.fixture +def ehr_debug_eid(ehr_debug_eids): + return ehr_debug_eids[101] + + +@pytest.fixture +def metadata_individuals_mini1(): + return { + "eid": [327, 578, 111], + "birth_date": [ + "1938-05-20", + "1949-08-07", + "1956-08-16", + ], + "recruitment_date": [ + "2002-03-14", + "2005-08-23", + "2010-01-07", + ], + "death_date": ["nat", "nat", "nat"], + "cens_data": [ + "2019-08-19", + "2019-08-19", + "2019-08-19", + ], + "exit_date": [ + "2019-08-19", + "2019-08-19", + "2019-08-19", + ], + } + + +@pytest.fixture +def metadata_individuals_mini1_str(metadata_individuals_mini1): + metadata = metadata_individuals_mini1 + metadata.update({"eid": ["a327", "b578", "c111"]}) + return metadata + + +def prepare_metadata_individuals(temp_dir, metadata_individuals): + TMP_PATH = os.path.join(temp_dir, "metadata_individuals.feather") + metdata_individuals_df = pd.DataFrame(metadata_individuals) + metdata_individuals_df.to_feather(TMP_PATH) + return TMP_PATH + + +@pytest.fixture +def prepare_metadata_individuals_mini1(temp_dir, metadata_individuals_mini1): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_metadata_individuals(temp_dir, metadata_individuals_mini1) + + +@pytest.fixture +def prepare_metadata_individuals_mini1_str(temp_dir, metadata_individuals_mini1_str): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_metadata_individuals(temp_dir, metadata_individuals_mini1_str) + + +@pytest.fixture +def metadata_individuals_mini2(): + return { + "eid": [181, 996, 733], + "birth_date": [ + "1942-01-07", + "1960-02-23", + "1962-10-25", + ], + "recruitment_date": [ + "1999-02-03", + "2019-12-21", + "2003-08-18", + ], + "death_date": ["2013-03-08", "2017-09-01", "nat"], + "cens_data": [ + "2013-03-08", + "2017-09-01", + "2019-08-19", + ], + "exit_date": [ + "2013-03-08", + "2017-09-01", + "2019-08-19", + ], + } + + +@pytest.fixture +def metadata_individuals_mini2_str(metadata_individuals_mini2): + metadata = metadata_individuals_mini2 + metadata.update({"eid": ["d181", "e996", "f733"]}) + return metadata + + +@pytest.fixture +def prepare_metadata_individuals_mini2(temp_dir, metadata_individuals_mini2): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_metadata_individuals(temp_dir, metadata_individuals_mini2) + + +@pytest.fixture +def prepare_metadata_individuals_mini2_str(temp_dir, metadata_individuals_mini2_str): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_metadata_individuals(temp_dir, metadata_individuals_mini2_str) + + +@pytest.fixture +def ehr_debug_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug.yaml", + overrides=[ + "min_record_counts=0", + "use_top_n_phecodes=1683", + "t0_mode=recruitment", + ], + ) + config.phecode_definitions_path = os.path.join( + config.data_path, config.phecode_definitions_path + ) + config.records_path = os.path.join(config.data_path, config.records_path) + config.records_frequencies_path = os.path.join( + config.data_path, config.records_frequencies_path + ) + config.metadata_individuals_path = os.path.join( + config.data_path, config.metadata_individuals_path + ) + yield config + + +@pytest.fixture +def ehr_debug_birth_date_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug.yaml", + overrides=[ + "min_record_counts=0", + "use_top_n_phecodes=10000", + "label_definition.all_cause_death=False", + "t0_mode=birth_date", + ], + ) + config.phecode_definitions_path = os.path.join( + config.data_path, config.phecode_definitions_path + ) + config.records_path = os.path.join(config.data_path, config.records_path) + config.records_frequencies_path = os.path.join( + config.data_path, config.records_frequencies_path + ) + config.metadata_individuals_path = os.path.join( + config.data_path, config.metadata_individuals_path + ) + yield config + + +@pytest.fixture +def ehr_mini_config( + prepare_mini1, + prepare_mini2, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1, +): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + TMP_PATH1, TMP_PATH2, TMP_PATH_PHE_DEF, TMP_PATH_REC_FRE, TMP_PATH_EIDS = ( + prepare_mini1, + prepare_mini2, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1, + ) + config = hydra.compose( + "debug.yaml", + overrides=[ + f"use_top_n_phecodes=2", + f"data_path=[{TMP_PATH1}, {TMP_PATH2}]", + f"eids_path={TMP_PATH_EIDS}", + f"phecode_definitions_path={TMP_PATH_PHE_DEF}", + f"records_frequencies_path={TMP_PATH_REC_FRE}", + ], + ) + yield config + + +@pytest.fixture +def ehr_mini1_config( + prepare_mini1, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1, + prepare_metadata_individuals_mini1, +): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + TMP_PATH, TMP_PATH_PHE_DEF, TMP_PATH_REC_FRE, TMP_PATH_EIDS, TMP_PATH_MET_IND = ( + prepare_mini1, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1, + prepare_metadata_individuals_mini1, + ) + config = hydra.compose( + "debug.yaml", + overrides=[ + f"records_path={TMP_PATH}", + f"eids_path={TMP_PATH_EIDS}", + f"phecode_definitions_path={TMP_PATH_PHE_DEF}", + f"records_frequencies_path={TMP_PATH_REC_FRE}", + f"metadata_individuals_path={TMP_PATH_MET_IND}", + f"use_top_n_phecodes=2", + f"drop_phecode_strings=[]", + ], + ) + yield config + + +@pytest.fixture +def ehr_mini1_str_config( + prepare_mini1_str, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1_str, + prepare_metadata_individuals_mini1_str, +): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + TMP_PATH, TMP_PATH_PHE_DEF, TMP_PATH_REC_FRE, TMP_PATH_EIDS, TMP_PATH_MET_IND = ( + prepare_mini1_str, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini1_str, + prepare_metadata_individuals_mini1_str, + ) + config = hydra.compose( + "debug.yaml", + overrides=[ + f"records_path={TMP_PATH}", + f"eids_path={TMP_PATH_EIDS}", + f"phecode_definitions_path={TMP_PATH_PHE_DEF}", + f"records_frequencies_path={TMP_PATH_REC_FRE}", + f"metadata_individuals_path={TMP_PATH_MET_IND}", + f"use_top_n_phecodes=2", + f"drop_phecode_strings=[]", + ], + ) + yield config + + +@pytest.fixture +def ehr_mini2_config( + prepare_mini2, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini2, + prepare_metadata_individuals_mini2, +): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + TMP_PATH, TMP_PATH_PHE_DEF, TMP_PATH_REC_FRE, TMP_PATH_EIDS, TMP_PATH_MET_IND = ( + prepare_mini2, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini2, + prepare_metadata_individuals_mini2, + ) + config = hydra.compose( + "debug.yaml", + overrides=[ + f"use_top_n_phecodes=2", + f"records_path={TMP_PATH}", + f"eids_path={TMP_PATH_EIDS}", + f"phecode_definitions_path={TMP_PATH_PHE_DEF}", + f"records_frequencies_path={TMP_PATH_REC_FRE}", + f"metadata_individuals_path={TMP_PATH_MET_IND}", + f"drop_phecode_strings=[]", + ], + ) + yield config + + +@pytest.fixture +def ehr_mini2_str_config( + prepare_mini2_str, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini2_str, + prepare_metadata_individuals_mini2_str, +): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + TMP_PATH, TMP_PATH_PHE_DEF, TMP_PATH_REC_FRE, TMP_PATH_EIDS, TMP_PATH_MET_IND = ( + prepare_mini2_str, + prepare_phecode_def_mini, + prepare_record_freq_mini, + prepare_eids_mini2_str, + prepare_metadata_individuals_mini2_str, + ) + config = hydra.compose( + "debug.yaml", + overrides=[ + f"use_top_n_phecodes=2", + f"records_path={TMP_PATH}", + f"eids_path={TMP_PATH_EIDS}", + f"phecode_definitions_path={TMP_PATH_PHE_DEF}", + f"records_frequencies_path={TMP_PATH_REC_FRE}", + f"metadata_individuals_path={TMP_PATH_MET_IND}", + f"drop_phecode_strings=[]", + ], + ) + yield config + + +@pytest.fixture +def ehr_full_wandb_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "ehr") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="from_wandb_default", + overrides=[ + "min_record_counts=0", + "use_top_n_phecodes=1683", + "t0_mode=recruitment", + "records_data_name=final_records_omop_subsampled:latest", + ], + ) + yield config + + +class TestEHRPlugin(object): + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("ehr_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + lazy_fixture("ehr_mini_config"), + lazy_fixture("ehr_mini1_config"), + lazy_fixture("ehr_mini1_str_config"), + lazy_fixture("ehr_mini2_config"), + lazy_fixture("ehr_mini2_str_config"), + ], + ) + def test__init__(self, config): + ehr_plugin = EHRPlugin.from_config(config) + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("ehr_full_wandb_config"), marks=pytest.mark.slow), + ], + ) + def test_from_wandb(self, config): + ehr_plugin = EHRPlugin.from_wandb(**config) + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("ehr_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + pytest.param(lazy_fixture("ehr_mini_config"), marks=pytest.mark.xfail), + lazy_fixture("ehr_mini1_config"), + lazy_fixture("ehr_mini1_str_config"), + lazy_fixture("ehr_mini2_config"), + lazy_fixture("ehr_mini2_str_config"), + pytest.param( + lazy_fixture("ehr_full_wandb_config"), + marks=[SKIPIF_NOT_BIH_HPC, pytest.mark.slow], + ), + ], + ) + def test_prepare_data(self, config): + if config["__init__"] == "from_wandb": + ehr_plugin = EHRPlugin.from_wandb(**config) + else: + ehr_plugin = EHRPlugin.from_config(config) + ehr_plugin.prepare_data() + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("ehr_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + pytest.param( + lazy_fixture("ehr_debug_birth_date_config"), marks=SKIPIF_NOT_BIH_HPC + ), + pytest.param(lazy_fixture("ehr_mini_config"), marks=pytest.mark.xfail), + lazy_fixture("ehr_mini1_config"), + lazy_fixture("ehr_mini1_str_config"), + lazy_fixture("ehr_mini2_config"), + lazy_fixture("ehr_mini2_str_config"), + pytest.param( + lazy_fixture("ehr_full_wandb_config"), + marks=[SKIPIF_NOT_BIH_HPC, pytest.mark.slow], + ), + ], + ) + def test_setup(self, config): + if config["__init__"] == "from_wandb": + ehr_plugin = EHRPlugin.from_wandb(**config) + else: + ehr_plugin = EHRPlugin.from_config(config) + ehr_plugin.prepare_data() + ehr_plugin.setup() + + @pytest.mark.parametrize( + "config, exp_eids", + [ + pytest.param( + lazy_fixture("ehr_debug_config"), + lazy_fixture("ehr_debug_eids_head"), + marks=SKIPIF_NOT_BIH_HPC, + ), + (lazy_fixture("ehr_mini1_config"), [327, 578, 111]), + (lazy_fixture("ehr_mini1_str_config"), ["a327", "b578", "c111"]), + (lazy_fixture("ehr_mini2_config"), [181, 733]), + (lazy_fixture("ehr_mini2_str_config"), ["d181", "f733"]), + pytest.param( + lazy_fixture("ehr_full_wandb_config"), + None, + marks=[SKIPIF_NOT_BIH_HPC, pytest.mark.slow], + ), + ], + ) + def test_get_eids(self, config, exp_eids): + if config["__init__"] == "from_wandb": + ehr_plugin = EHRPlugin.from_wandb(**config) + else: + ehr_plugin = EHRPlugin.from_config(config) + ehr_plugin.prepare_data() + ehr_plugin.setup() + eids = ehr_plugin.get_eids() + assert isinstance(eids, list) + if exp_eids is not None: + if isinstance(exp_eids, Head): + assert all([eid in eids for eid in exp_eids]) + else: + assert set(eids) == set(exp_eids) + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("ehr_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + lazy_fixture("ehr_mini1_config"), + lazy_fixture("ehr_mini2_config"), + pytest.param( + lazy_fixture("ehr_full_wandb_config"), + marks=[SKIPIF_NOT_BIH_HPC, pytest.mark.slow], + ), + ], + ) + def test_get_metadata(self, config): + if config["__init__"] == "from_wandb": + ehr_plugin = EHRPlugin.from_wandb(**config) + else: + ehr_plugin = EHRPlugin.from_config(config) + ehr_plugin.prepare_data() + ehr_plugin.setup() + metadata = ehr_plugin.get_metadata() + KEYS_METADATA = set(["eids", "features", "feature_types"]) + assert isinstance(metadata, dict) + assert set(metadata.keys()).intersection(KEYS_METADATA) == KEYS_METADATA + + DEFAULT_KEYS = [ + "records", + "censorings", + "exclusions", + "label_events", + "label_times", + ] + + @pytest.mark.parametrize( + "config, eids, features, subset, exp_item", + [ + pytest.param( + lazy_fixture("ehr_debug_config"), + lazy_fixture("ehr_debug_eid"), + None, + None, + [*DEFAULT_KEYS, "label_auxiliary_events", "label_auxiliary_times"], + marks=[pytest.mark.slow, SKIPIF_NOT_BIH_HPC], + ), + pytest.param( + lazy_fixture("ehr_debug_config"), + lazy_fixture("ehr_debug_three_eids"), + None, + None, + DEFAULT_KEYS, + marks=[pytest.mark.slow, SKIPIF_NOT_BIH_HPC], + ), + pytest.param( + lazy_fixture("ehr_full_wandb_config"), + lazy_fixture("ehr_debug_eid"), + None, + None, + DEFAULT_KEYS, + marks=[pytest.mark.slow, SKIPIF_NOT_BIH_HPC], + ), + ( + lazy_fixture("ehr_mini1_config"), + 327, + None, + None, + {"censorings": torch.tensor([[17.432254791259766]])}, + ), + ( + lazy_fixture("ehr_mini1_str_config"), + "a327", + None, + None, + {"censorings": torch.tensor([[17.432254791259766]])}, + ), + ( + lazy_fixture("ehr_mini1_config"), + [111, 578, 327], + None, + None, + { + "records": torch.tensor( + [ + [0.0, 0.0, 0.0, 0.0, 1.0, 1.0], + [0.0, 1.0, 0.0, 1.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ), + "censorings": torch.tensor( + [ + [9.612791061401367], + [13.987966537475586], + [17.432254791259766], + ] + ), + "exclusions": torch.tensor( + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ), + "label_events": torch.tensor( + [[1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ), + "label_times": torch.tensor( + [ + [9.614636421203613, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + ] + ), + }, + ), + ( + lazy_fixture("ehr_mini1_config"), + [111, 578, 327], + { + "records": (["phecode_247.72", "phecode_557"],), + "censorings": (["censored"],), + "exclusions": ([True, True, False],), + "label_events": ( + torch.tensor([True, True, False], dtype=torch.bool), + ), + "label_times": ([True, True, False],), + }, + None, + { + "records": torch.tensor([[1.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "censorings": torch.tensor( + [ + [9.612791061401367], + [13.987966537475586], + [17.432254791259766], + ] + ), + "exclusions": torch.tensor([[0.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "label_events": torch.tensor([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0]]), + "label_times": torch.tensor( + [[9.614636421203613, 0.0], [0.0, 0.0], [0.0, 0.0]] + ), + }, + ), + ( + lazy_fixture("ehr_mini1_config"), + [111, 578, 327], + { + "records": (["phecode_247.72", "phecode_557"],), + "exclusions": ([True, True, False],), + "label_times": ([True, True, False],), + }, + None, + { + "records": torch.tensor([[1.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "exclusions": torch.tensor([[0.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "label_times": torch.tensor( + [[9.614636421203613, 0.0], [0.0, 0.0], [0.0, 0.0]] + ), + }, + ), + ( + lazy_fixture("ehr_mini1_str_config"), + ["c111", "b578", "a327"], + { + "records": (["phecode_247.72", "phecode_557"],), + "exclusions": ([True, True, False],), + "label_times": ([True, True, False],), + }, + None, + { + "records": torch.tensor([[1.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "exclusions": torch.tensor([[0.0, 1.0], [0.0, 0.0], [0.0, 0.0]]), + "label_times": torch.tensor( + [[9.614636421203613, 0.0], [0.0, 0.0], [0.0, 0.0]] + ), + }, + ), + (lazy_fixture("ehr_mini2_config"), 733, None, None, None), + ( + # 996 should be filtered + lazy_fixture("ehr_mini2_config"), + [181, 996, 733], + None, + None, + { + "censorings": torch.tensor( + [[14.092007637023926], [16.003067016601562]] + ) + }, + ), + ( + # e996 should be filtered + lazy_fixture("ehr_mini2_str_config"), + ["d181", "e996", "f733"], + None, + None, + { + "censorings": torch.tensor( + [[14.092007637023926], [16.003067016601562]] + ) + }, + ), + ( + lazy_fixture("ehr_mini2_config"), + [181, 733], + None, + None, + { + "records": torch.tensor( + [[0.0, 0.0, 0.0, 0.0, 1.0], [1.0, 0.0, 0.0, 1.0, 0.0]] + ), + "censorings": torch.tensor( + [[14.092007637023926], [16.003067016601562]] + ), + "exclusions": torch.tensor([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]), + "label_events": torch.tensor([[1.0, 0.0, 0.0], [0.0, 0.0, 0.0]]), + "label_times": torch.tensor( + [[14.092129707336426, 0.0, 0.0], [0.0, 0.0, 0.0]] + ), + }, + ), + ( + lazy_fixture("ehr_mini2_config"), + [181, 733], + { + "records": (["OMOP_19020124", "OMOP_4214579", "OMOP_4306655"],), + "censorings": (["censored"],), + "exclusions": ( + torch.tensor([True, False, True], dtype=torch.bool), + ), + "label_events": ( + torch.tensor([True, False, False], dtype=torch.bool), + ), + "label_times": ([True, False, False],), + }, + None, + { + "records": torch.tensor([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0]]), + "censorings": torch.tensor( + [[14.092007637023926], [16.003067016601562]] + ), + "exclusions": torch.tensor([[0.0, 0.0], [0.0, 1.0]]), + "label_events": torch.tensor([[1.0], [0.0]]), + "label_times": torch.tensor([[14.092129707336426], [0.0]]), + }, + ), + ( + lazy_fixture("ehr_mini1_config"), + 327, + None, + ["label_events"], + ["label_events"], + ), + ( + lazy_fixture("ehr_mini1_config"), + [327, 578, 111], + None, + ["label_events"], + ["label_events"], + ), + ], + ) + def test__getitem__(self, config, eids, features, subset, exp_item): + if config["__init__"] == "from_wandb": + ehr_plugin = EHRPlugin.from_wandb(**config) + else: + ehr_plugin = EHRPlugin.from_config(config) + ehr_plugin.prepare_data() + ehr_plugin.setup() + if isinstance(eids, int): + item = ehr_plugin[eids, features] + else: + item = ehr_plugin[eids, features] + assert isinstance(item, dict) + if isinstance(exp_item, set): + assert set(item.keys()) == exp_item + elif isinstance(exp_item, dict): + for key in exp_item.keys(): + assert torch.allclose(item[key], exp_item[key]) diff --git a/test/plugins/genetic/__init__.py b/test/plugins/genetic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/genetic/genetic_plugin_test.py b/test/plugins/genetic/genetic_plugin_test.py new file mode 100644 index 0000000..525d663 --- /dev/null +++ b/test/plugins/genetic/genetic_plugin_test.py @@ -0,0 +1,738 @@ +import copy +import inspect +import os +from os.path import abspath, dirname +import torch +import hydra +import numpy as np +import pandas as pd +import pytest +from pytest_lazyfixture import lazy_fixture +import pickle +from tempfile import TemporaryDirectory +from omegaconf import OmegaConf, DictConfig + +from udm.plugins.genetic.genetic_plugin import GeneticPlugin +from test.utils import SKIPIF_NOT_BIH_HPC, try_clear_hydra + +UDM_HOME = "../../.." + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def mini_train_data(): + # Train data + X = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0]], dtype=np.uint8) + obs = pd.DataFrame({"index": [0, 1, 2]}) + var = pd.DataFrame( + { + "Name": ["snp1", "snp2", "snp3"], + "ID": ["rs1", "rs2", "rs3"], + "GeneticAlt": ["G", "A", "G"], + "Field": ["ENSG001", "ENSG002", "ENSG003"], + } + ) + return X, obs, var + + +@pytest.fixture +def mini_train_str_data(mini_train_data): + X, _, var = mini_train_data + obs = pd.DataFrame({"index": ["a1", "b2", "c3"]}) + return X, obs, var + + +def prepare_data(temp_dir, name, X, obs, var): + basename = os.path.join(temp_dir, name) + with open(f"{basename}.X.npy", "wb+") as f: + data_mmap = np.memmap(f, dtype=np.uint8, mode="w+", shape=(3, 3)) + data_mmap[:, :] = X[:, :] + data_mmap.flush() + obs.to_csv(f"{basename}.obs.csv", index=False) + var.to_csv(f"{basename}.var.csv") + return basename + + +@pytest.fixture +def prepare_mini_train(temp_dir, mini_train_data): + X, obs, var = mini_train_data + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_data(temp_dir, "train", X, obs, var) + + +@pytest.fixture +def prepare_mini_train_str(temp_dir, mini_train_str_data): + X, obs, var = mini_train_str_data + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_data(temp_dir, "train", X, obs, var) + + +@pytest.fixture +def prepare_mini_train_uns(prepare_mini_train): + basename = prepare_mini_train + uns = {"dtype": "uint8"} + with open(f"{basename}.uns.pkl", "wb+") as f: + pickle.dump(uns, f) + yield basename + + +@pytest.fixture +def prepare_mini_valid(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + # Valid data + valid_data = np.array([[1, 0, 0], [1, 0, 1], [1, 1, 0]]) + obs = pd.DataFrame({"index": [3, 4, 6]}) + var = pd.DataFrame( + { + "Name": ["snp1", "snp2", "snp3"], + "ID": ["rs1", "rs2", "rs3"], + "GeneticAlt": ["G", "A", "G"], + "Field": ["ENSG001", "ENSG002", "ENSG003"], + } + ) + yield prepare_data(temp_dir, "valid", valid_data, obs, var) + + +@pytest.fixture +def prepare_mini_valid_uns(prepare_mini_valid): + basename = prepare_mini_valid + uns = {"dtype": "uint8"} + with open(f"{basename}.uns.pkl", "wb+") as f: + pickle.dump(uns, f) + yield basename + + +@pytest.fixture +def geno_debug_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + UDM_HOME_ABSPATH = abspath(os.path.join(dirname(__file__), UDM_HOME)) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + overrides_config = OmegaConf.create( + { + "src": [ + "genetic_extended_tr", + "genetic_valid", + "genetic_test", + ], + "src_names": ["train", "valid", "test"], + "dataset_path": FAKE_DATA_PATH, + } + ) + config = hydra.compose(config_name="default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def geno_debug_nomap_config(geno_debug_config): + config = geno_debug_config + config.update({"eid_map_path": None}) + yield config + + +@pytest.fixture +def mini_train_config(prepare_mini_train): + try_clear_hydra() + basename_train = prepare_mini_train + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + overrides_config = OmegaConf.create( + { + "src": [basename_train], + "src_names": ["train"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def mini_train_str_config(prepare_mini_train_str): + try_clear_hydra() + basename_train = prepare_mini_train_str + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + overrides_config = OmegaConf.create( + { + "src": [basename_train], + "src_names": ["train"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def mini_train_uns_config(prepare_mini_train_uns): + try_clear_hydra() + basename_train = prepare_mini_train_uns + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + overrides_config = OmegaConf.create( + { + "src": [basename_train], + "src_names": ["train"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def plugin_trafo_config(): + trafo_cfg = DictConfig( + { + "name": "PluginTransform", + "transforms": { + "data": { + "name": "ListTransform", + "transforms": [ + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + ], + } + }, + } + ) + yield trafo_cfg + + +@pytest.fixture +def mini_train_trafo_config(mini_train_config, plugin_trafo_config): + config = mini_train_config + config.transforms = plugin_trafo_config + yield config + + +@pytest.fixture +def mini_valid_config(prepare_mini_valid): + try_clear_hydra() + basename_valid = prepare_mini_valid + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + overrides_config = OmegaConf.create( + { + "src": [basename_valid], + "src_names": ["valid"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def mini_valid_uns_config(prepare_mini_valid_uns): + try_clear_hydra() + basename_valid = prepare_mini_valid_uns + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + overrides_config = OmegaConf.create( + { + "src": [basename_valid], + "src_names": ["valid"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def mini_valid_trafo_config(mini_valid_config, plugin_trafo_config): + config = mini_valid_config + config.transforms = plugin_trafo_config + yield config + + +@pytest.fixture +def mini_train_valid_config(temp_dir, prepare_mini_train, prepare_mini_valid): + try_clear_hydra() + basename_train = prepare_mini_train + basename_valid = prepare_mini_valid + config_path = os.path.join(UDM_HOME, "config", "plugins", "genetic") + hydra.initialize(version_base=None, config_path=config_path) + with TemporaryDirectory(dir=temp_dir) as temp_dir: + overrides_config = OmegaConf.create( + { + "src": [basename_train, basename_valid], + "src_names": ["train", "valid"], + "eid_map_path": None, + "transforms": None, + } + ) + config = hydra.compose("default.yaml", overrides=[]) + config = OmegaConf.merge(config, overrides_config) + yield config + + +@pytest.fixture +def mini_train_valid_map_config(temp_dir, mini_train_valid_config): + config = mini_train_valid_config + eid_map = pd.DataFrame( + {"eid_map_from": [0, 1, 2, 3, 4, 6], "eid_map_to": [5, 6, 7, 8, 9, 10]} + ) + eid_map_path = f"{temp_dir}/eid_map.csv" + eid_map.to_csv(eid_map_path, sep="\t") + config.update( + { + "eid_map_path": f"{temp_dir}/eid_map.csv", + "eid_map_from": "eid_map_from", + "eid_map_to": "eid_map_to", + } + ) + yield config + + +class TestGeneticPlugin(object): + + cls = GeneticPlugin + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("geno_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + lazy_fixture("mini_train_config"), + lazy_fixture("mini_train_str_config"), + lazy_fixture("mini_train_uns_config"), + lazy_fixture("mini_train_trafo_config"), + lazy_fixture("mini_valid_config"), + lazy_fixture("mini_valid_uns_config"), + lazy_fixture("mini_valid_trafo_config"), + ], + ) + def test__init__(self, config): + genetic_plugin = self.cls(**config) + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("geno_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + lazy_fixture("mini_train_config"), + lazy_fixture("mini_train_str_config"), + lazy_fixture("mini_train_uns_config"), + lazy_fixture("mini_train_trafo_config"), + lazy_fixture("mini_valid_config"), + lazy_fixture("mini_valid_uns_config"), + lazy_fixture("mini_valid_trafo_config"), + ], + ) + def test_prepare_data(self, config): + genetic_plugin = self.cls(**config) + genetic_plugin.prepare_data() + del genetic_plugin + + @pytest.mark.parametrize( + "config", + [ + pytest.param(lazy_fixture("geno_debug_config"), marks=SKIPIF_NOT_BIH_HPC), + lazy_fixture("mini_train_config"), + lazy_fixture("mini_train_str_config"), + lazy_fixture("mini_train_uns_config"), + lazy_fixture("mini_train_trafo_config"), + lazy_fixture("mini_valid_config"), + lazy_fixture("mini_valid_uns_config"), + lazy_fixture("mini_valid_trafo_config"), + ], + ) + def test_setup(self, config): + genetic_plugin = self.cls(**config) + genetic_plugin.prepare_data() + genetic_plugin.setup() + del genetic_plugin + + @pytest.mark.parametrize( + "config, setup, eid, features, exp_item", + [ + (lazy_fixture("mini_train_valid_config"), False, 0, None, RuntimeError), + *[ + (lazy_fixture("mini_train_config"), True, eid, None, exp_item) + for eid, exp_item in zip( + [0, 1, 2], + [ + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + ], + ) + ], + *[ + (lazy_fixture("mini_train_str_config"), True, eid, None, exp_item) + for eid, exp_item in zip( + ["a1", "b2", "c3"], + [ + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + ], + ) + ], + *[ + (lazy_fixture("mini_train_valid_config"), True, eid, None, exp_item) + for eid, exp_item in zip( + [0, 1, 2, 3, 4, 6], + [ + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ], + ) + ], + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([0, 2], dtype=torch.int64), + None, + np.array([[0, 1, 0], [1, 0, 0]]), + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([0, 2], dtype=torch.int64), + {"genetic": (torch.tensor([True, True, False], dtype=torch.bool),)}, + np.array([[0, 1], [1, 0]]), + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([0, 2], dtype=torch.int64), + {"genetic": (["rs1", "rs2"],)}, + np.array([[0, 1], [1, 0]]), + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([1, 4], dtype=torch.int64), + None, + np.array( + [ + [0, 0, 1], + [1, 0, 1], + ] + ), + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([1, 4], dtype=torch.int64), + {"genetic": (torch.tensor([True, False, True], dtype=torch.bool),)}, + np.array( + [ + [0, 1], + [1, 1], + ] + ), + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + torch.tensor([1, 4], dtype=torch.int64), + {"genetic": (["rs1", "rs3"],)}, + np.array( + [ + [0, 1], + [1, 1], + ] + ), + ), + *[ + (lazy_fixture("mini_train_valid_map_config"), True, eid, None, exp_item) + for eid, exp_item in zip( + [5, 6, 7, 8, 9, 10], + [ + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ], + ) + ], + ], + ) + def test__getitem__(self, config, setup, eid, features, exp_item): + # genetic_plugin = self.cls(**config) + genetic_plugin = self.cls.from_config(config) + if setup: + genetic_plugin.prepare_data() + genetic_plugin.setup() + if inspect.isclass(exp_item) and issubclass(exp_item, Exception): + with pytest.raises(RuntimeError): + genetic_plugin[eid, features] + else: + item = copy.deepcopy(genetic_plugin[eid, features]) + # .clone() is required because otherweise a view of the dataset is returned + assert isinstance(item, dict) + assert list(item.keys()) == ["genetic"] + assert np.all(item["genetic"].numpy() == exp_item) + del genetic_plugin + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_len", + [ + (lazy_fixture("mini_train_config"), True, True, 3), + (lazy_fixture("mini_train_str_config"), True, True, 3), + pytest.param( + lazy_fixture("geno_debug_config"), + False, + False, + 0, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + False, + 0, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + True, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + ], + ) + def test__len__(self, config, prepare_data, setup, exp_len): + genetic_plugin = self.cls(**config) + if exp_len is None: + obs_lens = [] + for s_file in config.src: + obs_train_path = os.path.join(config.dataset_path, f"{s_file}.obs.csv") + obs_df = pd.read_csv(obs_train_path) + obs_lens += [len(obs_df)] + # obs_valid_path = os.path.join( + # config.dataset_path, "genetic_valid.obs.csv" + # ) + # obs_valid = pd.read_csv(obs_valid_path) + # obs_test_path = os.path.join( + # config.dataset_path, "genetic_test.obs.csv" + # ) + # obs_test = pd.read_csv(obs_test_path) + exp_len = sum(obs_lens) + if prepare_data: + genetic_plugin.prepare_data() + if setup: + genetic_plugin.setup() + assert len(genetic_plugin) == exp_len + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_eids", + [ + pytest.param( + lazy_fixture("geno_debug_config"), + False, + False, + [], + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + False, + [], + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + True, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + (lazy_fixture("mini_train_config"), True, True, [0, 1, 2]), + (lazy_fixture("mini_train_str_config"), True, True, ["a1", "b2", "c3"]), + (lazy_fixture("mini_train_valid_config"), True, True, [0, 1, 2, 3, 4, 6]), + ( + lazy_fixture("mini_train_valid_map_config"), + True, + True, + [0, 1, 2, 3, 4, 6], + ), + ], + ) + def test_get_native_eids(self, config, prepare_data, setup, exp_eids): + if exp_eids is None: + eids_train = pd.read_csv( + f"{config.dataset_path}/genetic_extended_tr.obs.csv" + )["index"].to_list() + eids_valid = pd.read_csv(f"{config.dataset_path}/genetic_valid.obs.csv")[ + "index" + ].to_list() + eids_test = pd.read_csv(f"{config.dataset_path}/genetic_test.obs.csv")[ + "index" + ].to_list() + exp_eids = [*eids_train, *eids_valid, *eids_test] + genetic_plugin = self.cls(**config) + if prepare_data: + genetic_plugin.prepare_data() + if setup: + genetic_plugin.setup() + eids = genetic_plugin.get_native_eids() + assert isinstance(eids, list) + assert set(eids) == set(exp_eids) + assert eids == exp_eids + del genetic_plugin + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_eids", + [ + pytest.param( + lazy_fixture("geno_debug_config"), + False, + False, + {}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + False, + {}, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + True, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + ( + lazy_fixture("mini_train_valid_config"), + True, + True, + {"train": [0, 1, 2], "valid": [3, 4, 6]}, + ), + ( + lazy_fixture("mini_train_valid_map_config"), + True, + True, + {"train": [0, 1, 2], "valid": [3, 4, 6]}, + ), + ], + ) + def test_get_native_eids_as_struct(self, config, prepare_data, setup, exp_eids): + if exp_eids is None: + eids_train = pd.read_csv( + f"{config.dataset_path}/genetic_extended_tr.obs.csv" + )["index"].to_list() + eids_valid = pd.read_csv(f"{config.dataset_path}/genetic_valid.obs.csv")[ + "index" + ].to_list() + eids_test = pd.read_csv(f"{config.dataset_path}/genetic_test.obs.csv")[ + "index" + ].to_list() + exp_eids = { + "train": eids_train, + "valid": eids_valid, + "test": eids_test, + } + genetic_plugin = self.cls(**config) + if prepare_data: + genetic_plugin.prepare_data() + if setup: + genetic_plugin.setup() + eids = genetic_plugin.get_native_eids_as_struct() + assert isinstance(eids, dict) + assert set(eids.keys()) == set(exp_eids.keys()) + for key in eids: + assert isinstance(eids[key], list) + assert set(eids[key]) == set(exp_eids[key]) + assert eids[key] == exp_eids[key] + assert eids == exp_eids + del genetic_plugin + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_eids", + [ + (lazy_fixture("mini_train_config"), True, True, [0, 1, 2]), + (lazy_fixture("mini_train_str_config"), True, True, ["a1", "b2", "c3"]), + pytest.param( + lazy_fixture("geno_debug_config"), + False, + False, + [], + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + False, + [], + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("geno_debug_config"), + True, + True, + None, + marks=SKIPIF_NOT_BIH_HPC, + ), + (lazy_fixture("geno_debug_nomap_config"), True, True, None), + ], + ) + def test_get_eids(self, config, prepare_data, setup, exp_eids): + genetic_plugin = self.cls(**config) + if prepare_data: + genetic_plugin.prepare_data() + if setup: + genetic_plugin.setup() + eids = genetic_plugin.get_eids() + if exp_eids is None: + if config.eid_map_path is None: + exp_eids = genetic_plugin.get_native_eids() + else: + eid_map = pd.read_csv(config.eid_map_path, sep="\t") + eid_map = dict(zip(eid_map["EID.49966"], eid_map["EID.51157"])) + exp_eids = [eid_map[eid] for eid in genetic_plugin.get_native_eids()] + assert isinstance(eids, list) + assert set(eids) == set(exp_eids) + assert eids == exp_eids + + @pytest.mark.parametrize( + "config, exp_keys", + [ + pytest.param( + lazy_fixture("geno_debug_config"), + ["eids", "features", "feature_types"], + marks=SKIPIF_NOT_BIH_HPC, + ) + ], + ) + def test_get_metadata(self, config, exp_keys): + eid_map_path = "data/analysis/UKBB.application.link.file.20210608.txt" + eid_map = pd.read_csv(eid_map_path, sep="\t") + eid_map = dict(zip(eid_map["EID.49966"], eid_map["EID.51157"])) + genetic_plugin = self.cls(**config) + genetic_plugin.prepare_data() + genetic_plugin.setup() + metadata = genetic_plugin.get_metadata() + METADATA_KEYS = set(exp_keys) + assert isinstance(metadata, dict) + assert set(metadata.keys()).intersection(METADATA_KEYS) == METADATA_KEYS diff --git a/test/plugins/genetic/genetic_ts_plugin_test.py b/test/plugins/genetic/genetic_ts_plugin_test.py new file mode 100644 index 0000000..d17ed03 --- /dev/null +++ b/test/plugins/genetic/genetic_ts_plugin_test.py @@ -0,0 +1,734 @@ +import os +import pytest +from typing import Tuple, List +import json +from inspect import isclass +from addict import Dict +from pytest_lazyfixture import lazy_fixture +import numpy as np +import torch +import tensorstore as ts +from omegaconf import DictConfig, OmegaConf +from tempfile import TemporaryDirectory + +from udm.plugins.genetic.genetic_ts_plugin import ( + GeneticTSPlugin, + compress_mask, + decompress_mask, + compress_uint4, + compress_uint4_rot, + probs_to_int, +) + + +def get_ts_write_config( + dimensions: Tuple[int], + block_size: Tuple[int], + path: str = ".", + cname: str = "lz4", + clevel: int = 9, + dtype: str = "uint8", + **kwargs, +) -> Dict: + assert len(dimensions) == len(block_size) + config = Dict( + { + "driver": "n5", + "kvstore": { + "driver": "file", + "path": path, + }, + "metadata": { + "compression": { + "type": "blosc", + "cname": cname, + "clevel": clevel, + "shuffle": 0, + }, + "dataType": dtype, + "dimensions": dimensions, + "blockSize": block_size, + }, + "create": True, + "delete_existing": True, + } + ) + return config + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +def generate_dataset( + src_path, + config, + data, + eids: List[int] = None, + variants: List[str] = None, + dtype: str = None, +): + variants = variants or [f"var{i+1}" for i in range(data.shape[1])] + eids = eids or [*range(data.shape[0])] + dtype = dtype or str(data.dtype) + store = ts.open(config).result() + store[:].write(data).result() + meta_path = os.path.join(src_path, "attributes.json") + with open(meta_path, "r") as f: + metadata = json.load(f) + metadata["custom"] = {"eids": eids, "variants": variants, "dtype": dtype} + with open(meta_path, "w+") as f: + json.dump(metadata, f) + + +@pytest.fixture +def base_config(temp_dir): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + "as_dosages": True, + } + ) + data = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0]], dtype=np.float32) + block_size = (1, 1) + ts_config = get_ts_write_config( + data.shape, block_size, src_path, dtype=str(data.dtype) + ) + generate_dataset(src_path, ts_config, data) + yield config + + +@pytest.fixture +def probs_data(): + data = np.array( + [ + [[0.7, 0.3], [0.5, 0.2], [0.8, 0.1]], + [[0.4, 0.4], [0.6, 0.3], [0.55, 0.23]], + [[0.66, 0.33], [0.8, 0.2], [0.7, 0.3]], + ], + dtype=np.float32, + ) + return data + + +@pytest.fixture +def dosage_data(): + data = np.array( + [[0.5, 1.8, 0.2], [0.2, 0.3, 0.1], [1.2, 1.1, 0.9]], dtype=np.float32 + ) + return data + + +@pytest.fixture +def probs_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + } + ) + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + probs_data.shape, block_size, src_path, dtype=str(probs_data.dtype) + ) + generate_dataset(src_path, ts_config, probs_data) + yield config + + +@pytest.fixture +def probs_str_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + } + ) + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + probs_data.shape, block_size, src_path, dtype=str(probs_data.dtype) + ) + eids = ["a1", "b2", "c3"] + generate_dataset(src_path, ts_config, probs_data, eids=eids) + yield config + + +@pytest.fixture +def probs3_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + } + ) + probs3_data = np.empty([*probs_data.shape[:2], 3], dtype=probs_data.dtype) + probs3_data[:, :, :2] = probs_data + probs3_data[:, :, 2] = 1 - probs3_data[:, :, 0] - probs3_data[:, :, 1] + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + probs3_data.shape, block_size, src_path, dtype=str(probs3_data.dtype) + ) + generate_dataset(src_path, ts_config, probs3_data) + yield config + + +@pytest.fixture +def probs3_str_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + } + ) + probs3_data = np.empty([*probs_data.shape[:2], 3], dtype=probs_data.dtype) + probs3_data[:, :, :2] = probs_data + probs3_data[:, :, 2] = 1 - probs3_data[:, :, 0] - probs3_data[:, :, 1] + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + probs3_data.shape, block_size, src_path, dtype=str(probs3_data.dtype) + ) + eids = ["a1", "b2", "c3"] + generate_dataset(src_path, ts_config, probs3_data, eids=eids) + yield config + + +@pytest.fixture +def dosage_uint8_config(temp_dir, dosage_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + "as_dosages": True, + } + ) + uint8_data = probs_to_int(dosage_data / 2, "uint8") + block_size = (1, 2) + ts_config = get_ts_write_config( + uint8_data.shape, block_size, src_path, dtype=str(uint8_data.dtype) + ) + generate_dataset( + src_path, + ts_config, + uint8_data, + variants=[f"var{i+1}" for i in range(dosage_data.shape[1])], + dtype="uint8", + ) + yield config + + +@pytest.fixture +def probs3_uint8_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + "as_dosages": True, + } + ) + probs3_data = np.empty([*probs_data.shape[:2], 3], dtype=probs_data.dtype) + probs3_data[:, :, :2] = probs_data + probs3_data[:, :, 2] = 1 - probs3_data[:, :, 0] - probs3_data[:, :, 1] + probs3_data = probs_to_int(probs3_data, "uint8") + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + probs3_data.shape, block_size, src_path, dtype=str(probs3_data.dtype) + ) + generate_dataset(src_path, ts_config, probs3_data) + yield config + + +@pytest.fixture +def dosage_uint4_config(temp_dir, dosage_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + "as_dosages": True, + } + ) + dosage_data = probs_to_int(dosage_data / 2, "uint4") + uint4_data, remainder = compress_uint4_rot(dosage_data) + block_size = (1, 2) + ts_config = get_ts_write_config( + uint4_data.shape, block_size, src_path, dtype=str(uint4_data.dtype) + ) + generate_dataset( + src_path, + ts_config, + uint4_data, + variants=[f"var{i+1}" for i in range(dosage_data.shape[1])], + dtype="uint4", + ) + yield config + + +@pytest.fixture +def probs_uint4_config(temp_dir, probs_data): + with TemporaryDirectory(dir=temp_dir) as src_path: + config = DictConfig( + { + "src": os.path.basename(src_path), + "dataset_path": temp_dir, + "as_dosages": True, + } + ) + probs_data = probs_to_int(probs_data, "uint4") + uint4_data = compress_uint4(probs_data[:, :, :1], probs_data[:, :, 1:]) + block_size = (1, 2, 1) + ts_config = get_ts_write_config( + uint4_data.shape, block_size, src_path, dtype=str(uint4_data.dtype) + ) + generate_dataset( + src_path, + ts_config, + uint4_data, + variants=[f"var{i}" for i in range(probs_data.shape[1])], + dtype="uint4", + ) + yield config + + +@pytest.mark.parametrize( + "mask, exp_mask", + [ + (np.array([0], dtype=bool), np.array([0], dtype=bool)), + (np.array([1], dtype=bool), np.array([1], dtype=bool)), + (np.array([0, 0], dtype=bool), np.array([0], dtype=bool)), + (np.array([0, 1], dtype=bool), np.array([1], dtype=bool)), + (np.array([1, 0], dtype=bool), np.array([1], dtype=bool)), + (np.array([1, 1], dtype=bool), np.array([1], dtype=bool)), + (np.array([0, 0, 0], dtype=bool), np.array([0, 0], dtype=bool)), + (np.array([0, 1, 0], dtype=bool), np.array([1, 0], dtype=bool)), + (np.array([1, 0, 0], dtype=bool), np.array([1, 0], dtype=bool)), + (np.array([1, 1, 0], dtype=bool), np.array([1, 0], dtype=bool)), + (np.array([0, 0, 1], dtype=bool), np.array([0, 1], dtype=bool)), + (np.array([0, 1, 1], dtype=bool), np.array([1, 1], dtype=bool)), + (np.array([1, 0, 1], dtype=bool), np.array([1, 1], dtype=bool)), + (np.array([1, 1, 1], dtype=bool), np.array([1, 1], dtype=bool)), + ( + np.array([0, 0, 0, 1, 1, 0, 1, 1, 1], dtype=bool), + np.array([0, 1, 1, 1, 1], dtype=bool), + ), + ], +) +def test_compress_mask(mask, exp_mask): + mask_, remainder = compress_mask(mask) + assert np.array_equal(mask_, exp_mask) + + +@pytest.mark.parametrize( + "mask, remainder, exp_mask", + [ + (np.array([0], dtype=bool), 0, np.array([0, 0], dtype=bool)), + (np.array([1], dtype=bool), 0, np.array([1, 1], dtype=bool)), + (np.array([0], dtype=bool), 1, np.array([0], dtype=bool)), + (np.array([1], dtype=bool), 1, np.array([1], dtype=bool)), + (np.array([0, 0], dtype=bool), 0, np.array([0, 0, 0, 0], dtype=bool)), + (np.array([0, 1], dtype=bool), 0, np.array([0, 0, 1, 1], dtype=bool)), + (np.array([1, 0], dtype=bool), 0, np.array([1, 1, 0, 0], dtype=bool)), + (np.array([1, 1], dtype=bool), 0, np.array([1, 1, 1, 1], dtype=bool)), + (np.array([0, 0], dtype=bool), 1, np.array([0, 0, 0], dtype=bool)), + (np.array([0, 1], dtype=bool), 1, np.array([0, 0, 1], dtype=bool)), + (np.array([1, 0], dtype=bool), 1, np.array([1, 1, 0], dtype=bool)), + (np.array([1, 1], dtype=bool), 1, np.array([1, 1, 1], dtype=bool)), + ( + np.array([0, 1, 0, 0, 1, 1], dtype=bool), + 0, + np.array([0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1], dtype=bool), + ), + ( + np.array([0, 1, 0, 0, 1, 1], dtype=bool), + 1, + np.array([0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1], dtype=bool), + ), + ], +) +def test_decompress_mask(mask, remainder, exp_mask): + mask_ = decompress_mask(mask, remainder) + assert np.array_equal(mask_, exp_mask) + + +class TestGeneticTSPlugin: + @pytest.mark.parametrize( + "config, overrides, error", + [ + (lazy_fixture("base_config"), {}, None), + (lazy_fixture("base_config"), {"format": "probs"}, ValueError), + (lazy_fixture("base_config"), {"format": "probs3"}, ValueError), + (lazy_fixture("probs_config"), {}, None), + (lazy_fixture("probs_config"), {"format": "probs"}, None), + (lazy_fixture("probs_config"), {"format": "probs3"}, None), + (lazy_fixture("probs_str_config"), {}, None), + (lazy_fixture("probs3_config"), {}, None), + (lazy_fixture("probs3_config"), {"format": "probs"}, None), + (lazy_fixture("probs3_config"), {"format": "probs3"}, None), + ], + ) + def test__init__(self, config, overrides, error): + if overrides: + overrides = DictConfig(overrides) + config = OmegaConf.merge(config, overrides) + + if error: + if issubclass(error, Warning): + with pytest.warns(error): + GeneticTSPlugin(**config) + elif issubclass(error, Exception): + with pytest.raises(error): + GeneticTSPlugin(**config) + else: + plugin = GeneticTSPlugin(**config) + + @pytest.mark.parametrize( + "config, overrides, error", + [ + (lazy_fixture("base_config"), {}, None), + (lazy_fixture("probs_config"), {}, None), + (lazy_fixture("probs_config"), {"format": "probs"}, None), + (lazy_fixture("probs_config"), {"format": "probs3"}, None), + (lazy_fixture("probs_str_config"), {}, None), + (lazy_fixture("probs3_config"), {}, None), + (lazy_fixture("probs3_config"), {"format": "probs"}, None), + (lazy_fixture("probs3_config"), {"format": "probs3"}, None), + ], + ) + def test_prepare_data(self, config, overrides, error): + if overrides: + overrides = DictConfig(overrides) + config = OmegaConf.merge(config, overrides) + plugin = GeneticTSPlugin(**config) + if error and issubclass(error, Exception): + with pytest.raises(error): + plugin.prepare_data() + else: + plugin.prepare_data() + + @pytest.mark.parametrize( + "config, overrides, error", + [ + (lazy_fixture("base_config"), {}, None), + (lazy_fixture("probs_config"), {}, None), + (lazy_fixture("probs_config"), {"format": "probs"}, None), + (lazy_fixture("probs_config"), {"format": "probs3"}, None), + (lazy_fixture("probs_str_config"), {}, None), + (lazy_fixture("probs3_config"), {}, None), + (lazy_fixture("probs3_config"), {"format": "probs"}, None), + (lazy_fixture("probs3_config"), {"format": "probs3"}, None), + ], + ) + def test_setup(self, config, overrides, error): + if overrides: + overrides = DictConfig(overrides) + config = OmegaConf.merge(config, overrides) + plugin = GeneticTSPlugin(**config) + plugin.prepare_data() + if error and issubclass(error, Exception): + with pytest.raises(error): + plugin.setup() + else: + plugin.setup() + + @pytest.mark.parametrize( + "config, overrides, exp_features", + [ + (lazy_fixture("base_config"), {}, {"genetic": (["var1", "var2", "var3"],)}), + ( + lazy_fixture("probs_config"), + {"format": "probs"}, + {"genetic": (["var1", "var2", "var3"], ["p0", "p1"])}, + ), + ( + lazy_fixture("probs_str_config"), + {"format": "probs"}, + {"genetic": (["var1", "var2", "var3"], ["p0", "p1"])}, + ), + ( + lazy_fixture("probs3_config"), + {"format": "probs3"}, + {"genetic": (["var1", "var2", "var3"], ["p0", "p1", "p2"])}, + ), + ], + ) + def test_get_features(self, config, overrides, exp_features): + if overrides: + overrides = DictConfig(overrides) + config = OmegaConf.merge(config, overrides) + plugin = GeneticTSPlugin(**config) + plugin.prepare_data() + plugin.setup() + features = plugin.get_features() + assert set(features.keys()) == set(exp_features.keys()) + assert features["genetic"] == exp_features["genetic"] + + @pytest.mark.parametrize( + "config, overrides, eids, features, exp_result", + [ + ( + lazy_fixture("base_config"), + {}, + 0, + None, + {"genetic": torch.tensor([0, 1, 0], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + 1, + None, + { + "genetic": torch.tensor([0, 0, 1], dtype=torch.float32), + }, + ), + ( + lazy_fixture("base_config"), + {}, + 2, + None, + {"genetic": torch.tensor([1, 0, 0], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + [0, 2], + None, + {"genetic": torch.tensor([[0, 1, 0], [1, 0, 0]], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + 0, + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([[0, 0]], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + 1, + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([[0, 1]], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + 1, + {"genetic": (torch.tensor([1, 0, 1], dtype=torch.bool),)}, + {"genetic": torch.tensor([[0, 1]], dtype=torch.float32)}, + ), + ( + lazy_fixture("base_config"), + {}, + [0, 2], + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([[0, 0], [1, 0]], dtype=torch.float32)}, + ), + ( + lazy_fixture("probs_config"), + {}, + 0, + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([0.3, 0.3])}, + ), + ( + lazy_fixture("probs_config"), + {"format": "dosages"}, + 0, + None, + {"genetic": torch.tensor([0.3, 0.8, 0.3])}, + ), + ( + lazy_fixture("probs_config"), + {"format": "probs"}, + 0, + None, + {"genetic": torch.tensor([[0.7, 0.3], [0.5, 0.2], [0.8, 0.1]])}, + ), + ( + lazy_fixture("probs_config"), + {"format": "probs"}, + 0, + {"genetic": (["var1", "var3"], slice(None))}, + {"genetic": torch.tensor([[0.7, 0.3], [0.8, 0.1]])}, + ), + ( + lazy_fixture("probs_config"), + {"format": "probs3"}, + 0, + None, + { + "genetic": torch.tensor( + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]] + ) + }, + ), + ( + lazy_fixture("probs_config"), + {"format": "probs3"}, + 0, + {"genetic": (["var1", "var3"], slice(None))}, + {"genetic": torch.tensor([[0.7, 0.3, 0.0], [0.8, 0.1, 0.1]])}, + ), + ( + lazy_fixture("probs_str_config"), + {}, + "a1", + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([0.3, 0.3])}, + ), + ( + lazy_fixture("probs3_config"), + {"format": "dosages"}, + 0, + None, + {"genetic": torch.tensor([0.3, 0.8, 0.3])}, + ), + ( + lazy_fixture("probs3_str_config"), + {"format": "dosages"}, + "a1", + None, + {"genetic": torch.tensor([0.3, 0.8, 0.3])}, + ), + ( + lazy_fixture("probs3_config"), + {"format": "probs3"}, + 0, + None, + { + "genetic": torch.tensor( + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]] + ) + }, + ), + ( + lazy_fixture("probs3_str_config"), + {"format": "probs3"}, + "a1", + None, + { + "genetic": torch.tensor( + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]] + ) + }, + ), + ( + lazy_fixture("probs3_config"), + {"format": "probs3"}, + [0, 2], + None, + { + "genetic": torch.tensor( + [ + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]], + [[0.66, 0.33, 0.01], [0.8, 0.2, 0.0], [0.7, 0.3, 0.0]], + ] + ) + }, + ), + ( + lazy_fixture("probs3_str_config"), + {"format": "probs3"}, + ["a1", "c3"], + None, + { + "genetic": torch.tensor( + [ + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]], + [[0.66, 0.33, 0.01], [0.8, 0.2, 0.0], [0.7, 0.3, 0.0]], + ] + ) + }, + ), + ( + lazy_fixture("probs3_uint8_config"), + {"format": "probs3"}, + 0, + None, + { + "genetic": torch.tensor( + [[0.7, 0.3, 0.0], [0.5, 0.2, 0.3], [0.8, 0.1, 0.1]] + ) + }, + ), + ( + lazy_fixture("probs_uint4_config"), + {"format": "probs"}, + 0, + None, + {"genetic": torch.tensor([[0.7, 0.3], [0.5, 0.2], [0.8, 0.1]])}, + ), + ( + lazy_fixture("dosage_uint8_config"), + {}, + 0, + None, + {"genetic": torch.tensor([0.5, 1.8, 0.2])}, + ), + ( + lazy_fixture("dosage_uint8_config"), + {}, + [0, 2], + None, + {"genetic": torch.tensor([[0.5, 1.8, 0.2], [1.2, 1.1, 0.9]])}, + ), + ( + lazy_fixture("dosage_uint8_config"), + {}, + [0, 2], + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([[0.5, 0.2], [1.2, 0.9]])}, + ), + ( + lazy_fixture("dosage_uint4_config"), + {}, + 0, + None, + {"genetic": torch.tensor([0.5, 1.8, 0.2])}, + ), + ( + lazy_fixture("dosage_uint4_config"), + {}, + [0, 2], + None, + {"genetic": torch.tensor([[0.5, 1.8, 0.2], [1.2, 1.1, 0.9]])}, + ), + ( + lazy_fixture("dosage_uint4_config"), + {}, + [0, 2], + {"genetic": (["var1", "var3"],)}, + {"genetic": torch.tensor([[0.5, 0.2], [1.2, 0.9]])}, + ), + ], + ) + def test__getitem__(self, config, overrides, eids, features, exp_result): + if overrides: + overrides = DictConfig(overrides) + config = OmegaConf.merge(config, overrides) + plugin = GeneticTSPlugin(**config) + plugin.prepare_data() + plugin.setup() + if isclass(exp_result) and issubclass(exp_result, Exception): + with pytest.raises(error): + plugin[eids, features] + else: + result = plugin[eids, features] + assert set(result.keys()) == set(exp_result.keys()) + if plugin.dtype.startswith("uint"): + dtype_bits = int(plugin.dtype[4]) + atol = 1 / 2**dtype_bits + elif plugin.dtype.startswith("float"): + dtype_bits = int(plugin.dtype[5:]) + atol = 1 / 2**dtype_bits + assert torch.allclose( + result["genetic"], exp_result["genetic"], atol=max(2 * atol, 1e-7) + ) diff --git a/test/plugins/h5ad/__init__.py b/test/plugins/h5ad/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/h5ad/h5ad_plugin_test.py b/test/plugins/h5ad/h5ad_plugin_test.py new file mode 100644 index 0000000..2063c16 --- /dev/null +++ b/test/plugins/h5ad/h5ad_plugin_test.py @@ -0,0 +1,755 @@ +from inspect import isclass +from copy import deepcopy +import os +from os.path import dirname, abspath +from omegaconf import OmegaConf +from tempfile import TemporaryDirectory +import numpy as np +import hydra +import pytest +from pytest_lazyfixture import lazy_fixture +import pandas as pd +import torch + +from udm.plugins.h5ad.h5ad_plugin import H5adPlugin +from test.utils import try_clear_hydra + + +UDM_HOME = "../../.." +UDM_HOME_ABSPATH = abspath(os.path.join(dirname(__file__), UDM_HOME)) + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def mini1_data(): + data = np.array([[60, 1], [50, 1], [71, 0]], dtype=np.uint8) + obs = pd.DataFrame({"index": [0, 1, 2]}) + var = pd.DataFrame( + { + "ValueType": ["uint8", "uint8"], + "ID": ["age", "sex"], + } + ) + return data, obs, var + + +@pytest.fixture +def mini1_str_data(mini1_data): + data, obs, var = mini1_data + obs = pd.DataFrame({"index": ["a1", "b2", "c3"]}) + return data, obs, var + + +@pytest.fixture +def mini2_data(): + data = np.array([[81, 0], [64, 0]], dtype=np.uint8) + obs = pd.DataFrame({"index": [3, 4]}) + var = pd.DataFrame( + { + "ValueType": ["uint8", "uint8"], + "ID": ["age", "sex"], + } + ) + return data, obs, var + + +@pytest.fixture +def mini2_str_data(mini2_data): + data, obs, var = mini2_data + obs = pd.DataFrame({"index": ["d4", "e5"]}) + return data, obs, var + + +def prepare_data(temp_path, name, X, obs, var): + basename = os.path.join(temp_path, name) + with open(f"{basename}.X.npy", "wb+") as f: + np.save(f, X, allow_pickle=True) + obs.to_csv(f"{basename}.obs.csv", index=False) + var.to_csv(f"{basename}.var.csv", index=False) + return basename + + +@pytest.fixture +def prepare_mini1(temp_dir, mini1_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + data, obs, var = mini1_data + yield prepare_data(temp_dir, "mini1_config", data, obs, var) + + +@pytest.fixture +def prepare_mini1_str(temp_dir, mini1_str_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + data, obs, var = mini1_str_data + yield prepare_data(temp_dir, "mini1_config", data, obs, var) + + +@pytest.fixture +def prepare_mini2(temp_dir, mini2_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + data, obs, var = mini2_data + yield prepare_data(temp_dir, "mini2_config", data, obs, var) + + +@pytest.fixture +def prepare_mini2_str(temp_dir, mini2_str_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + data, obs, var = mini2_str_data + yield prepare_data(temp_dir, "mini2_config", data, obs, var) + + +@pytest.fixture +def prepare_mini_eid_map(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + eid_map = pd.DataFrame({"from": [0, 1, 2, 3, 4], "to": [5, 6, 7, 8, 9]}) + eid_map_path = f"{temp_dir}/eid_map.csv" + eid_map.to_csv(eid_map_path, sep="\t") + yield eid_map_path + + +@pytest.fixture +def mini1_config(prepare_mini1): + basename = prepare_mini1 + yield OmegaConf.create( + {"src": [basename], "src_names": ["mini1_config"], "eid_map_path": None} + ) + + +@pytest.fixture +def mini1_str_config(prepare_mini1_str): + basename = prepare_mini1_str + yield OmegaConf.create( + {"src": [basename], "src_names": ["mini1_config"], "eid_map_path": None} + ) + + +@pytest.fixture +def mini1_map_config(mini1_config, prepare_mini_eid_map): + eid_map_path = prepare_mini_eid_map + mini1_config.update( + { + "eid_map_path": eid_map_path, + "eid_map_from": "from", + "eid_map_to": "to", + } + ) + yield mini1_config + + +@pytest.fixture +def mini2_config(prepare_mini2): + basename = prepare_mini2 + yield OmegaConf.create( + {"src": [basename], "src_names": ["mini2_config"], "eid_map_path": None} + ) + + +@pytest.fixture +def mini2_str_config(prepare_mini2_str): + basename = prepare_mini2_str + yield OmegaConf.create( + {"src": [basename], "src_names": ["mini2_config"], "eid_map_path": None} + ) + + +@pytest.fixture +def mini2_map_config(mini2_config, prepare_mini_eid_map): + eid_map_path = prepare_mini_eid_map + mini2_config.update( + { + "eid_map_path": eid_map_path, + "eid_map_from": "from", + "eid_map_to": "to", + } + ) + yield mini2_config + + +@pytest.fixture +def mini_config(prepare_mini1, prepare_mini2): + basename1, basename2 = prepare_mini1, prepare_mini2 + yield OmegaConf.create( + { + "src": [basename1, basename2], + "src_names": ["mini1_config", "mini2_config"], + "eid_map_path": None, + } + ) + + +@pytest.fixture +def mini_map_config(mini_config, prepare_mini_eid_map): + eid_map_path = prepare_mini_eid_map + mini_config.update( + { + "eid_map_path": eid_map_path, + "eid_map_from": "from", + "eid_map_to": "to", + } + ) + yield mini_config + + +@pytest.fixture +def geno_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "h5ad") + hydra.initialize(version_base=None, config_path=config_path) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/geno/") + overrides_config = OmegaConf.create( + { + "src": [ + "genetic_extended_tr", + "genetic_valid", + "genetic_test", + ], + "src_names": ["train", "valid", "test"], + "dataset_path": FAKE_DATA_PATH, + } + ) + config = hydra.compose(config_name="debug.yaml", overrides=[]) + yield OmegaConf.merge(config, overrides_config) + + +@pytest.fixture +def cov_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "h5ad") + hydra.initialize(version_base=None, config_path=config_path) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/covariate/") + overrides_config = OmegaConf.create( + { + "src": [ + "covariates_new_base_extended_tr.h5ad", + "covariates_new_base_valid.h5ad", + "covariates_new_base_test.h5ad", + ], + "src_names": ["train", "valid", "test"], + "dataset_path": FAKE_DATA_PATH, + } + ) + config = hydra.compose(config_name="debug.yaml", overrides=[]) + yield OmegaConf.merge(config, overrides_config) + + +@pytest.fixture +def metabol_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "h5ad") + hydra.initialize(version_base=None, config_path=config_path) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/metabolomics/") + overrides_config = OmegaConf.create( + { + "src": [ + "metabolomics_extended_tr.h5ad", + "metabolomics_valid.h5ad", + "metabolomics_test.h5ad", + ], + "src_names": ["train", "valid", "test"], + "dataset_path": FAKE_DATA_PATH, + } + ) + config = hydra.compose(config_name="debug.yaml", overrides=[]) + yield OmegaConf.merge(config, overrides_config) + + +@pytest.fixture +def pheno_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "h5ad") + hydra.initialize(version_base=None, config_path=config_path) + FAKE_DATA_PATH = os.path.join(UDM_HOME_ABSPATH, "res/datasets/fake/pheno/") + overrides_config = OmegaConf.create( + { + "src": [ + "cad_extended_tr.h5ad", + "cad_valid.h5ad", + "cad_test.h5ad", + ], + "src_names": ["train", "valid", "test"], + "dataset_path": FAKE_DATA_PATH, + } + ) + config = hydra.compose(config_name="debug.yaml", overrides=[]) + yield OmegaConf.merge(config, overrides_config) + + +@pytest.fixture +def h5ad_eids(): + yield [ + 5797, + 5099, + 1791, + 7810, + 1972, + 3055, + 9086, + 5933, + 1103, + 4152, + ] + + +@pytest.fixture +def h5ad_eids_head(h5ad_eids): + yield h5ad_eids + + +@pytest.fixture +def h5ad_eid(h5ad_eids_head): + yield h5ad_eids_head[0] + + +class TestH5adPlugin(object): + @pytest.mark.parametrize( + "config", + [ + lazy_fixture("mini1_config"), + lazy_fixture("mini1_str_config"), + lazy_fixture("mini2_config"), + lazy_fixture("mini2_str_config"), + lazy_fixture("mini_config"), + lazy_fixture("geno_config"), + lazy_fixture("cov_config"), + lazy_fixture("metabol_config"), + lazy_fixture("pheno_config"), + ], + ) + def test__init__(self, config): + plugin = H5adPlugin.from_config(config) + + @pytest.mark.parametrize( + "config", + [ + lazy_fixture("mini1_config"), + lazy_fixture("mini1_str_config"), + lazy_fixture("mini2_config"), + lazy_fixture("mini2_str_config"), + lazy_fixture("mini_config"), + lazy_fixture("geno_config"), + lazy_fixture("cov_config"), + lazy_fixture("metabol_config"), + lazy_fixture("pheno_config"), + ], + ) + def test_prepare_data(self, config): + plugin = H5adPlugin.from_config(config) + plugin.prepare_data() + + @pytest.mark.parametrize( + "config", + [ + lazy_fixture("mini1_config"), + lazy_fixture("mini1_str_config"), + lazy_fixture("mini2_config"), + lazy_fixture("mini2_str_config"), + lazy_fixture("mini_config"), + lazy_fixture("geno_config"), + lazy_fixture("cov_config"), + lazy_fixture("metabol_config"), + lazy_fixture("pheno_config"), + ], + ) + def test_setup(self, config): + plugin = H5adPlugin.from_config(config) + plugin.prepare_data() + plugin.setup() + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_len", + [ + (lazy_fixture("mini1_config"), False, False, 0), + (lazy_fixture("mini1_config"), True, False, 0), + (lazy_fixture("mini1_config"), True, True, 3), + (lazy_fixture("mini1_str_config"), True, True, 3), + (lazy_fixture("mini2_config"), True, True, 2), + (lazy_fixture("mini2_str_config"), True, True, 2), + (lazy_fixture("mini_config"), True, True, 5), + (lazy_fixture("geno_config"), False, False, 0), + (lazy_fixture("geno_config"), True, False, 0), + (lazy_fixture("geno_config"), True, True, 4096), + (lazy_fixture("cov_config"), True, True, 4096), + (lazy_fixture("metabol_config"), True, True, 4096), + (lazy_fixture("pheno_config"), True, True, 4096), + ], + ) + def test__len__(self, config, prepare_data, setup, exp_len): + plugin = H5adPlugin.from_config(config) + if prepare_data: + plugin.prepare_data() + if setup: + plugin.setup() + assert len(plugin) == exp_len + + @pytest.mark.parametrize( + "config, prepare_data, setup, eids, features, shape_only, exp_item", + [ + (lazy_fixture("mini1_config"), False, False, 0, None, False, RuntimeError), + (lazy_fixture("mini1_config"), True, False, 0, None, False, RuntimeError), + *[ + ( + lazy_fixture("mini1_config"), + True, + True, + eid, + None, + False, + {"tabular_data": item}, + ) + for eid, item in zip( + [0, 1, 2], + torch.tensor([[60, 1], [50, 1], [71, 0]], dtype=torch.float32), + ) + ], + *[ + ( + lazy_fixture("mini1_str_config"), + True, + True, + eid, + None, + False, + {"tabular_data": item}, + ) + for eid, item in zip( + ["a1", "b2", "c3"], + torch.tensor([[60, 1], [50, 1], [71, 0]], dtype=torch.float32), + ) + ], + ( + lazy_fixture("mini1_config"), + True, + True, + [0, 1, 2], + {"tabular_data": (torch.tensor([True, False], dtype=torch.bool),)}, + False, + {"tabular_data": torch.tensor([[60], [50], [71]], dtype=torch.float32)}, + ), + ( + lazy_fixture("mini1_config"), + True, + True, + [0, 1, 2], + { + "tabular_data": (["age"],), + }, + False, + {"tabular_data": torch.tensor([[60], [50], [71]], dtype=torch.float32)}, + ), + ( + lazy_fixture("mini1_config"), + True, + True, + [0, 1, 2], + {"tabular_data": (torch.tensor([False, True], dtype=torch.bool),)}, + False, + {"tabular_data": torch.tensor([[1], [1], [0]], dtype=torch.float32)}, + ), + ( + lazy_fixture("mini1_config"), + True, + True, + [0, 1, 2], + {"tabular_data": (["sex"],)}, + False, + {"tabular_data": torch.tensor([[1], [1], [0]], dtype=torch.float32)}, + ), + (lazy_fixture("mini2_config"), True, True, 0, None, False, KeyError), + *[ + ( + lazy_fixture("mini2_config"), + True, + True, + eid, + None, + False, + {"tabular_data": item}, + ) + for eid, item in zip( + [3, 4], torch.tensor([[81, 0], [64, 0]], dtype=torch.float32) + ) + ], + *[ + ( + lazy_fixture("mini2_str_config"), + True, + True, + eid, + None, + False, + {"tabular_data": item}, + ) + for eid, item in zip( + ["d4", "e5"], torch.tensor([[81, 0], [64, 0]], dtype=torch.float32) + ) + ], + ( + lazy_fixture("mini_config"), + True, + True, + 4, + None, + False, + {"tabular_data": torch.tensor([64, 0], dtype=torch.float32)}, + ), + ( + lazy_fixture("geno_config"), + True, + True, + lazy_fixture("h5ad_eid"), + None, + True, + {"tabular_data": [10]}, + ), + ( + lazy_fixture("metabol_config"), + True, + True, + lazy_fixture("h5ad_eid"), + None, + True, + {"tabular_data": [168]}, + ), + ( + lazy_fixture("cov_config"), + True, + True, + lazy_fixture("h5ad_eid"), + None, + True, + {"tabular_data": [9]}, + ), + ( + lazy_fixture("pheno_config"), + True, + True, + lazy_fixture("h5ad_eid"), + None, + True, + {"tabular_data": [1]}, + ), + ], + ) + def test__getitem__( + self, config, prepare_data, setup, eids, features, shape_only, exp_item + ): + plugin = H5adPlugin.from_config(config) + if prepare_data: + plugin.prepare_data() + if setup: + plugin.setup() + if isclass(exp_item) and issubclass(exp_item, Exception): + with pytest.raises(exp_item): + plugin[eids, features] + else: + item = deepcopy(plugin[eids, features]) + assert isinstance(item, dict) + assert set(item.keys()) == set(exp_item.keys()) + for key in item: + if shape_only: + assert list(item[key].shape) == exp_item[key] + else: + assert torch.all(item[key] == exp_item[key]) + del plugin + + @pytest.mark.parametrize( + "config, prepare_data, setup, head_only, exp_eids", + [ + (lazy_fixture("mini1_config"), False, False, False, []), + (lazy_fixture("mini1_config"), True, False, False, []), + (lazy_fixture("mini1_config"), True, True, False, [0, 1, 2]), + (lazy_fixture("mini1_str_config"), True, True, False, ["a1", "b2", "c3"]), + (lazy_fixture("mini1_map_config"), True, True, False, [0, 1, 2]), + (lazy_fixture("mini2_config"), True, True, False, [3, 4]), + (lazy_fixture("mini2_str_config"), True, True, False, ["d4", "e5"]), + (lazy_fixture("mini2_map_config"), True, True, False, [3, 4]), + (lazy_fixture("mini_config"), True, True, False, [0, 1, 2, 3, 4]), + ( + lazy_fixture("geno_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("metabol_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("cov_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("pheno_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ], + ) + def test_get_native_eids(self, config, prepare_data, setup, head_only, exp_eids): + self._test_get_eids( + config, prepare_data, setup, head_only, exp_eids, native=True + ) + + @pytest.mark.parametrize( + "config, prepare_data, setup, head_only, exp_eids", + [ + (lazy_fixture("mini1_config"), False, False, False, []), + (lazy_fixture("mini1_config"), True, False, False, []), + (lazy_fixture("mini1_config"), True, True, False, [0, 1, 2]), + (lazy_fixture("mini1_str_config"), True, True, False, ["a1", "b2", "c3"]), + (lazy_fixture("mini1_map_config"), True, True, False, [5, 6, 7]), + (lazy_fixture("mini2_config"), True, True, False, [3, 4]), + (lazy_fixture("mini2_str_config"), True, True, False, ["d4", "e5"]), + (lazy_fixture("mini2_map_config"), True, True, False, [8, 9]), + (lazy_fixture("mini_config"), True, True, False, [0, 1, 2, 3, 4]), + ( + lazy_fixture("geno_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("metabol_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("cov_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ( + lazy_fixture("pheno_config"), + True, + True, + True, + lazy_fixture("h5ad_eids_head"), + ), + ], + ) + def test_get_eids(self, config, prepare_data, setup, head_only, exp_eids): + self._test_get_eids(config, prepare_data, setup, head_only, exp_eids) + + def _test_get_eids( + self, config, prepare_data, setup, head_only, exp_eids, native=False + ): + plugin = H5adPlugin.from_config(config) + if prepare_data: + plugin.prepare_data() + if setup: + plugin.setup() + if native: + eids = plugin.get_native_eids() + else: + eids = plugin.get_eids() + if head_only: + eids = eids[:10] + assert eids == exp_eids + del plugin + + @pytest.mark.parametrize( + "config, prepare_data, setup, head_only, exp_meta", + [ + ( + lazy_fixture("mini1_config"), + False, + False, + False, + { + "tags": [], + "eids": [], + "features": {}, + "feature_types": {}, + "variables": {}, + }, + ), + ( + lazy_fixture("mini1_config"), + True, + False, + False, + { + "tags": [], + "eids": [], + "features": {}, + "feature_types": {}, + "variables": {}, + }, + ), + ( + lazy_fixture("mini1_config"), + True, + True, + False, + { + "tags": [], + "eids": [0, 1, 2], + "features": {"tabular_data": (["age", "sex"],)}, + "feature_types": {"tabular_data": np.float32}, + "variables": { + "0": {"ValueType": "uint8", "ID": "age"}, + "1": {"ValueType": "uint8", "ID": "sex"}, + }, + }, + ), + ( + lazy_fixture("mini2_config"), + True, + True, + False, + { + "tags": [], + "eids": [3, 4], + "features": {"tabular_data": (["age", "sex"],)}, + "feature_types": {"tabular_data": np.float32}, + "variables": { + "0": {"ValueType": "uint8", "ID": "age"}, + "1": {"ValueType": "uint8", "ID": "sex"}, + }, + }, + ), + ( + lazy_fixture("mini_config"), + True, + True, + False, + { + "tags": [], + "eids": [0, 1, 2, 3, 4], + "features": {"tabular_data": (["age", "sex"],)}, + "feature_types": {"tabular_data": np.float32}, + "variables": { + "0": {"ValueType": "uint8", "ID": "age"}, + "1": {"ValueType": "uint8", "ID": "sex"}, + }, + }, + ), + ], + ) + def test_get_metadata(self, config, prepare_data, setup, head_only, exp_meta): + plugin = H5adPlugin.from_config(config) + if prepare_data: + plugin.prepare_data() + if setup: + plugin.setup() + meta = plugin.get_metadata() + assert set(meta.keys()) == set(exp_meta.keys()) + assert meta == exp_meta diff --git a/test/plugins/tabular/__init__.py b/test/plugins/tabular/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/plugins/tabular/tabular_plugin_test.py b/test/plugins/tabular/tabular_plugin_test.py new file mode 100644 index 0000000..d5a2362 --- /dev/null +++ b/test/plugins/tabular/tabular_plugin_test.py @@ -0,0 +1,900 @@ +import os +from typing import Callable +import hydra +import pandas as pd +import yaml +import pyarrow as pa +import pyarrow.feather as feather +import pytest +from pytest_lazyfixture import lazy_fixture +import torch +import json +from tempfile import TemporaryDirectory +from hydra.core.global_hydra import GlobalHydra +import json + +from udm.plugins.tabular.tabular_plugin import TabularPlugin +from test.utils import SKIPIF_NOT_BIH_HPC, try_clear_hydra + +UDM_HOME = "../../.." + + +@pytest.fixture +def temp_dir(): + with TemporaryDirectory() as temp_dir: + yield temp_dir + + +@pytest.fixture +def mini1_data(): + return { + "eid": [0, 1, 2], + "age": [-0.7, -1.59, 0.9], + "sex_female": [1, 0, 0], + "sex_male": [0, 1, 1], + "midi-chlorian-level": [0.5, 0.3, -1.3], # may the force be with you + "home_planet_tatooine": [1, 0, 0], + } + + +@pytest.fixture +def mini1_str_data(mini1_data): + mini1_data.update({"eid": ["a1", "b2", "c3"]}) + return mini1_data + + +def prepare_feather(temp_path, name, data): + TMP_PATH = os.path.join(temp_path, f"{name}.feather") + tabular_pa = pa.Table.from_pydict(data) + feather.write_feather(tabular_pa, TMP_PATH) + return TMP_PATH + + +@pytest.fixture +def prepare_mini1(temp_dir, mini1_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "tabular1", mini1_data) + + +@pytest.fixture +def prepare_mini1_str(temp_dir, mini1_str_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "tabular1", mini1_str_data) + + +@pytest.fixture +def mini2_data(): + return { + "eid": [3, 4, 5], + "age": [-0.9, -1.380, 1.22], + "sex_female": [0, 0, 1], # female,male,female + "sex_male": [1, 1, 0], + "midi-chlorian-level": [-0.6, 0.353, 2.46], + "home_planet_tatooine": [0, 1, 0], + } + + +@pytest.fixture +def mini2_str_data(mini2_data): + mini2_data.update({"eid": ["d4", "e5", "f6"]}) + return mini2_data + + +@pytest.fixture +def prepare_mini2(temp_dir, mini2_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "tabular2", mini2_data) + + +@pytest.fixture +def prepare_mini2_str(temp_dir, mini2_str_data): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + yield prepare_feather(temp_dir, "tabular2", mini2_str_data) + + +@pytest.fixture +def prepare_variable_annotations(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + var_ann_data = { + "index": [ + "eid", + "age", + "sex_female", + "sex_male", + "midi-chlorian-level", + "home_planet_tatooine", + ], + "description": [ + "the electronic identifier", + "individual age", + "female", + "male", + "the amount of force sensitive microbes", + "who wants to live on tatooine?", + ], + } + VAR_ANNO_PATH = os.path.join(temp_dir, "var_annotations.csv") + var_ann_df = pd.DataFrame(var_ann_data) + var_ann_df.to_csv(VAR_ANNO_PATH) # TODO fix weird double index column + yield VAR_ANNO_PATH # TODO create a var anno anno debug dataset + + +@pytest.fixture +def prepare_phecode_dataset(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + phecode_data = { + "eid": [ + "0", + "1", + "2", + "3", + "4", + "5", + ], + "phecode_089": [ + 0, + 1, + 1, + 0, + 1, + 0, + ], + "phecode_460": [ + 1, + 1, + 0, + 0, + 0, + 1, + ], + "phecode_713": [ + 0, + 1, + 0, + 0, + 1, + 0, + ], + "phecode_713.3": [ + 1, + 1, + 1, + 0, + 0, + 1, + ], + "phecode_718": [ + 0, + 0, + 1, + 1, + 0, + 1, + ], + "phecode_283": [ + 0, + 0, + 0, + 1, + 0, + 0, + ], + "phecode_460.1": [ + 0, + 1, + 0, + 0, + 1, + 1, + ], + "phecode_660": [ + 0, + 1, + 1, + 0, + 1, + 1, + ], + "phecode_401": [ + 0, + 0, + 1, + 0, + 1, + 1, + ], + "phecode_401.1": [ + 1, + 1, + 0, + 0, + 0, + 1, + ], + } + PHECODE_DS_PATH = os.path.join(temp_dir, "phecode_ds.csv") + phecode_ds_df = pd.DataFrame(phecode_data) + phecode_ds_df.to_feather(PHECODE_DS_PATH) + yield PHECODE_DS_PATH + + +@pytest.fixture +def prepare_columns_subset_csv(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + subset = {"columns_to_pick": ["age", "sex_female", "home_planet_tatooine"]} + COL_SUBSET_PATH = os.path.join(temp_dir, "column_subset.csv") + col_subset_df = pd.DataFrame.from_dict(subset) + col_subset_df.to_csv(COL_SUBSET_PATH, index=False) + yield COL_SUBSET_PATH + + +@pytest.fixture +def prepare_composite_features1(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + subset = {"cad_endpoint": ["phecode_401", "phecode_401.1", "phecode_713.3"]} + COMP_FEATURES_PATH = os.path.join(temp_dir, "composite_features_test1.json") + with open(COMP_FEATURES_PATH, "w") as f: + json.dump(subset, f) + yield COMP_FEATURES_PATH + + +@pytest.fixture +def prepare_composite_features2(temp_dir): + subset = { + "cad_endpoint": ["phecode_401", "phecode_401.1", "phecode_713.3"], + "infection_endpoints": ["phecode_089", "phecode_713", "phecode_283"], + } + COMP_FEATURES_PATH = os.path.join(temp_dir, "composite_features_test2.json") + with open(COMP_FEATURES_PATH, "w") as f: + json.dump(subset, f) + yield COMP_FEATURES_PATH + + +@pytest.fixture +def prepare_intersection_features1(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + subset = {"cad_endpoint": ["phecode_401", "phecode_401.1", "phecode_713.3"]} + INTER_FEATURES_PATH = os.path.join(temp_dir, "intersection_features_test1.json") + with open(INTER_FEATURES_PATH, "w") as f: + json.dump(subset, f) + yield INTER_FEATURES_PATH + + +@pytest.fixture +def prepare_intersection_features2(temp_dir): + with TemporaryDirectory(dir=temp_dir) as temp_dir: + subset = { + "cad_endpoint": ["phecode_401", "phecode_401.1", "phecode_713.3"], + "infection_endpoints": ["phecode_089", "phecode_713", "phecode_283"], + } + INTER_FEATURES_PATH = os.path.join(temp_dir, "intersection_features_test2.json") + with open(INTER_FEATURES_PATH, "w") as f: + json.dump(subset, f) + yield INTER_FEATURES_PATH + + +@pytest.fixture +def tabular_debug_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose(config_name="default") + data_root = f"{config.data_paths}/{config.debug_data_paths}" + path = os.path.join( + data_root, + "covariates_new_base_outliers_te_debug.feather", + ) + config.data_paths = path + yield config + + +@pytest.fixture +def tabular_debug_usecols_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose(config_name="debug", overrides=["dropcols=null"]) + data_root = f"{config.data_paths}/{config.debug_data_paths}" + path = os.path.join( + data_root, + "covariates_new_base_outliers_te_debug.feather", + ) + config.data_paths = path + yield config + + +@pytest.fixture +def tabular_debug_usecols_csv_config(prepare_mini1, prepare_columns_subset_csv): + try_clear_hydra() + mini1_path, col_subset_path = prepare_mini1, prepare_columns_subset_csv + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={mini1_path}", + f"usecols={col_subset_path}", + f"dropcols=null", + ], + ) + yield config + + +@pytest.fixture +def tabular_debug_dropcols_config(): + try_clear_hydra() + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose(config_name="default", overrides=["usecols=null"]) + data_root = f"{config.data_paths}/{config.debug_data_paths}" + path = os.path.join( + data_root, + "covariates_new_base_outliers_te_debug.feather", + ) + config.data_paths = path + yield config + + +@pytest.fixture +def tabular_debug_dropcols_csv_config(prepare_mini1, prepare_columns_subset_csv): + try_clear_hydra() + mini1_path, col_subset_path = prepare_mini1, prepare_columns_subset_csv + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={mini1_path}", + f"usecols=null", + f"dropcols={col_subset_path}", + ], + ) + yield config + + +@pytest.fixture +def tabular_mini_config(prepare_mini1, prepare_mini2): + try_clear_hydra() + mini1_path, mini2_path = prepare_mini1, prepare_mini2 + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + "default.yaml", overrides=[f"data_paths=[{mini1_path}, {mini2_path}]"] + ) + yield config + + +@pytest.fixture +def tabular_mini1_config(prepare_mini1, prepare_variable_annotations): + try_clear_hydra() + mini1_path, var_anno_path = prepare_mini1, prepare_variable_annotations + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + "default.yaml", + overrides=[ + f"data_paths={mini1_path}", + f"variable_annotations_path={var_anno_path}", + ], + ) + yield config + + +@pytest.fixture +def tabular_mini1_str_config(prepare_mini1_str, prepare_variable_annotations): + try_clear_hydra() + mini1_path, var_anno_path = prepare_mini1_str, prepare_variable_annotations + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + "default.yaml", + overrides=[ + f"data_paths={mini1_path}", + f"variable_annotations_path={var_anno_path}", + ], + ) + yield config + + +@pytest.fixture +def tabular_mini2_config(prepare_mini2): + try_clear_hydra() + mini2_path = prepare_mini2 + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[f"data_paths={mini2_path}"]) + return config + + +@pytest.fixture +def tabular_mini2_str_config(prepare_mini2_str): + try_clear_hydra() + mini2_path = prepare_mini2_str + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose("default.yaml", overrides=[f"data_paths={mini2_path}"]) + return config + + +@pytest.fixture +def tabular_debug_composite_features1_config( + prepare_phecode_dataset, prepare_composite_features1 +): + try_clear_hydra() + phecode_ds_path, comp_feature_path = ( + prepare_phecode_dataset, + prepare_composite_features1, + ) + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={phecode_ds_path}", + f"composite_features={comp_feature_path}", + f"only_keep_composite_features=True", + f"usecols=null", + f"dropcols=null", + ], + ) + yield config + + +@pytest.fixture +def tabular_debug_composite_features2_config( + prepare_phecode_dataset, prepare_composite_features2 +): + try_clear_hydra() + phecode_ds_path, comp_feature_path = ( + prepare_phecode_dataset, + prepare_composite_features2, + ) + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={phecode_ds_path}", + f"composite_features={comp_feature_path}", + f"drop_individual_cols_of_composite_features=True", + f"usecols=null", + f"dropcols=null", + ], + ) + yield config + + +@pytest.fixture +def tabular_debug_intersection_features1_config( + prepare_phecode_dataset, prepare_intersection_features1 +): + try_clear_hydra() + phecode_ds_path, inter_feature_path = ( + prepare_phecode_dataset, + prepare_intersection_features1, + ) + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={phecode_ds_path}", + f"intersection_features={inter_feature_path}", + f"only_keep_intersection_features=True", + f"usecols=null", + f"dropcols=null", + ], + ) + yield config + + +@pytest.fixture +def tabular_debug_intersection_features2_config( + prepare_phecode_dataset, prepare_intersection_features2 +): + try_clear_hydra() + phecode_ds_path, inter_feature_path = ( + prepare_phecode_dataset, + prepare_intersection_features2, + ) + config_path = os.path.join(UDM_HOME, "config", "plugins", "tabular") + hydra.initialize(version_base=None, config_path=config_path) + config = hydra.compose( + config_name="debug", + overrides=[ + f"data_paths={phecode_ds_path}", + f"intersection_features={inter_feature_path}", + f"drop_individual_cols_of_intersection_features=True", + f"usecols=null", + f"dropcols=null", + ], + ) + yield config + + +@pytest.fixture() +def tabular_debug_eids(): + TABULAR_DEBUG_EIDS_PATH = os.path.join( + UDM_HOME, "data/analysis/results/genopheno/datasets_by_pheno_feather/eids.yaml" + ) + eids = yaml.load(TABULAR_DEBUG_EIDS_PATH, Loader=yaml.CSafeLoader) + yield list(eids) + + +@pytest.fixture() +def tabular_debug_eid(tabular_debug_eids): + yield tabular_debug_eids[0] + + +class TestTabularPlugin(object): + def setUp(self) -> None: + if GlobalHydra.instance().is_initialized(): + GlobalHydra().instance().clear() + + @pytest.mark.parametrize( + "config", + [ + lazy_fixture("tabular_debug_config"), + lazy_fixture("tabular_mini1_config"), + lazy_fixture("tabular_mini1_str_config"), + lazy_fixture("tabular_mini2_config"), + lazy_fixture("tabular_mini2_str_config"), + ], + ) + def test__init__(self, config): + tabular_plugin = TabularPlugin.from_config(config) + + @pytest.mark.parametrize( + "config", + [ + pytest.param( + lazy_fixture("tabular_debug_config"), marks=SKIPIF_NOT_BIH_HPC + ), + lazy_fixture("tabular_mini1_config"), + lazy_fixture("tabular_mini1_str_config"), + lazy_fixture("tabular_mini2_config"), + lazy_fixture("tabular_mini2_str_config"), + ], + ) + def test_prepare_data(self, config): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + + @pytest.mark.parametrize( + "config", + [ + pytest.param( + lazy_fixture("tabular_debug_config"), marks=SKIPIF_NOT_BIH_HPC + ), + lazy_fixture("tabular_mini1_config"), + lazy_fixture("tabular_mini1_str_config"), + lazy_fixture("tabular_mini2_config"), + lazy_fixture("tabular_mini2_str_config"), + ], + ) + def test_setup(self, config): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + + @pytest.mark.parametrize( + "config, eid, features, exp_item, check_equal", + [ + pytest.param( + lazy_fixture("tabular_debug_config"), + lazy_fixture("tabular_debug_eid"), + None, + {"tabular_data": None}, + False, + marks=SKIPIF_NOT_BIH_HPC, + ), + ( + lazy_fixture("tabular_mini1_config"), + 0, + None, + { + "tabular_data": torch.tensor( + [-0.7, 1, 0, 0.5, 1], dtype=torch.float32 + ) + }, + True, + ), + ( + lazy_fixture("tabular_mini1_str_config"), + "a1", + None, + { + "tabular_data": torch.tensor( + [-0.7, 1, 0, 0.5, 1], dtype=torch.float32 + ) + }, + True, + ), + ( + lazy_fixture("tabular_mini1_config"), + [0, 2], + None, + { + "tabular_data": torch.tensor( + [ + [-0.7, 0.9], # age + [1, 0], # sex_female + [0, 1], # sex_male + [0.5, -1.3], # midi-chlorian-level + [1, 0], # home_planet_tatooine + ], + dtype=torch.float32, + ).T + }, + True, + ), + ( + lazy_fixture("tabular_mini1_str_config"), + ["a1", "c3"], + None, + { + "tabular_data": torch.tensor( + [ + [-0.7, 0.9], # age + [1, 0], # sex_female + [0, 1], # sex_male + [0.5, -1.3], # midi-chlorian-level + [1, 0], # home_planet_tatooine + ], + dtype=torch.float32, + ).T + }, + True, + ), + ( + lazy_fixture("tabular_mini1_config"), + [0, 2], + { + "tabular_data": ( + torch.tensor( + [True, False, False, True, True], dtype=torch.bool + ), + ) + }, + { + "tabular_data": torch.tensor( + [ + [-0.7, 0.9], # age + [0.5, -1.3], # midi-chlorian-level + [1, 0], # home_planet_tatooine + ], + dtype=torch.float32, + ).T + }, + True, + ), + ( + lazy_fixture("tabular_mini1_config"), + [0, 2], + {"tabular_data": (["sex_female", "sex_male", "midi-chlorian-level"],)}, + { + "tabular_data": torch.tensor( + [ + [1, 0], # sex_female + [0, 1], # sex_male + [0.5, -1.3], # midi-chlorian-level + ], + dtype=torch.float32, + ).T + }, + True, + ), + *[ + ( + lazy_fixture("tabular_mini2_str_config"), + eid, + None, + { + "tabular_data": torch.tensor( + [ + [-0.9, -1.380, 1.22], + [0, 0, 1], + [1, 1, 0], + [-0.6, 0.353, 2.46], + [0, 1, 0], + ] + ).T[i, :] + }, + True, + ) + for i, eid in enumerate(["d4", "e5", "f6"]) + ], + ( + lazy_fixture("tabular_mini2_str_config"), + ["d4", "f6"], + None, + { + "tabular_data": torch.tensor( + [[-0.9, 1.22], [0, 1], [1, 0], [-0.6, 2.46], [0, 0]] + ).T + }, + True, + ), + ], + ) + def test__getitem__(self, config, eid, features, exp_item, check_equal): + # usecols = ["sex_female", "home_planet_tatooine"] + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + item = tabular_plugin[eid, features] + assert isinstance(item, dict) + assert list(item.keys()) == ["tabular_data"] + if check_equal: + assert torch.all(item["tabular_data"] == exp_item["tabular_data"]) + + @pytest.mark.parametrize( + "config, exp_meta_keys", + [ + pytest.param( + lazy_fixture("tabular_debug_config"), + {"eids", "features", "feature_types"}, + marks=SKIPIF_NOT_BIH_HPC, + ), + ( + lazy_fixture("tabular_mini1_config"), + {"eids", "features", "feature_types", "variable_annotations"}, + ), + ], + ) + def test_get_metadata(self, config, exp_meta_keys): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + metadata = tabular_plugin.get_metadata() + assert isinstance(metadata, dict) + assert set(metadata.keys()).intersection(exp_meta_keys) == exp_meta_keys + + def _get_plugin_eids(tabular_plugin): + return list(tabular_plugin.tabular_df.index) + + @pytest.mark.parametrize( + "config, prepare_data, setup, exp_eids", + [ + (lazy_fixture("tabular_debug_config"), False, False, []), + pytest.param( + lazy_fixture("tabular_debug_config"), + True, + False, + _get_plugin_eids, + marks=SKIPIF_NOT_BIH_HPC, + ), + pytest.param( + lazy_fixture("tabular_debug_config"), + True, + True, + _get_plugin_eids, + marks=SKIPIF_NOT_BIH_HPC, + ), # TODO change when we have proper debug files + (lazy_fixture("tabular_mini1_config"), True, True, [0, 1, 2]), + (lazy_fixture("tabular_mini1_str_config"), True, True, ["a1", "b2", "c3"]), + (lazy_fixture("tabular_mini2_config"), True, True, [3, 4, 5]), + (lazy_fixture("tabular_mini2_str_config"), True, True, ["d4", "e5", "f6"]), + ], + ) + def test_get_eids(self, config, prepare_data, setup, exp_eids): + tabular_plugin = TabularPlugin.from_config(config) + if prepare_data: + tabular_plugin.prepare_data() + if setup: + tabular_plugin.setup() + eids = tabular_plugin.get_eids() + assert isinstance(eids, list) + if isinstance(exp_eids, Callable): + exp_eids = exp_eids(tabular_plugin) + assert eids == exp_eids + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, exp_features", + [ + ( + lazy_fixture("tabular_debug_usecols_config"), + [ + "genetic_principal_components_f22009_0_1", + "genetic_principal_components_f22009_0_2", + "genetic_principal_components_f22009_0_3", + "genetic_principal_components_f22009_0_4", + "genetic_principal_components_f22009_0_5", + "age", + "sex_f31_0_0_Female", + ], + ), + ( + lazy_fixture("tabular_debug_usecols_csv_config"), + ["age", "sex_female", "home_planet_tatooine"], + ), + ], + ) + def test_usecols(self, config, exp_features): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + assert list(tabular_plugin.tabular_data_after_setup.columns) == exp_features + + @SKIPIF_NOT_BIH_HPC + @pytest.mark.parametrize( + "config, exp_features", + [ + ( + lazy_fixture("tabular_debug_dropcols_config"), + [ + "genetic_principal_components_f22009_0_1", + "genetic_principal_components_f22009_0_2", + "genetic_principal_components_f22009_0_3", + "genetic_principal_components_f22009_0_4", + "genetic_principal_components_f22009_0_5", + "age", + "batch", + "sex_f31_0_0_Female", + "sex_f31_0_0_Male", + ], + ), + ( + lazy_fixture("tabular_debug_dropcols_csv_config"), + ["sex_male", "midi-chlorian-level"], + ), + ], + ) + def test_dropcols(self, config, exp_features): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + assert list(tabular_plugin.tabular_data_after_setup.columns) == exp_features + + @pytest.mark.parametrize( + "config, exp_features", + [ + ( + lazy_fixture("tabular_debug_composite_features1_config"), + ["cad_endpoint"], + ), + ( + lazy_fixture("tabular_debug_composite_features2_config"), + [ + "phecode_460", + "phecode_718", + "phecode_460.1", + "phecode_660", + "cad_endpoint", + "infection_endpoints", + ], + ), + ], + ) + def test_composite_features(self, config, exp_features): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + assert list(tabular_plugin.tabular_data_after_setup.columns) == exp_features + + @pytest.mark.parametrize( + "config, exp_features", + [ + ( + lazy_fixture("tabular_debug_intersection_features1_config"), + ["cad_endpoint"], + ), + ( + lazy_fixture("tabular_debug_intersection_features2_config"), + [ + "phecode_460", + "phecode_718", + "phecode_460.1", + "phecode_660", + "cad_endpoint", + "infection_endpoints", + ], + ), + ], + ) + def test_intersection_features(self, config, exp_features): + tabular_plugin = TabularPlugin.from_config(config) + tabular_plugin.prepare_data() + tabular_plugin.setup() + assert list(tabular_plugin.tabular_data_after_setup.columns) == exp_features diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 0000000..a4071d6 --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = --strict-markers +markers = + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/test/transforms/__init__.py b/test/transforms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/transforms/dict_transforms_test.py b/test/transforms/dict_transforms_test.py new file mode 100644 index 0000000..03d0ce6 --- /dev/null +++ b/test/transforms/dict_transforms_test.py @@ -0,0 +1,339 @@ +import pytest +import torch +from typing import Mapping +from omegaconf import DictConfig + +from udm.transforms.tensor_transforms import ListTransform +from udm.transforms.dict_transforms import ( + DictTransform, + PluginTransform, + DatasetTransform, +) +from test.transforms.tensor_transforms.factory_test import ScaleTrafo + + +def nestedTensorDictEqual(item1, item2): + assert type(item1) == type(item2) + if isinstance(item1, torch.Tensor): + assert torch.all(item1 == item2) + elif isinstance(item1, Mapping): + assert set(item1.keys()) == set(item2.keys()) + for key in item1.keys(): + _item1, _item2 = item1[key], item2[key] + nestedTensorDictEqual(_item1, _item2) + + +@pytest.fixture +def scale_trafo_cfg(): + return DictConfig( + { + "name": "DictTransform", + "transforms": { + "data": {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2} + }, + } + ) + + +@pytest.fixture +def composed_trafo_cfg(): + return DictConfig( + { + "name": "DictTransform", + "transforms": { + "data": { + "name": "ListTransform", + "transforms": [ + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + ], + } + }, + } + ) + + +@pytest.fixture +def nested_trafo_cfg(): + return DictConfig( + { + "name": "DictTransform", + "transforms": { + "data": { + "name": "DictTransform", + "transforms": { + "a": {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + "b": {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 3}, + }, + } + }, + } + ) + + +class TestDictTransform(object): + @pytest.mark.parametrize( + "transforms", + [ + {"data": ScaleTrafo(2.0)}, + {"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])}, + {"data": DictTransform({"data": ScaleTrafo(2.0)})}, + {"data": DictTransform({"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)})}, + ], + ) + def test__init__(self, transforms): + trafo = DictTransform(transforms) + + @pytest.mark.parametrize( + "config,custom", + [ + (pytest.lazy_fixture("scale_trafo_cfg"), {"ScaleTrafo": ScaleTrafo}), + (pytest.lazy_fixture("composed_trafo_cfg"), {"ScaleTrafo": ScaleTrafo}), + (pytest.lazy_fixture("nested_trafo_cfg"), {"ScaleTrafo": ScaleTrafo}), + ], + ) + def test_from_config(self, config, custom): + trafo = DictTransform.from_config(config, custom=custom) + + @pytest.mark.parametrize( + "transforms,custom,input,exp_result,warning", + [ + ( + {"data": ScaleTrafo(2.0)}, + {}, + {"data": torch.tensor([1.0, 2.0, 3.0])}, + {"data": torch.tensor([2.0, 4.0, 6.0])}, + None, + ), + ( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)}, + {}, + {"a": torch.tensor([1.0, 2.0, 3.0])}, + {"a": torch.tensor([2.0, 4.0, 6.0])}, + UserWarning, + ), + ( + {"a": ScaleTrafo(2.0)}, + {}, + { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([4.0, 5.0, 6.0]), + }, + { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([4.0, 5.0, 6.0]), + }, + UserWarning, + ), + ( + {"data": DictTransform({"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)})}, + {}, + { + "data": { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([4.0, 5.0, 6.0]), + } + }, + { + "data": { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([12.0, 15.0, 18.0]), + } + }, + None, + ), + ( + pytest.lazy_fixture("scale_trafo_cfg"), + {"ScaleTrafo": ScaleTrafo}, + {"data": torch.tensor([1.0, 2.0, 3.0])}, + {"data": torch.tensor([2.0, 4.0, 6.0])}, + None, + ), + ( + pytest.lazy_fixture("nested_trafo_cfg"), + {"ScaleTrafo": ScaleTrafo}, + { + "data": { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([4.0, 5.0, 6.0]), + } + }, + { + "data": { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([12.0, 15.0, 18.0]), + } + }, + None, + ), + ], + ) + def test__call__(self, transforms, custom, input, exp_result, warning): + if isinstance(transforms, DictConfig): + trafo = DictTransform.from_config(transforms, custom=custom) + else: + trafo = DictTransform(transforms, custom=custom) + if warning is not None: + with pytest.warns(warning): + result = trafo(input) + else: + result = trafo(input) + nestedTensorDictEqual(result, exp_result) + + +class TestPluginTransform(object): + @pytest.mark.parametrize( + "transforms,error", + [ + ({"data": ScaleTrafo(2.0)}, None), + ({"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])}, None), + ({"data": DictTransform({"data": ScaleTrafo(2.0)})}, AssertionError), + ({"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)}, None), + ], + ) + def test__init__(self, transforms, error): + if error is not None: + with pytest.raises(error): + PluginTransform(transforms) + else: + trafo = PluginTransform(transforms) + + @pytest.mark.parametrize( + "transforms,custom,input,exp_result", + [ + ( + {"data": ScaleTrafo(2.0)}, + {}, + {"data": torch.tensor([1.0, 2.0, 3.0])}, + {"data": torch.tensor([2.0, 4.0, 6.0])}, + ), + ( + {"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])}, + {}, + {"data": torch.tensor([1.0, 2.0, 3.0])}, + {"data": torch.tensor([4.0, 8.0, 12.0])}, + ), + ( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)}, + {}, + { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([4.0, 5.0, 6.0]), + }, + { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([12.0, 15.0, 18.0]), + }, + ), + ], + ) + def test__call__(self, transforms, custom, input, exp_result): + if isinstance(transforms, DictConfig): + trafo = PluginTransform.from_config(transforms, custom=custom) + else: + trafo = PluginTransform(transforms, custom=custom) + result = trafo(input) + nestedTensorDictEqual(result, exp_result) + + +class TestDatasetTransform(object): + @pytest.mark.parametrize( + "transforms,error", + [ + ({"data": ScaleTrafo(2.0)}, AssertionError), + ( + {"data": ListTransform([ScaleTrafo(2.0), ScaleTrafo(2.0)])}, + AssertionError, + ), + ({"data": DictTransform({"data": ScaleTrafo(2.0)})}, AssertionError), + ( + { + "plugin1": PluginTransform( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)} + ) + }, + None, + ), + ( + { + "plugin1": PluginTransform( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)} + ), + "plugin2": PluginTransform( + {"c": ScaleTrafo(4.0), "d": ScaleTrafo(5.0)} + ), + }, + None, + ), + ], + ) + def test__init__(self, transforms, error): + if error is not None: + with pytest.raises(error): + DatasetTransform(transforms) + else: + trafo = DatasetTransform(transforms) + + @pytest.mark.parametrize( + "transforms,custom,input,exp_result", + [ + ( + { + "plugin1": PluginTransform( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)} + ) + }, + {}, + { + "plugin1": { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([2.0, 3.0, 4.0]), + } + }, + { + "plugin1": { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([6.0, 9.0, 12.0]), + } + }, + ), + ( + { + "plugin1": PluginTransform( + {"a": ScaleTrafo(2.0), "b": ScaleTrafo(3.0)} + ), + "plugin2": PluginTransform( + {"c": ScaleTrafo(4.0), "d": ScaleTrafo(5.0)} + ), + }, + {}, + { + "plugin1": { + "a": torch.tensor([1.0, 2.0, 3.0]), + "b": torch.tensor([2.0, 3.0, 4.0]), + }, + "plugin2": { + "c": torch.tensor([3.0, 4.0, 5.0]), + "d": torch.tensor([4.0, 5.0, 6.0]), + }, + }, + { + "plugin1": { + "a": torch.tensor([2.0, 4.0, 6.0]), + "b": torch.tensor([6.0, 9.0, 12.0]), + }, + "plugin2": { + "c": torch.tensor([12.0, 16.0, 20.0]), + "d": torch.tensor([20.0, 25.0, 30.0]), + }, + }, + ), + ], + ) + def test__call__(self, transforms, custom, input, exp_result): + if isinstance(transforms, DictConfig): + trafo = DatasetTransform.from_config(transforms, custom=custom) + else: + trafo = DatasetTransform(transforms, custom=custom) + result = trafo(input) + nestedTensorDictEqual(result, exp_result) diff --git a/test/transforms/tensor_transforms/__init__.py b/test/transforms/tensor_transforms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/transforms/tensor_transforms/factory_test.py b/test/transforms/tensor_transforms/factory_test.py new file mode 100644 index 0000000..601fb92 --- /dev/null +++ b/test/transforms/tensor_transforms/factory_test.py @@ -0,0 +1,98 @@ +import pytest +from omegaconf import DictConfig +from typing import Callable, Optional +import torch + +from udm.transforms.tensor_transforms.factory import get_transform + + +def scale_trafo(x: torch.Tensor, alpha: float): + return alpha * x + + +class ScaleTrafo(Callable): + def __init__(self, alpha: float): + self.alpha = alpha + + def __call__(self, x: torch.Tensor): + return self.alpha * x + + +class TestTransforms(object): + @pytest.mark.parametrize( + "config,custom,data,exp_data,error", + [ + ( + DictConfig( + { + "name": "torch.nn.functional.pad", + "pad": (1, 1), + "mode": "constant", + "value": 0, + } + ), + {}, + torch.tensor([1.0, 2.0, 3.0], dtype=torch.float), + torch.tensor([0.0, 1.0, 2.0, 3.0, 0.0], dtype=torch.float), + None, + ), + ( + DictConfig( + { + "name": "ConstantPad1d", + "__init__": "__init__", + "padding": (1, 1), + "value": 0, + } + ), + {}, + torch.tensor([1.0, 2.0, 3.0], dtype=torch.float), + torch.tensor([0.0, 1.0, 2.0, 3.0, 0.0], dtype=torch.float), + KeyError, + ), + ( + DictConfig({"name": "scale_trafo", "alpha": 2}), + {"scale_trafo": scale_trafo}, + torch.tensor([0.0, 1.0, 2.0, 3.0], dtype=torch.float), + torch.tensor([0.0, 2.0, 4.0, 6.0], dtype=torch.float), + None, + ), + ( + DictConfig({"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}), + {"ScaleTrafo": ScaleTrafo}, + torch.tensor([0.0, 1.0, 2.0, 3.0], dtype=torch.float), + torch.tensor([0.0, 2.0, 4.0, 6.0], dtype=torch.float), + None, + ), + ( + DictConfig( + { + "name": "ListTransform", + "transforms": [ + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + {"name": "ScaleTrafo", "__init__": "__init__", "alpha": 2}, + ], + } + ), + {"ScaleTrafo": ScaleTrafo}, + torch.tensor([0.0, 1.0, 2.0, 3.0], dtype=torch.float), + torch.tensor([0.0, 4.0, 8.0, 12.0], dtype=torch.float), + None, + ), + ], + ) + def test_get_transform( + self, + config, + custom, + data: torch.Tensor, + exp_data: torch.Tensor, + error: Optional[Exception], + ): + if error is not None: + with pytest.raises(error): + get_transform(config, custom) + else: + trafo = get_transform(config, custom=custom) + trafo_data = trafo(data) + assert torch.allclose(trafo_data, exp_data) diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000..bac4325 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,71 @@ +import gc +import os +from contextlib import ExitStack +from os.path import abspath, dirname +from tempfile import TemporaryDirectory +from typing import Callable, Mapping + +import pytest +import torch +from hydra.core.global_hydra import GlobalHydra + + +UDM_HOME = ".." +SKIPIF_NOT_BIH_HPC = pytest.mark.skipif( + os.environ.get("BIH_HPC", "0") == "0", + reason="Run this test only on BIH-HPC environment.", +) + + +def try_clear_hydra(): + if GlobalHydra.instance().is_initialized(): + GlobalHydra.instance().clear() + + +def assertBatchEqual(batch, expected_batch, shapes_only=False): + # TODO: Extend test to also check eids + assert isinstance(batch, dict) + assert set(batch.keys()) == set([*expected_batch.keys(), "eids"]) + for p_key in set(batch.keys()) - set(["eids"]): + assert set(batch[p_key].keys()) == set(expected_batch[p_key].keys()) + for t_key in batch[p_key].keys(): + if shapes_only: + shape = [*batch[p_key][t_key].shape] + exp_shape = expected_batch[p_key][t_key] + assert shape == exp_shape + else: + assert torch.all(batch[p_key][t_key] == expected_batch[p_key][t_key]) + + +class BaseConfigHandler(ExitStack): + + name: str + # The name of the config to initialize + _configs: Mapping[str, Callable] + # A mapping of config name to a callable returning the config + + def __init__(self, name: str): + super().__init__() + self.name = name + self._configs = {} + + def __enter__(self): + DIR = abspath(os.path.join(dirname(__file__), UDM_HOME)) + self.TMP_DIR = TemporaryDirectory(dir=DIR) + if GlobalHydra.instance().is_initialized(): + GlobalHydra.instance().clear() + return self._configs[self.name]() + + def __exit__(self, exc_type, exc_val, exc_tb): + super().__exit__(exc_type, exc_val, exc_tb) + gc.collect() + self.TMP_DIR.cleanup() + + +class Head(list): + def __init__(self, *elements): + if len(elements) == 1 and isinstance(elements[0], list): + seq = elements[0] + else: + seq = [*elements] + super().__init__(seq[:10]) diff --git a/udm/transforms/tensor_transforms/factory.py b/udm/transforms/tensor_transforms/factory.py index 8ddf9e4..e8a3449 100644 --- a/udm/transforms/tensor_transforms/factory.py +++ b/udm/transforms/tensor_transforms/factory.py @@ -1,5 +1,6 @@ from typing import Callable from abc import ABC +from functools import partial from omegaconf import OmegaConf, DictConfig from inspect import isabstract, isclass, isroutine import torch @@ -157,7 +158,7 @@ def get_transform(trafo_config: DictConfig, custom={}): trafo_config_subset = { k: v for k, v in trafo_config.items() if k in KEY_SUBSET } - transform = lambda tensor: module(tensor, **trafo_config_subset) + transform = partial(module, **trafo_config_subset) return transform else: raise KeyError(f"{trafo_config['name']} is not a supported transform!")