diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 185e3b97..4ee42bd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,19 +12,11 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version-file: ".python-version" - - name: Install uv uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 with: enable-cache: true - - name: Install the project - run: uv sync --locked --dev - - name: Cache pre-commit uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: @@ -33,9 +25,3 @@ jobs: - name: Run pre-commit run: uvx pre-commit run --show-diff-on-failure --color=always --all-files - - - name: Generate tables - run: uv run --no-sync dat generate-tables - - - name: Run tests - run: uv run --no-sync pytest tests diff --git a/.python-version b/.python-version deleted file mode 100644 index 24ee5b1b..00000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 diff --git a/contributing.md b/contributing.md index 6b8dec07..bc3eac3d 100644 --- a/contributing.md +++ b/contributing.md @@ -2,33 +2,23 @@ ## Getting started -This repository uses `poetry` to handle Python dependencies and an isolated environment. To install the environment, run +The workload generator is an sbt project. To compile it, run ``` -poetry install +cd workload-generator +sbt Test/compile ``` -Most common tasks are defined in the Makefile. For example, to lint the Python files run +Most common tasks are defined in the justfile. For example, to run tests: ``` -make lint +just test ``` -To open a shell session within the poetry environment, use +## Generating workloads locally -``` -poetry shell -``` - -## Generating reference tables locally - -Reader test cases are defined in `dat/generated_tables.py`. Currently, they are all defined in terms of PySpark code. If there is a table that doesn't fit into that paradigm, create an issue on the repository and we will consider other modes of adding tables. - -To regenerate tables, run - -``` -make write-generated-tables -``` +Workload suites are defined under `workload-generator/src/test/scala/io/delta/workload/tables`. +See `workload-generator/docs/authoring-guide.md` for authoring and generation instructions. ## Adding writer tests @@ -36,9 +26,4 @@ TBD ## Running unit tests -Note: You need to generate the reference tables locally in order to run the unit tests. - -You can run the unit tests with `poetry run pytest tests`. - -If you've already run `poetry shell`, you can run the tests with `pytest tests`. - +Run `just test`, or use the focused sbt commands documented in `workload-generator/README.md`. diff --git a/justfile b/justfile index 55cd141b..4a494cc6 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,5 @@ -# setup the local environment to develop dat +# setup local pre-commit hooks setup: - uv sync uvx pre-commit install # update pre-commit hooks @@ -11,12 +10,9 @@ update-hooks: lint: uvx pre-commit run --all-files -# run unit tests +# run workload-generator tests test: - uv run --no-sync pytest tests + cd workload-generator && sbt test -generate-tables: - uv run dat generate-tables - -generate-schemas: - uv run dat generate-schemas +generate-workloads output="/tmp/dat-workloads": + cd workload-generator && WORKLOAD_OUTPUT_DIR={{output}} sbt "testOnly io.delta.workload.tables.*" diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index ea4eed1e..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[project] -name = "dat" -version = "0.0.3" -description = "Accetance testing for Delta Lake implementations." -readme = "README.md" -license = "MIT" -requires-python = ">=3.11" -keywords = ["deltalake", "delta", "acceptance"] -classifiers = [ - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] -dependencies = [ - "pydantic>=2.11.7,<3.0.0", - "pyspark>=4.0.0,<5.0.0", - "delta-spark>=4.0.0,<5.0.0", - "click>=8.2.1,<9.0.0", - "pyarrow>=21", - "typer>=0.16.0", -] - -[project.scripts] -dat = "dat.main:app" - -[project.urls] -repository = "https://github.com/delta-incubator/dat" - -[dependency-groups] -dev = [{ include-group = "lint" }, { include-group = "test" }] -lint = ["ruff"] -test = ["pytest", "chispa>=0.11.1"] - -[build-system] -requires = ["uv_build>=0.8.6,<0.9.0"] -build-backend = "uv_build" diff --git a/src/dat/__init__.py b/src/dat/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/dat/generated_tables.py b/src/dat/generated_tables.py deleted file mode 100644 index 72ce72d0..00000000 --- a/src/dat/generated_tables.py +++ /dev/null @@ -1,573 +0,0 @@ -import os -import random -from datetime import date, datetime, timedelta -from decimal import Decimal -from pathlib import Path -from typing import Callable, List, Tuple - -import pyspark.sql -import pyspark.sql.types as types -from delta.tables import DeltaTable -from pyspark.sql import SparkSession -from pyspark.sql.functions import col, lit, now - -from dat.models import TableVersionMetadata, TestCaseInfo -from dat.spark_builder import get_spark_session - -registered_reference_tables: List[Tuple[TestCaseInfo, Callable]] = [] - - -def get_version_metadata(case: TestCaseInfo) -> TableVersionMetadata: - spark = get_spark_session() - table = DeltaTable.forPath(spark, case.delta_root) - detail = table.detail().collect()[0] - - return TableVersionMetadata( - version=table.history(1).collect()[0].version, - properties=detail["properties"], - min_reader_version=detail["minReaderVersion"], - min_writer_version=detail["minWriterVersion"], - ) - - -def save_expected(case: TestCaseInfo, as_latest=False) -> None: - """Save the specified version of a Delta Table as a Parquet file.""" - spark = get_spark_session() - # we want all golden data to be in microsecond format, as that's what delta is always - # supposed to read - spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS") - df = spark.read.format("delta").load(case.delta_root) - - version_metadata = get_version_metadata(case) - version = None if as_latest else version_metadata.version - - # Need to ensure directory exists first - os.makedirs(case.expected_root(version)) - - df.write.parquet(case.expected_path(version)) - - out_path = case.expected_root(version) / "table_version_metadata.json" - with open(out_path, "w") as f: - f.write(version_metadata.model_dump_json(indent=2)) - - -def reference_table(name: str, description: str): - case = TestCaseInfo(name=name, description=description) - - def wrapper(create_table): - def inner(): - spark = get_spark_session() - create_table(case, spark) - - with open(case.root / "test_case_info.json", "w") as f: - f.write(case.model_dump_json(indent=2)) - - # Write out latest - save_expected(case, as_latest=True) - - registered_reference_tables.append((case, inner)) - - return inner - - return wrapper - - -@reference_table( - name="basic_append", description="A basic table with two append writes." -) -def create_basic_append(case: TestCaseInfo, spark: SparkSession): - columns = ["letter", "number", "a_float"] - data = [("a", 1, 1.1), ("b", 2, 2.2), ("c", 3, 3.3)] - df = spark.createDataFrame(data, schema=columns) - - df.repartition(1).write.format("delta").save(case.delta_root) - save_expected(case) - - data = [("d", 4, 4.4), ("e", 5, 5.5)] - df = spark.createDataFrame(data, schema=columns) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - save_expected(case) - - -@reference_table( - name="basic_partitioned", - description="A basic partitioned table", -) -def create_basic_partitioned(case: TestCaseInfo, spark: SparkSession): - columns = ["letter", "number", "a_float"] - data = [("a", 1, 1.1), ("b", 2, 2.2), ("c", 3, 3.3)] - df = spark.createDataFrame(data, schema=columns) - - df.repartition(1).write.partitionBy("letter").format("delta").save(case.delta_root) - save_expected(case) - - data2 = [("a", 4, 4.4), ("e", 5, 5.5), ("f", 6, 6.6)] - df = spark.createDataFrame(data2, schema=columns) - - df.repartition(1).write.partitionBy("letter").format("delta").mode("append").save( - case.delta_root - ) - save_expected(case) - - -@reference_table( - name="partitioned_with_null", - description="A partitioned table with a null partition", -) -def create_partitioned_with_null(case: TestCaseInfo, spark: SparkSession): - columns = ["letter", "number", "a_float"] - data = [("a", 1, 1.1), ("b", 2, 2.2), ("c", 3, 3.3)] - df = spark.createDataFrame(data, schema=columns) - - df.repartition(1).write.partitionBy("letter").format("delta").save(case.delta_root) - save_expected(case) - - data2 = [("a", 4, 4.4), ("e", 5, 5.5), (None, 6, 6.6)] - df = spark.createDataFrame(data2, schema=columns) - - df.repartition(1).write.partitionBy("letter").format("delta").mode("append").save( - case.delta_root - ) - save_expected(case) - - -@reference_table( - name="multi_partitioned", - description=( - "A table with multiple partitioning columns. Partition " - "values include nulls and escape characters." - ), -) -def create_multi_partitioned(case: TestCaseInfo, spark: SparkSession): - columns = ["letter", "date", "data", "number"] - partition_columns = ["letter", "date", "data"] - data = [ - ("a", date(1970, 1, 1), b"hello", 1), - ("b", date(1970, 1, 1), b"world", 2), - ("b", date(1970, 1, 2), b"world", 3), - ] - df = spark.createDataFrame(data, schema=columns) - schema = df.schema - - df.repartition(1).write.format("delta").partitionBy(*partition_columns).save( - case.delta_root - ) - save_expected(case) - - # Introduce null values in partition columns - data2 = [("a", None, b"x", 4), (None, None, None, 5)] - df = spark.createDataFrame(data2, schema=schema) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - save_expected(case) - - # Introduce escape characters - data3 = [ - ("/%20%f", date(1970, 1, 1), b"hello", 6), - ("b", date(1970, 1, 1), "😈".encode(), 7), - ] - df = spark.createDataFrame(data3, schema=schema) - - df.repartition(1).write.format("delta").mode("overwrite").save(case.delta_root) - save_expected(case) - - -@reference_table( - name="multi_partitioned_2", - description=( - "Multiple levels of partitioning, with boolean, timestamp, and " - "decimal partition columns" - ), -) -def create_multi_partitioned_2(case: TestCaseInfo, spark: SparkSession): - columns = ["bool", "time", "amount", "int"] - partition_columns = ["bool", "time", "amount"] - data = [ - (True, datetime(1970, 1, 1), Decimal("200.00"), 1), - (True, datetime(1970, 1, 1, 12, 30), Decimal("200.00"), 2), - (False, datetime(1970, 1, 2, 8, 45), Decimal("12.00"), 3), - ] - df = spark.createDataFrame(data, schema=columns) - df.repartition(1).write.format("delta").partitionBy(*partition_columns).save( - case.delta_root - ) - - -@reference_table( - name="with_schema_change", - description="Table which has schema change using overwriteSchema=True.", -) -def with_schema_change(case: TestCaseInfo, spark: SparkSession): - columns = ["letter", "number"] - data = [("a", 1), ("b", 2), ("c", 3)] - df = spark.createDataFrame(data, schema=columns) - - df.repartition(1).write.format("delta").save(case.delta_root) - - columns = ["num1", "num2"] - data2 = [(22, 33), (44, 55), (66, 77)] - df = spark.createDataFrame(data2, schema=columns) - - df.repartition(1).write.mode("overwrite").option("overwriteSchema", True).format( - "delta" - ).save(case.delta_root) - save_expected(case) - - -@reference_table( - name="all_primitive_types", - description="Table containing all non-nested types", -) -def create_all_primitive_types(case: TestCaseInfo, spark: SparkSession): - schema = types.StructType( - [ - types.StructField("utf8", types.StringType()), - types.StructField("int64", types.LongType()), - types.StructField("int32", types.IntegerType()), - types.StructField("int16", types.ShortType()), - types.StructField("int8", types.ByteType()), - types.StructField("float32", types.FloatType()), - types.StructField("float64", types.DoubleType()), - types.StructField("bool", types.BooleanType()), - types.StructField("binary", types.BinaryType()), - types.StructField("decimal", types.DecimalType(5, 3)), - types.StructField("date32", types.DateType()), - types.StructField("timestamp", types.TimestampType()), - ] - ) - - df = spark.createDataFrame( - [ - ( - str(i), - i, - i, - i, - i, - float(i), - float(i), - i % 2 == 0, - bytes(i), - Decimal("10.000") + i, - date(1970, 1, 1) + timedelta(days=i), - datetime(1970, 1, 1) + timedelta(hours=i), - ) - for i in range(5) - ], - schema=schema, - ) - - df.repartition(1).write.format("delta").save(case.delta_root) - - -@reference_table( - name="nested_types", - description="Table containing various nested types", -) -def create_nested_types(case: TestCaseInfo, spark: SparkSession): - schema = types.StructType( - [ - types.StructField("pk", types.IntegerType()), - types.StructField( - "struct", - types.StructType( - [ - types.StructField("float64", types.DoubleType()), - types.StructField("bool", types.BooleanType()), - ] - ), - ), - types.StructField("array", types.ArrayType(types.ShortType())), - types.StructField( - "map", types.MapType(types.StringType(), types.IntegerType()) - ), - ] - ) - - df = spark.createDataFrame( - [ - ( - i, - {"float64": float(i), "bool": i % 2 == 0}, - list(range(i + 1)), - {str(i): i for i in range(i)}, - ) - for i in range(5) - ], - schema=schema, - ) - - df.repartition(1).write.format("delta").save(case.delta_root) - - -def get_sample_data( - spark: SparkSession, seed: int = 42, nrows: int = 5 -) -> pyspark.sql.DataFrame: - # Use seed to get consistent data between runs, for reproducibility - random.seed(seed) - return spark.createDataFrame( - [ - ( - random.choice(["a", "b", "c", None]), - random.randint(0, 1000), - date(random.randint(1970, 2020), random.randint(1, 12), 1), - ) - for i in range(nrows) - ], - schema=["letter", "int", "date"], - ) - - -@reference_table( - name="with_checkpoint", - description="Table with a checkpoint", -) -def create_with_checkpoint(case: TestCaseInfo, spark: SparkSession): - spark.conf.set( - "spark.databricks.delta.legacy.allowAmbiguousPathsInCreateTable", "true" - ) - df = get_sample_data(spark) - - ( - DeltaTable.create(spark) - .location(str(Path(case.delta_root).absolute())) - .addColumns(df.schema) - .property("delta.checkpointInterval", "2") - .execute() - ) - - for i in range(3): - df = get_sample_data(spark, seed=i, nrows=5) - df.repartition(1).write.format("delta").mode("overwrite").save(case.delta_root) - - assert any( - path.suffixes == [".checkpoint", ".parquet"] - for path in (Path(case.delta_root) / "_delta_log").iterdir() - ) - - -def remove_log_file(delta_root: str, version: int): - os.remove(os.path.join(delta_root, "_delta_log", f"{version:0>20}.json")) - - -@reference_table( - name="no_replay", - description="Table with a checkpoint and prior commits cleaned up", -) -def create_no_replay(case: TestCaseInfo, spark: SparkSession): - spark.conf.set("spark.databricks.delta.retentionDurationCheck.enabled", "false") - - df = get_sample_data(spark) - - table = ( - DeltaTable.create(spark) - .location(str(Path(case.delta_root).absolute())) - .addColumns(df.schema) - .property("delta.checkpointInterval", "2") - .execute() - ) - - for i in range(3): - df = get_sample_data(spark, seed=i, nrows=5) - df.repartition(1).write.format("delta").mode("overwrite").save(case.delta_root) - - table.vacuum(retentionHours=0) - - remove_log_file(case.delta_root, version=0) - remove_log_file(case.delta_root, version=1) - - files_in_log = list((Path(case.delta_root) / "_delta_log").iterdir()) - assert any(path.suffixes == [".checkpoint", ".parquet"] for path in files_in_log) - assert not any(path.name == f"{0:0>20}.json" for path in files_in_log) - - -@reference_table( - name="stats_as_struct", - description="Table with stats only written as struct (not JSON) with Checkpoint", -) -def create_stats_as_struct(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - ( - DeltaTable.create(spark) - .location(str(Path(case.delta_root).absolute())) - .addColumns(df.schema) - .property("delta.checkpointInterval", "2") - .property("delta.checkpoint.writeStatsAsStruct", "true") - .property("delta.checkpoint.writeStatsAsJson", "false") - .execute() - ) - - for i in range(3): - df = get_sample_data(spark, seed=i, nrows=5) - df.repartition(1).write.format("delta").mode("overwrite").save(case.delta_root) - - -@reference_table( - name="no_stats", - description="Table with no stats", -) -def create_no_stats(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - ( - DeltaTable.create(spark) - .location(str(Path(case.delta_root).absolute())) - .addColumns(df.schema) - .property("delta.checkpointInterval", "2") - .property("delta.checkpoint.writeStatsAsStruct", "false") - .property("delta.checkpoint.writeStatsAsJson", "false") - .property("delta.dataSkippingNumIndexedCols", "0") - .execute() - ) - - for i in range(3): - df = get_sample_data(spark, seed=i, nrows=5) - df.repartition(1).write.format("delta").mode("overwrite").save(case.delta_root) - - -@reference_table( - name="deletion_vectors", - description="Table with deletion vectors", -) -def create_deletion_vectors(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - - delta_path = str(Path(case.delta_root).absolute()) - delta_table: DeltaTable = ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .property("delta.enableDeletionVectors", "true") - .execute() - ) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - - delta_table.delete(col("letter") == "a") - - -@reference_table(name="check_constraints", description="Table with a check constraint") -def check_constraint_table(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .property("delta.enableDeletionVectors", "true") - .execute() - ) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - spark.sql( - f"ALTER TABLE delta.`{delta_path}` ADD CONSTRAINT const1 CHECK (int > 0);" - ) - - -@reference_table( - name="cdf", - description="Table with cdf turned on", -) -def create_change_data_feed(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - delta_table: DeltaTable = ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .property("delta.enableChangeDataFeed", "true") - .execute() - ) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - - delta_table.update(condition=col("letter") == "c", set={"letter": lit("a")}) - delta_table.delete(col("letter") == "a") - - -@reference_table( - name="generated_columns", - description="Table with a generated column", -) -def create_generated_columns(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .addColumn( - "creation", types.DateType(), generatedAlwaysAs="CAST(now() AS DATE)" - ) - .execute() - ) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - - -@reference_table( - name="column_mapping", - description="Table with column mapping turned on", -) -def create_column_mapping(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .property("delta.columnMapping.mode", "name") - .execute() - ) - - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) - spark.sql(f"ALTER TABLE delta.`{delta_path}` RENAME COLUMN int TO new_int;") - ( - df.withColumnRenamed("int", "new_int") - .repartition(1) - .write.format("delta") - .mode("append") - .save(case.delta_root) - ) - - -@reference_table( - name="timestamp_ntz", - description="Table with not timezone timestamps in it", -) -def create_timestamp_ntz(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - delta_table: DeltaTable = ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .addColumn("timestampNTZ", types.TimestampNTZType()) - .execute() - ) - delta_table.upgradeTableProtocol(3, 7) - ( - df.withColumn("timestampNTZ", now().cast(types.TimestampNTZType())) - .repartition(1) - .write.format("delta") - .mode("append") - .save(case.delta_root) - ) - - -@reference_table( - name="iceberg_compat_v1", - description="Table with Iceberg compatibility v1 turned on", -) -def create_iceberg_compat_v1(case: TestCaseInfo, spark: SparkSession): - df = get_sample_data(spark) - delta_path = str(Path(case.delta_root).absolute()) - delta_table: DeltaTable = ( - DeltaTable.create(spark) - .location(delta_path) - .addColumns(df.schema) - .property("delta.enableIcebergCompatV1", "true") - .execute() - ) - delta_table.upgradeTableProtocol(3, 7) - df.repartition(1).write.format("delta").mode("append").save(case.delta_root) diff --git a/src/dat/main.py b/src/dat/main.py deleted file mode 100644 index 846f173f..00000000 --- a/src/dat/main.py +++ /dev/null @@ -1,84 +0,0 @@ -import json -import logging -import os -import shutil -from pathlib import Path -from typing import Annotated - -import typer - -from dat import generated_tables -from dat.models import TableVersionMetadata, TestCaseInfo - -logging.basicConfig( - level=logging.INFO, -) - - -app = typer.Typer() - - -@app.callback() -def main(): - """ - DAT (Delta Acceptance Testing) CLI helper. - - This CLI tool helps to perform mundane tasks related to managing - reference tables for delta acceptance testing, including: - - - generating tables from python code - - generating json schemas of the metadata for code generation - in other programming languages - - - """ - pass - - -@app.command() -def generate_tables( - table_name: Annotated[ - str | None, - typer.Option( - help="Name of the table to write. Will write only the specified table." - ), - ] = None, -): - """Generate DAT reference Tables.""" - - if table_name: - for metadata, create_table in generated_tables.registered_reference_tables: - if metadata.name == table_name: - logging.info("Writing table '%s'", metadata.name) - out_base = Path("out/reader_tests/generated") / table_name - shutil.rmtree(out_base, ignore_errors=True) - - create_table() - break - else: - raise ValueError(f"Could not find generated table named '{table_name}'") - else: - out_base = Path("out/reader_tests/generated") - shutil.rmtree(out_base, ignore_errors=True) - - for metadata, create_table in generated_tables.registered_reference_tables: - logging.info("Writing table '%s'", metadata.name) - create_table() - - -@app.command() -def generate_schemas(): - """Generate JSON schemas.""" - - out_base = Path("out/schemas") - os.makedirs(out_base, exist_ok=True) - - with open(out_base / "TestCaseInfo.json", "w") as f: - f.write(json.dumps(TestCaseInfo.model_json_schema(), indent=2)) - - with open(out_base / "TableVersionMetadata.json", "w") as f: - f.write(json.dumps(TableVersionMetadata.model_json_schema(), indent=2)) - - -if __name__ == "__main__": - app() diff --git a/src/dat/models.py b/src/dat/models.py deleted file mode 100644 index a2279ff8..00000000 --- a/src/dat/models.py +++ /dev/null @@ -1,33 +0,0 @@ -from pathlib import Path -from typing import Dict, Optional - -from pydantic import BaseModel - -OUT_ROOT = Path("./out/reader_tests/generated/") - - -class TestCaseInfo(BaseModel): - name: str - description: str - - @property - def root(self) -> Path: - return OUT_ROOT / self.name - - @property - def delta_root(self) -> str: - return str(self.root / "delta") - - def expected_root(self, version: Optional[int] = None) -> Path: - version_path = "latest" if version is None else f"v{version}" - return self.root / "expected" / version_path - - def expected_path(self, version: Optional[int] = None) -> str: - return str(self.expected_root(version) / "table_content") - - -class TableVersionMetadata(BaseModel): - version: int - properties: Dict[str, str] - min_reader_version: int - min_writer_version: int diff --git a/src/dat/spark_builder.py b/src/dat/spark_builder.py deleted file mode 100644 index 21ba2edd..00000000 --- a/src/dat/spark_builder.py +++ /dev/null @@ -1,25 +0,0 @@ -import delta -from pyspark.sql import SparkSession - -builder = None - - -def get_spark_session(): - global builder - # Only configure the builder once - if builder is None: - builder = ( - SparkSession.builder.appName( - "DAT", - ) - .config( - "spark.sql.extensions", - "io.delta.sql.DeltaSparkSessionExtension", - ) - .config( - "spark.sql.catalog.spark_catalog", - "org.apache.spark.sql.delta.catalog.DeltaCatalog", - ) - ) - builder = delta.configure_spark_with_delta_pip(builder) - return builder.getOrCreate() diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 48245b35..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - -from dat import spark_builder - - -@pytest.fixture(scope="session") -def spark_session(request): - spark = spark_builder.get_spark_session() - spark.sparkContext.setLogLevel("ERROR") - request.addfinalizer(spark.stop) - return spark diff --git a/tests/pyspark_delta/test_pyspark_delta.py b/tests/pyspark_delta/test_pyspark_delta.py deleted file mode 100644 index 03e680ea..00000000 --- a/tests/pyspark_delta/test_pyspark_delta.py +++ /dev/null @@ -1,78 +0,0 @@ -import json -from pathlib import Path -from typing import List, NamedTuple, Optional - -import chispa -import pytest - -TEST_ROOT = Path("out/reader_tests/") - -MAX_SUPPORTED_READER_VERSION = 3 - - -class ReadCase(NamedTuple): - delta_root: Path - version: Optional[int] - parquet_root: Path - name: str - description: str - min_reader_version: int - min_writer_version: int - - -cases: List[ReadCase] = [] - -for path in (TEST_ROOT / "generated").iterdir(): - if path.is_dir(): - with open(path / "test_case_info.json") as f: - case_metadata = json.load(f) - - for version_path in (path / "expected").iterdir(): - if version_path.is_dir(): - if version_path.name[0] == "v": - version = int(version_path.name[1:]) - elif version_path.name == "latest": - version = None - else: - continue - - with open(version_path / "table_version_metadata.json") as f: - expected_metadata = json.load(f) - - case = ReadCase( - delta_root=path / "delta", - version=version, - parquet_root=version_path / "table_content", - name=case_metadata["name"], - description=case_metadata["description"], - min_reader_version=expected_metadata["min_reader_version"], - min_writer_version=expected_metadata["min_writer_version"], - ) - cases.append(case) - - -@pytest.mark.parametrize( - "case", cases, ids=lambda case: f"{case.name} (version={case.version})" -) -def test_readers_dat(spark_session, case: ReadCase): - query = spark_session.read.format("delta") - if case.version is not None: - query = query.option("versionAsOf", case.version) - - if case.min_reader_version > MAX_SUPPORTED_READER_VERSION: - # If it's a reader version we don't support, assert failure - with pytest.raises(Exception): - query.load(str(case.delta_root)) - else: - actual_df = query.load(str(case.delta_root)) - - expected_df = spark_session.read.format("parquet").load( - str(case.parquet_root) + "/*.parquet" - ) - - if "pk" in actual_df.columns: - actual_df = actual_df.orderBy("pk") - expected_df = expected_df.orderBy("pk") - chispa.assert_df_equality(actual_df, expected_df) - else: - chispa.assert_df_equality(actual_df, expected_df, ignore_row_order=True) diff --git a/uv.lock b/uv.lock deleted file mode 100644 index 2627d7ca..00000000 --- a/uv.lock +++ /dev/null @@ -1,439 +0,0 @@ -version = 1 -revision = 2 -requires-python = ">=3.11" - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "chispa" -version = "0.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "prettytable" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/a8/b27d253dcec4ab951f8f713e78a25dcb5ffa44d593f164c55bf18a4d0166/chispa-0.11.1.tar.gz", hash = "sha256:0fc1255cd78291381bfbc29d800fc9e6ea45ec9a0b4da723e40f6cc8dd589fb9", size = 19216, upload-time = "2025-04-12T15:35:53.281Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/58/ce0319dbe9d4933086fd4f3f09d3313edff54a04fded8ab5bc12b35159a2/chispa-0.11.1-py3-none-any.whl", hash = "sha256:a6fc103880b0ad196432dd7dcda9f904069026ec9232db60c4540fbeabca4279", size = 20307, upload-time = "2025-04-12T15:35:51.539Z" }, -] - -[[package]] -name = "click" -version = "8.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "dat" -version = "0.0.3" -source = { editable = "." } -dependencies = [ - { name = "click" }, - { name = "delta-spark" }, - { name = "pyarrow" }, - { name = "pydantic" }, - { name = "pyspark" }, - { name = "typer" }, -] - -[package.dev-dependencies] -dev = [ - { name = "chispa" }, - { name = "pytest" }, - { name = "ruff" }, -] -lint = [ - { name = "ruff" }, -] -test = [ - { name = "chispa" }, - { name = "pytest" }, -] - -[package.metadata] -requires-dist = [ - { name = "click", specifier = ">=8.2.1,<9.0.0" }, - { name = "delta-spark", specifier = ">=4.0.0,<5.0.0" }, - { name = "pyarrow", specifier = ">=21" }, - { name = "pydantic", specifier = ">=2.11.7,<3.0.0" }, - { name = "pyspark", specifier = ">=4.0.0,<5.0.0" }, - { name = "typer", specifier = ">=0.16.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "chispa", specifier = ">=0.11.1" }, - { name = "pytest" }, - { name = "ruff" }, -] -lint = [{ name = "ruff" }] -test = [ - { name = "chispa", specifier = ">=0.11.1" }, - { name = "pytest" }, -] - -[[package]] -name = "delta-spark" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "pyspark" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/fd/37a0c2ee6fbf4dcbf034178dcb978ba586d37dc9c95214b90416d1042735/delta-spark-4.0.0.tar.gz", hash = "sha256:39325d76d6e5d8b5fea9d47827c59f5e0674844cb95f97b2cc613bfc5209c7ec", size = 35026, upload-time = "2025-06-06T01:41:46.286Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/d8/265a93d22ae79262cdff701496a6f5676926a342153f3855ae6060430660/delta_spark-4.0.0-py3-none-any.whl", hash = "sha256:4e4ded07bb9ee4f6a0df45606d84395239d4b82001e765a627fecc1e914f3029", size = 39756, upload-time = "2025-06-06T01:41:44.815Z" }, -] - -[[package]] -name = "importlib-metadata" -version = "8.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "prettytable" -version = "3.16.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/99/b1/85e18ac92afd08c533603e3393977b6bc1443043115a47bb094f3b98f94f/prettytable-3.16.0.tar.gz", hash = "sha256:3c64b31719d961bf69c9a7e03d0c1e477320906a98da63952bc6698d6164ff57", size = 66276, upload-time = "2025-03-24T19:39:04.008Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c7/5613524e606ea1688b3bdbf48aa64bafb6d0a4ac3750274c43b6158a390f/prettytable-3.16.0-py3-none-any.whl", hash = "sha256:b5eccfabb82222f5aa46b798ff02a8452cf530a352c31bddfa29be41242863aa", size = 33863, upload-time = "2025-03-24T19:39:02.359Z" }, -] - -[[package]] -name = "py4j" -version = "0.10.9.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, -] - -[[package]] -name = "pyarrow" -version = "21.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" }, - { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" }, - { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" }, - { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" }, - { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" }, - { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" }, - { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" }, - { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" }, - { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" }, - { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" }, - { url = "https://files.pythonhosted.org/packages/16/ca/c7eaa8e62db8fb37ce942b1ea0c6d7abfe3786ca193957afa25e71b81b66/pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a", size = 31154306, upload-time = "2025-07-18T00:56:04.42Z" }, - { url = "https://files.pythonhosted.org/packages/ce/e8/e87d9e3b2489302b3a1aea709aaca4b781c5252fcb812a17ab6275a9a484/pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe", size = 32680622, upload-time = "2025-07-18T00:56:07.505Z" }, - { url = "https://files.pythonhosted.org/packages/84/52/79095d73a742aa0aba370c7942b1b655f598069489ab387fe47261a849e1/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd", size = 41104094, upload-time = "2025-07-18T00:56:10.994Z" }, - { url = "https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61", size = 42825576, upload-time = "2025-07-18T00:56:15.569Z" }, - { url = "https://files.pythonhosted.org/packages/b3/62/0f29de6e0a1e33518dec92c65be0351d32d7ca351e51ec5f4f837a9aab91/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d", size = 43368342, upload-time = "2025-07-18T00:56:19.531Z" }, - { url = "https://files.pythonhosted.org/packages/90/c7/0fa1f3f29cf75f339768cc698c8ad4ddd2481c1742e9741459911c9ac477/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99", size = 45131218, upload-time = "2025-07-18T00:56:23.347Z" }, - { url = "https://files.pythonhosted.org/packages/01/63/581f2076465e67b23bc5a37d4a2abff8362d389d29d8105832e82c9c811c/pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636", size = 26087551, upload-time = "2025-07-18T00:56:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ab/357d0d9648bb8241ee7348e564f2479d206ebe6e1c47ac5027c2e31ecd39/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da", size = 31290064, upload-time = "2025-07-18T00:56:30.214Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8a/5685d62a990e4cac2043fc76b4661bf38d06efed55cf45a334b455bd2759/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7", size = 32727837, upload-time = "2025-07-18T00:56:33.935Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/c0828ee09525c2bafefd3e736a248ebe764d07d0fd762d4f0929dbc516c9/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6", size = 41014158, upload-time = "2025-07-18T00:56:37.528Z" }, - { url = "https://files.pythonhosted.org/packages/6e/26/a2865c420c50b7a3748320b614f3484bfcde8347b2639b2b903b21ce6a72/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8", size = 42667885, upload-time = "2025-07-18T00:56:41.483Z" }, - { url = "https://files.pythonhosted.org/packages/0a/f9/4ee798dc902533159250fb4321267730bc0a107d8c6889e07c3add4fe3a5/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503", size = 43276625, upload-time = "2025-07-18T00:56:48.002Z" }, - { url = "https://files.pythonhosted.org/packages/5a/da/e02544d6997037a4b0d22d8e5f66bc9315c3671371a8b18c79ade1cefe14/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79", size = 44951890, upload-time = "2025-07-18T00:56:52.568Z" }, - { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006, upload-time = "2025-07-18T00:56:56.379Z" }, -] - -[[package]] -name = "pydantic" -version = "2.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.33.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/8d/71db63483d518cbbf290261a1fc2839d17ff89fce7089e08cad07ccfce67/pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7", size = 2028584, upload-time = "2025-04-23T18:31:03.106Z" }, - { url = "https://files.pythonhosted.org/packages/24/2f/3cfa7244ae292dd850989f328722d2aef313f74ffc471184dc509e1e4e5a/pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246", size = 1855071, upload-time = "2025-04-23T18:31:04.621Z" }, - { url = "https://files.pythonhosted.org/packages/b3/d3/4ae42d33f5e3f50dd467761304be2fa0a9417fbf09735bc2cce003480f2a/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f", size = 1897823, upload-time = "2025-04-23T18:31:06.377Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f3/aa5976e8352b7695ff808599794b1fba2a9ae2ee954a3426855935799488/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc", size = 1983792, upload-time = "2025-04-23T18:31:07.93Z" }, - { url = "https://files.pythonhosted.org/packages/d5/7a/cda9b5a23c552037717f2b2a5257e9b2bfe45e687386df9591eff7b46d28/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de", size = 2136338, upload-time = "2025-04-23T18:31:09.283Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/b8f9ec8dd1417eb9da784e91e1667d58a2a4a7b7b34cf4af765ef663a7e5/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a", size = 2730998, upload-time = "2025-04-23T18:31:11.7Z" }, - { url = "https://files.pythonhosted.org/packages/47/bc/cd720e078576bdb8255d5032c5d63ee5c0bf4b7173dd955185a1d658c456/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef", size = 2003200, upload-time = "2025-04-23T18:31:13.536Z" }, - { url = "https://files.pythonhosted.org/packages/ca/22/3602b895ee2cd29d11a2b349372446ae9727c32e78a94b3d588a40fdf187/pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e", size = 2113890, upload-time = "2025-04-23T18:31:15.011Z" }, - { url = "https://files.pythonhosted.org/packages/ff/e6/e3c5908c03cf00d629eb38393a98fccc38ee0ce8ecce32f69fc7d7b558a7/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d", size = 2073359, upload-time = "2025-04-23T18:31:16.393Z" }, - { url = "https://files.pythonhosted.org/packages/12/e7/6a36a07c59ebefc8777d1ffdaf5ae71b06b21952582e4b07eba88a421c79/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30", size = 2245883, upload-time = "2025-04-23T18:31:17.892Z" }, - { url = "https://files.pythonhosted.org/packages/16/3f/59b3187aaa6cc0c1e6616e8045b284de2b6a87b027cce2ffcea073adf1d2/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf", size = 2241074, upload-time = "2025-04-23T18:31:19.205Z" }, - { url = "https://files.pythonhosted.org/packages/e0/ed/55532bb88f674d5d8f67ab121a2a13c385df382de2a1677f30ad385f7438/pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51", size = 1910538, upload-time = "2025-04-23T18:31:20.541Z" }, - { url = "https://files.pythonhosted.org/packages/fe/1b/25b7cccd4519c0b23c2dd636ad39d381abf113085ce4f7bec2b0dc755eb1/pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab", size = 1952909, upload-time = "2025-04-23T18:31:22.371Z" }, - { url = "https://files.pythonhosted.org/packages/49/a9/d809358e49126438055884c4366a1f6227f0f84f635a9014e2deb9b9de54/pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65", size = 1897786, upload-time = "2025-04-23T18:31:24.161Z" }, - { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" }, - { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" }, - { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" }, - { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" }, - { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" }, - { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" }, - { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" }, - { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" }, - { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" }, - { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" }, - { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" }, - { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" }, - { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859, upload-time = "2025-04-23T18:31:59.065Z" }, - { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810, upload-time = "2025-04-23T18:32:00.78Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498, upload-time = "2025-04-23T18:32:02.418Z" }, - { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611, upload-time = "2025-04-23T18:32:04.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924, upload-time = "2025-04-23T18:32:06.129Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196, upload-time = "2025-04-23T18:32:08.178Z" }, - { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389, upload-time = "2025-04-23T18:32:10.242Z" }, - { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223, upload-time = "2025-04-23T18:32:12.382Z" }, - { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473, upload-time = "2025-04-23T18:32:14.034Z" }, - { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269, upload-time = "2025-04-23T18:32:15.783Z" }, - { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921, upload-time = "2025-04-23T18:32:18.473Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162, upload-time = "2025-04-23T18:32:20.188Z" }, - { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560, upload-time = "2025-04-23T18:32:22.354Z" }, - { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" }, - { url = "https://files.pythonhosted.org/packages/7b/27/d4ae6487d73948d6f20dddcd94be4ea43e74349b56eba82e9bdee2d7494c/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8", size = 2025200, upload-time = "2025-04-23T18:33:14.199Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b8/b3cb95375f05d33801024079b9392a5ab45267a63400bf1866e7ce0f0de4/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593", size = 1859123, upload-time = "2025-04-23T18:33:16.555Z" }, - { url = "https://files.pythonhosted.org/packages/05/bc/0d0b5adeda59a261cd30a1235a445bf55c7e46ae44aea28f7bd6ed46e091/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612", size = 1892852, upload-time = "2025-04-23T18:33:18.513Z" }, - { url = "https://files.pythonhosted.org/packages/3e/11/d37bdebbda2e449cb3f519f6ce950927b56d62f0b84fd9cb9e372a26a3d5/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7", size = 2067484, upload-time = "2025-04-23T18:33:20.475Z" }, - { url = "https://files.pythonhosted.org/packages/8c/55/1f95f0a05ce72ecb02a8a8a1c3be0579bbc29b1d5ab68f1378b7bebc5057/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e", size = 2108896, upload-time = "2025-04-23T18:33:22.501Z" }, - { url = "https://files.pythonhosted.org/packages/53/89/2b2de6c81fa131f423246a9109d7b2a375e83968ad0800d6e57d0574629b/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8", size = 2069475, upload-time = "2025-04-23T18:33:24.528Z" }, - { url = "https://files.pythonhosted.org/packages/b8/e9/1f7efbe20d0b2b10f6718944b5d8ece9152390904f29a78e68d4e7961159/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf", size = 2239013, upload-time = "2025-04-23T18:33:26.621Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b2/5309c905a93811524a49b4e031e9851a6b00ff0fb668794472ea7746b448/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb", size = 2238715, upload-time = "2025-04-23T18:33:28.656Z" }, - { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" }, -] - -[[package]] -name = "pygments" -version = "2.19.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, -] - -[[package]] -name = "pyspark" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py4j" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/0e/5b38d51f1b1c2618cccfbf35093268665af9a3bdb493e5a3ecd991def633/pyspark-4.0.0.tar.gz", hash = "sha256:38db1b4f6095a080d7605e578d775528990e66dc326311d93e94a71cfc24e5a5", size = 434132212, upload-time = "2025-05-23T03:29:33.916Z" } - -[[package]] -name = "pytest" -version = "8.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, -] - -[[package]] -name = "rich" -version = "14.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, -] - -[[package]] -name = "ruff" -version = "0.12.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4b/da/5bd7565be729e86e1442dad2c9a364ceeff82227c2dece7c29697a9795eb/ruff-0.12.8.tar.gz", hash = "sha256:4cb3a45525176e1009b2b64126acf5f9444ea59066262791febf55e40493a033", size = 5242373, upload-time = "2025-08-07T19:05:47.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/1e/c843bfa8ad1114fab3eb2b78235dda76acd66384c663a4e0415ecc13aa1e/ruff-0.12.8-py3-none-linux_armv6l.whl", hash = "sha256:63cb5a5e933fc913e5823a0dfdc3c99add73f52d139d6cd5cc8639d0e0465513", size = 11675315, upload-time = "2025-08-07T19:05:06.15Z" }, - { url = "https://files.pythonhosted.org/packages/24/ee/af6e5c2a8ca3a81676d5480a1025494fd104b8896266502bb4de2a0e8388/ruff-0.12.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a9bbe28f9f551accf84a24c366c1aa8774d6748438b47174f8e8565ab9dedbc", size = 12456653, upload-time = "2025-08-07T19:05:09.759Z" }, - { url = "https://files.pythonhosted.org/packages/99/9d/e91f84dfe3866fa648c10512904991ecc326fd0b66578b324ee6ecb8f725/ruff-0.12.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2fae54e752a3150f7ee0e09bce2e133caf10ce9d971510a9b925392dc98d2fec", size = 11659690, upload-time = "2025-08-07T19:05:12.551Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ac/a363d25ec53040408ebdd4efcee929d48547665858ede0505d1d8041b2e5/ruff-0.12.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0acbcf01206df963d9331b5838fb31f3b44fa979ee7fa368b9b9057d89f4a53", size = 11896923, upload-time = "2025-08-07T19:05:14.821Z" }, - { url = "https://files.pythonhosted.org/packages/58/9f/ea356cd87c395f6ade9bb81365bd909ff60860975ca1bc39f0e59de3da37/ruff-0.12.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae3e7504666ad4c62f9ac8eedb52a93f9ebdeb34742b8b71cd3cccd24912719f", size = 11477612, upload-time = "2025-08-07T19:05:16.712Z" }, - { url = "https://files.pythonhosted.org/packages/1a/46/92e8fa3c9dcfd49175225c09053916cb97bb7204f9f899c2f2baca69e450/ruff-0.12.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb82efb5d35d07497813a1c5647867390a7d83304562607f3579602fa3d7d46f", size = 13182745, upload-time = "2025-08-07T19:05:18.709Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c4/f2176a310f26e6160deaf661ef60db6c3bb62b7a35e57ae28f27a09a7d63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dbea798fc0065ad0b84a2947b0aff4233f0cb30f226f00a2c5850ca4393de609", size = 14206885, upload-time = "2025-08-07T19:05:21.025Z" }, - { url = "https://files.pythonhosted.org/packages/87/9d/98e162f3eeeb6689acbedbae5050b4b3220754554526c50c292b611d3a63/ruff-0.12.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:49ebcaccc2bdad86fd51b7864e3d808aad404aab8df33d469b6e65584656263a", size = 13639381, upload-time = "2025-08-07T19:05:23.423Z" }, - { url = "https://files.pythonhosted.org/packages/81/4e/1b7478b072fcde5161b48f64774d6edd59d6d198e4ba8918d9f4702b8043/ruff-0.12.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ac9c570634b98c71c88cb17badd90f13fc076a472ba6ef1d113d8ed3df109fb", size = 12613271, upload-time = "2025-08-07T19:05:25.507Z" }, - { url = "https://files.pythonhosted.org/packages/e8/67/0c3c9179a3ad19791ef1b8f7138aa27d4578c78700551c60d9260b2c660d/ruff-0.12.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:560e0cd641e45591a3e42cb50ef61ce07162b9c233786663fdce2d8557d99818", size = 12847783, upload-time = "2025-08-07T19:05:28.14Z" }, - { url = "https://files.pythonhosted.org/packages/4e/2a/0b6ac3dd045acf8aa229b12c9c17bb35508191b71a14904baf99573a21bd/ruff-0.12.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:71c83121512e7743fba5a8848c261dcc454cafb3ef2934a43f1b7a4eb5a447ea", size = 11702672, upload-time = "2025-08-07T19:05:30.413Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ee/f9fdc9f341b0430110de8b39a6ee5fa68c5706dc7c0aa940817947d6937e/ruff-0.12.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:de4429ef2ba091ecddedd300f4c3f24bca875d3d8b23340728c3cb0da81072c3", size = 11440626, upload-time = "2025-08-07T19:05:32.492Z" }, - { url = "https://files.pythonhosted.org/packages/89/fb/b3aa2d482d05f44e4d197d1de5e3863feb13067b22c571b9561085c999dc/ruff-0.12.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a2cab5f60d5b65b50fba39a8950c8746df1627d54ba1197f970763917184b161", size = 12462162, upload-time = "2025-08-07T19:05:34.449Z" }, - { url = "https://files.pythonhosted.org/packages/18/9f/5c5d93e1d00d854d5013c96e1a92c33b703a0332707a7cdbd0a4880a84fb/ruff-0.12.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:45c32487e14f60b88aad6be9fd5da5093dbefb0e3e1224131cb1d441d7cb7d46", size = 12913212, upload-time = "2025-08-07T19:05:36.541Z" }, - { url = "https://files.pythonhosted.org/packages/71/13/ab9120add1c0e4604c71bfc2e4ef7d63bebece0cfe617013da289539cef8/ruff-0.12.8-py3-none-win32.whl", hash = "sha256:daf3475060a617fd5bc80638aeaf2f5937f10af3ec44464e280a9d2218e720d3", size = 11694382, upload-time = "2025-08-07T19:05:38.468Z" }, - { url = "https://files.pythonhosted.org/packages/f6/dc/a2873b7c5001c62f46266685863bee2888caf469d1edac84bf3242074be2/ruff-0.12.8-py3-none-win_amd64.whl", hash = "sha256:7209531f1a1fcfbe8e46bcd7ab30e2f43604d8ba1c49029bb420b103d0b5f76e", size = 12740482, upload-time = "2025-08-07T19:05:40.391Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5c/799a1efb8b5abab56e8a9f2a0b72d12bd64bb55815e9476c7d0a2887d2f7/ruff-0.12.8-py3-none-win_arm64.whl", hash = "sha256:c90e1a334683ce41b0e7a04f41790c429bf5073b62c1ae701c9dc5b3d14f0749", size = 11884718, upload-time = "2025-08-07T19:05:42.866Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - -[[package]] -name = "typer" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/8c/7d682431efca5fd290017663ea4588bf6f2c6aad085c7f108c5dbc316e70/typer-0.16.0.tar.gz", hash = "sha256:af377ffaee1dbe37ae9440cb4e8f11686ea5ce4e9bae01b84ae7c63b87f1dd3b", size = 102625, upload-time = "2025-05-26T14:30:31.824Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/42/3efaf858001d2c2913de7f354563e3a3a2f0decae3efe98427125a8f441e/typer-0.16.0-py3-none-any.whl", hash = "sha256:1f79bed11d4d02d4310e3c1b7ba594183bcedb0ac73b27a9e5f28f6fb5b98855", size = 46317, upload-time = "2025-05-26T14:30:30.523Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.14.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.2.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, -] - -[[package]] -name = "zipp" -version = "3.23.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, -] diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index ded051af..c557f995 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -171,6 +171,49 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t) } + test("prior_commits_removed") { + sql("CREATE TABLE tbl (id INT, value STRING) USING delta") + sql("INSERT INTO tbl VALUES (1, 'a')") + sql("INSERT OVERWRITE tbl VALUES (2, 'b')") + sql("INSERT OVERWRITE tbl VALUES (3, 'c')") + checkpoint("tbl") + val t = registerTable("tbl") + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + java.nio.file.Files.delete(logDir.resolve("00000000000000000000.json")) + java.nio.file.Files.delete(logDir.resolve("00000000000000000001.json")) + } + readSpec(t) + snapshotSpec(t) + } + + test("struct_stats_only") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ( + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + sql("INSERT INTO tbl VALUES (1, 'a'), (2, 'b')") + sql("INSERT INTO tbl VALUES (3, 'c')") + checkpoint("tbl") + val t = registerTable("tbl") + readSpec(t) + snapshotSpec(t) + } + + test("stats_disabled") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ( + 'delta.checkpoint.writeStatsAsStruct' = 'false', + 'delta.checkpoint.writeStatsAsJson' = 'false', + 'delta.dataSkippingNumIndexedCols' = '0')""") + sql("INSERT INTO tbl VALUES (1, 'a'), (2, 'b')") + sql("INSERT OVERWRITE tbl VALUES (3, 'c')") + checkpoint("tbl") + val t = registerTable("tbl") + readSpec(t) + snapshotSpec(t) + } + test("v2_basic") { sql("""CREATE TABLE tbl (id INT, name STRING) USING delta TBLPROPERTIES ('delta.checkpointPolicy' = 'v2', diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ProtocolVersionsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ProtocolVersionsSuite.scala index e918374c..9516c555 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ProtocolVersionsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ProtocolVersionsSuite.scala @@ -24,6 +24,15 @@ import io.delta.workload.WorkloadTestSuite */ class ProtocolVersionsSuite extends WorkloadTestSuite("protocol_versions") { + test("iceberg_compat_v1") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.enableIcebergCompatV1' = 'true')""") + sql("INSERT INTO tbl VALUES (1, 'a'), (2, 'b')") + val t = registerTable("tbl") + readSpec(t) + snapshotSpec(t) + } + // pv_001*: Basic protocol version tables test("protocol_1_1") {