feat: Add LoaderConfig defaults, align CI linting, and update examples

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ main, master ]
+    branches: [ main]
   pull_request:
-    branches: [ main, master ]
+    branches: [ main]
   schedule:
     # Run tests weekly on Monday at 00:00 UTC to catch dependency issues
     - cron: '0 0 * * 1'
@@ -17,12 +17,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: "pip"
-      - name: Install hatch
-        run: pip install hatch
-      - name: Lint
-        run: hatch run lint
+      - name: Install Ruff
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ruff
+      - name: Lint (Ruff, config from pyproject.toml)
+        run: ruff check --output-format=github .
 
   build:
     needs: lint
@@ -42,5 +44,17 @@ jobs:
       - name: Type check
         run: hatch run types:check
       - name: Test
-        run: hatch run test -- --cov=iceberg_loader --cov=tests --cov-report=xml
+        run: hatch run test -- --cov=iceberg_loader --cov=tests --cov-report=xml --junitxml=pytest-report.xml
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: coverage.xml
+          if-no-files-found: error
+      - name: Upload pytest report
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-report-${{ matrix.python-version }}
+          path: pytest-report.xml
+          if-no-files-found: error
 

.github/workflows/docs.yml

@@ -2,7 +2,7 @@ name: Documentation
 
 on:
   push:
-    branches: [ main, master ]
+    branches: [ main]
   workflow_dispatch:
 
 permissions:

.pre-commit-config.yaml

@@ -3,10 +3,10 @@ repos:
     rev: v0.6.9
     hooks:
       - id: ruff
-        args: [--fix, "--target-version=py310"]
+        args: [--fix, "--config=pyproject.toml"]
         exclude: ^examples/
       - id: ruff-format
-        args: ["--target-version=py310"]
+        args: ["--config=pyproject.toml"]
         exclude: ^examples/
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.13.0

examples/README.md

@@ -7,9 +7,9 @@ This directory contains runnable examples demonstrating various features of `ice
 You need a running Iceberg catalog (e.g., REST catalog) and MinIO/S3.
 A `docker-compose.yml` is provided to spin up a local environment.
 
-```bash
-docker-compose up -d
-```
+   ```bash
+   docker-compose up -d
+   ```
 
 ## Running Examples
 

examples/advanced_scenarios.py

@@ -1,8 +1,14 @@
 import logging
+import sys
+from pathlib import Path
+
+# Ensure parent directory (examples/) is on path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 from catalog import get_catalog
 
-from iceberg_loader import load_data_to_iceberg
+from iceberg_loader import LoaderConfig, load_data_to_iceberg
 from iceberg_loader.arrow_utils import create_arrow_table_from_data
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -27,14 +33,8 @@ def scenario_initial_append(catalog):
     ]
     table_arrow = create_arrow_table_from_data(data_day_1)
 
-    load_data_to_iceberg(
-        table_data=table_arrow,
-        table_identifier=table_id,
-        catalog=catalog,
-        write_mode='append',
-        partition_col='ts',
-        schema_evolution=True,
-    )
+    config = LoaderConfig(write_mode='append', partition_col='ts', schema_evolution=True)
+    load_data_to_iceberg(table_data=table_arrow, table_identifier=table_id, catalog=catalog, config=config)
     verify_table(catalog, table_id, expected_rows=2)
 
 
@@ -47,13 +47,12 @@ def scenario_append_new_partition(catalog):
         {'id': 1, 'category': 'A', 'ts': '2023-01-01', 'value': 100},
         {'id': 2, 'category': 'B', 'ts': '2023-01-01', 'value': 200},
     ]
+    config = LoaderConfig(write_mode='append', partition_col='month(ts)', schema_evolution=True)
     load_data_to_iceberg(
         table_data=create_arrow_table_from_data(data_day_1),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        partition_col='month(ts)',
-        schema_evolution=True,
+        config=config,
     )
     # Append day 2
     data_day_2 = [
@@ -63,7 +62,7 @@ def scenario_append_new_partition(catalog):
         table_data=create_arrow_table_from_data(data_day_2),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
+        config=LoaderConfig(write_mode='append'),
     )
     verify_table(catalog, table_id, expected_rows=3)
 
@@ -78,26 +77,25 @@ def scenario_idempotent_replace_partition(catalog):
         {'id': 2, 'category': 'B', 'ts': '2023-01-01', 'value': 200},
         {'id': 3, 'category': 'A', 'ts': '2023-01-02', 'value': 150},
     ]
+    config_base = LoaderConfig(write_mode='append', partition_col='ts', schema_evolution=True)
     load_data_to_iceberg(
         table_data=create_arrow_table_from_data(base_data),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        partition_col='ts',
-        schema_evolution=True,
+        config=config_base,
     )
 
     # Corrected day1
     corrected_day1 = [
         {'id': 1, 'category': 'A', 'ts': '2023-01-01', 'value': 999},
         {'id': 2, 'category': 'B', 'ts': '2023-01-01', 'value': 200},
     ]
+    config_replace = LoaderConfig(write_mode='append', replace_filter="ts == '2023-01-01'")
     load_data_to_iceberg(
         table_data=create_arrow_table_from_data(corrected_day1),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        replace_filter="ts == '2023-01-01'",
+        config=config_replace,
     )
     verify_table(catalog, table_id, expected_rows=3)
 
@@ -109,13 +107,12 @@ def scenario_schema_evolution(catalog):
     base_data = [
         {'id': 1, 'category': 'A', 'ts': '2023-01-01', 'value': 100},
     ]
+    config_base = LoaderConfig(write_mode='append', partition_col='ts', schema_evolution=True)
     load_data_to_iceberg(
         table_data=create_arrow_table_from_data(base_data),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        partition_col='ts',
-        schema_evolution=True,
+        config=config_base,
     )
 
     evolved = [
@@ -125,8 +122,7 @@ def scenario_schema_evolution(catalog):
         table_data=create_arrow_table_from_data(evolved),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        schema_evolution=True,
+        config=LoaderConfig(write_mode='append', schema_evolution=True),
     )
     verify_table(catalog, table_id, expected_rows=2)
 
@@ -145,13 +141,12 @@ def scenario_full_overwrite(catalog):
         {'id': 1, 'category': 'A', 'ts': '2023-01-01', 'value': 100},
         {'id': 2, 'category': 'B', 'ts': '2023-01-02', 'value': 200},
     ]
+    config_base = LoaderConfig(write_mode='append', partition_col='ts', schema_evolution=True)
     load_data_to_iceberg(
         table_data=create_arrow_table_from_data(initial),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        partition_col='ts',
-        schema_evolution=True,
+        config=config_base,
     )
 
     replace_all = [
@@ -161,8 +156,7 @@ def scenario_full_overwrite(catalog):
         table_data=create_arrow_table_from_data(replace_all),
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='overwrite',
-        schema_evolution=True,
+        config=LoaderConfig(write_mode='overwrite', schema_evolution=True),
     )
     verify_table(catalog, table_id, expected_rows=1)
 

examples/compare_complex_json_fail.py

@@ -1,9 +1,16 @@
 import logging
+import sys
+from pathlib import Path
 
 import pyarrow as pa
+
+# Ensure parent directory (examples/) is on path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from catalog import get_catalog
 
-from iceberg_loader import load_data_to_iceberg
+from iceberg_loader import LoaderConfig, load_data_to_iceberg
 from iceberg_loader.arrow_utils import create_arrow_table_from_data
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -40,12 +47,12 @@ def run_comparison():
         logger.info("Created Arrow table with schema:\n%s", arrow_table.schema)
 
         # 2. Load to Iceberg
+        config = LoaderConfig(write_mode='overwrite', schema_evolution=True)
         load_data_to_iceberg(
             table_data=arrow_table,
             table_identifier=table_id,
             catalog=catalog,
-            write_mode='overwrite',
-            schema_evolution=True,
+            config=config,
         )
         logger.info("Successfully loaded data to Iceberg table '%s'", table_id)
 

examples/load_complex_json.py

@@ -1,9 +1,15 @@
 import json
 import logging
+import sys
+from pathlib import Path
+
+# Ensure parent directory (examples/) is on path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 from catalog import get_catalog
 
-from iceberg_loader import load_data_to_iceberg
+from iceberg_loader import LoaderConfig, load_data_to_iceberg
 from iceberg_loader.arrow_utils import create_arrow_table_from_data
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -43,9 +49,8 @@ def run_complex_load():
     logger.info(arrow_table.to_pydict())
 
     logger.info('\nLoading to Iceberg...')
-    load_data_to_iceberg(
-        table_data=arrow_table, table_identifier=table_id, catalog=catalog, write_mode='append', schema_evolution=True
-    )
+    config = LoaderConfig(write_mode='append', schema_evolution=True)
+    load_data_to_iceberg(table_data=arrow_table, table_identifier=table_id, catalog=catalog, config=config)
 
     logger.info('\nVerifying data in Iceberg...')
     table = catalog.load_table(table_id)

examples/load_example.py

@@ -2,7 +2,7 @@
 
 from catalog import get_catalog
 
-from iceberg_loader import load_data_to_iceberg
+from iceberg_loader import LoaderConfig, load_data_to_iceberg
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -54,16 +54,17 @@ def run_example():
     except Exception:
         pass
 
+    config = LoaderConfig(
+        write_mode='append',
+        partition_col='signup_date',
+        schema_evolution=True,
+    )
+
     result = load_data_to_iceberg(
         table_data=arrow_table,
         table_identifier=table_id,
         catalog=catalog,
-        write_mode='append',
-        # New flexible partitioning:
-        partition_col='signup_date',
-        # Optional: idempotency filter (if we were reloading data for a specific date)
-        # replace_filter="signup_date == '2023-01-01'",
-        schema_evolution=True,
+        config=config,
     )
 
     logger.info('Load result: %s', result)

examples/load_upsert.py

@@ -1,11 +1,18 @@
 import logging
+import sys
 import time
 from datetime import datetime
+from pathlib import Path
 
 import pyarrow as pa
+
+# Ensure parent directory (examples/) is on path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from catalog import get_catalog
 
-from iceberg_loader import load_data_to_iceberg
+from iceberg_loader import LoaderConfig, load_data_to_iceberg
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -32,12 +39,8 @@ def run_upsert_example():
         }
     )
 
-    load_data_to_iceberg(
-        initial_data,
-        table_identifier,
-        catalog,
-        write_mode="overwrite",
-    )
+    config_overwrite = LoaderConfig(write_mode="overwrite")
+    load_data_to_iceberg(initial_data, table_identifier, catalog, config=config_overwrite)
 
     table = catalog.load_table(table_identifier)
     rows = table.scan().to_arrow()
@@ -56,13 +59,8 @@ def run_upsert_example():
         }
     )
 
-    load_data_to_iceberg(
-        upsert_data,
-        table_identifier,
-        catalog,
-        write_mode="upsert",
-        join_cols=["id"],
-    )
+    config_upsert = LoaderConfig(write_mode="upsert", join_cols=["id"])
+    load_data_to_iceberg(upsert_data, table_identifier, catalog, config_upsert)
 
     rows_after = table.scan().to_arrow()
     logger.info("Rows after upsert: %d", len(rows_after))