diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 0d3da54..feb1df3 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - run: python -m pip install flake8 - name: flake8 uses: liskin/gh-problem-matcher-wrap@v3 @@ -32,7 +32,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - run: python -m pip install isort - name: isort uses: liskin/gh-problem-matcher-wrap@v3 diff --git a/.github/workflows/snowflake_settings.py b/.github/workflows/snowflake_settings.py index 1fd26ea..c47173f 100644 --- a/.github/workflows/snowflake_settings.py +++ b/.github/workflows/snowflake_settings.py @@ -21,7 +21,6 @@ 'TEST': {'NAME': 'TEST_DJANGO_OTHER_' + str(uuid.uuid4()).upper()}, }, } -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', ) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2421955..66c7948 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'timgraham/django' - ref: 'snowflake-5.2.x' + ref: 'snowflake-6.0.x' path: 'django_repo' - name: Install system packages for Django's Python test dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 27af77e..d72fc18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## 5.2 - 2025-05-27 +## 6.0 - 2025-12-05 -Initial release for Django 5.2.x. +Initial release for Django 6.0.x. diff --git a/README.md b/README.md index 98dd7fc..8168e93 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ ## Install and usage Use the version of django-snowflake that corresponds to your version of -Django. For example, to get the latest compatible release for Django 5.2.x: +Django. For example, to get the latest compatible release for Django 6.0.x: -`pip install django-snowflake==5.2.*` +`pip install django-snowflake==6.0.*` The minor release number of Django doesn't correspond to the minor release number of django-snowflake. Use the latest minor release of each. diff --git a/django_snowflake/__init__.py b/django_snowflake/__init__.py index 777e76a..30c7df0 100644 --- a/django_snowflake/__init__.py +++ b/django_snowflake/__init__.py @@ -1,4 +1,4 @@ -__version__ = '5.2' +__version__ = '6.0' # Check Django compatibility before other imports which may fail if the # wrong version of Django is installed. @@ -6,10 +6,12 @@ check_django_compatability() +from .aggregates import register_aggregates # noqa from .expressions import register_expressions # noqa from .functions import register_functions # noqa from .lookups import register_lookups # noqa +register_aggregates() register_expressions() register_functions() register_lookups() diff --git a/django_snowflake/aggregates.py b/django_snowflake/aggregates.py new file mode 100644 index 0000000..c870dcd --- /dev/null +++ b/django_snowflake/aggregates.py @@ -0,0 +1,9 @@ +from django.db.models.aggregates import StringAgg + + +def string_agg(self, compiler, connection, **extra_context): + return self.as_sql(compiler, connection, function="LISTAGG", **extra_context) + + +def register_aggregates(): + StringAgg.as_snowflake = string_agg diff --git a/django_snowflake/compiler.py b/django_snowflake/compiler.py index 8bbf430..c03f077 100644 --- a/django_snowflake/compiler.py +++ b/django_snowflake/compiler.py @@ -1,3 +1,4 @@ +from functools import partial from itertools import chain from django.db.models import JSONField @@ -6,6 +7,7 @@ class SQLInsertCompiler(compiler.SQLInsertCompiler): def as_sql(self): + """Overridden to to wrap JSONField values with parse_json().""" # We don't need quote_name_unless_alias() here, since these are all # going to be column names (so we can avoid the extra overhead). qn = self.connection.ops.quote_name @@ -14,33 +16,74 @@ def as_sql(self): on_conflict=self.query.on_conflict, ) result = ["%s %s" % (insert_statement, qn(opts.db_table))] - fields = self.query.fields or [opts.pk] - result.append("(%s)" % ", ".join(qn(f.column) for f in fields)) - select_columns = [] - if self.query.fields: - value_rows = [ - [ - self.prepare_value(field, self.pre_save_val(field, obj)) - for field in fields - ] - for obj in self.query.objs - ] + if fields := list(self.query.fields): + from django.db.models.expressions import DatabaseDefault + + supports_default_keyword_in_bulk_insert = ( + self.connection.features.supports_default_keyword_in_bulk_insert + ) + value_cols = [] has_json_field = False - for i, field in enumerate(fields, 1): + for i, field in enumerate(list(fields), 1): if isinstance(field, JSONField): has_json_field = True select_columns.append(f'parse_json(${i})') else: select_columns.append(f'${i}') + + field_prepare = partial(self.prepare_value, field) + field_pre_save = partial(self.pre_save_val, field) + field_values = [ + field_prepare(field_pre_save(obj)) for obj in self.query.objs + ] + if not field.has_db_default(): + value_cols.append(field_values) + continue + + # If all values are DEFAULT don't include the field and its + # values in the query as they are redundant and could prevent + # optimizations. This cannot be done if we're dealing with the + # last field as INSERT statements require at least one. + if len(fields) > 1 and all( + isinstance(value, DatabaseDefault) for value in field_values + ): + fields.remove(field) + continue + + if supports_default_keyword_in_bulk_insert: + value_cols.append(field_values) + continue + + # If the field cannot be excluded from the INSERT for the + # reasons listed above and the backend doesn't support the + # DEFAULT keyword each values must be expanded into their + # underlying expressions. + prepared_db_default = field_prepare(field.db_default) + field_values = [ + ( + prepared_db_default + if isinstance(value, DatabaseDefault) + else value + ) + for value in field_values + ] + value_cols.append(field_values) + value_rows = list(zip(*value_cols)) + result.append("(%s)" % ", ".join(qn(f.column) for f in fields)) + if not has_json_field: select_columns = [] else: - # An empty object. + # No fields were specified but an INSERT statement must include at + # least one column. This can only happen when the model's primary + # key is composed of a single auto-field so default to including it + # as a placeholder to generate a valid INSERT statement. value_rows = [ [self.connection.ops.pk_default_value()] for _ in self.query.objs ] fields = [None] + result.append("(%s)" % qn(opts.pk.column)) # Currently the backends just accept values when generating bulk # queries and generate their own placeholders. Doing that isn't @@ -94,7 +137,7 @@ def as_sql(self): if on_conflict_suffix_sql: result.append(on_conflict_suffix_sql) return [ - (" ".join(result + ["VALUES (%s)" % ", ".join(p)]), vals) + (" ".join([*result, "VALUES (%s)" % ", ".join(p)]), vals) for p, vals in zip(placeholder_rows, param_rows) ] diff --git a/django_snowflake/features.py b/django_snowflake/features.py index 6c1f67d..b644842 100644 --- a/django_snowflake/features.py +++ b/django_snowflake/features.py @@ -45,6 +45,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_json_field_contains = False # This feature is specific to the Django fork used for testing. supports_limit_in_exists = False + supports_json_negative_indexing = False supports_over_clause = True supports_partial_indexes = False # https://docs.snowflake.com/en/sql-reference/functions-regexp.html#backreferences @@ -108,6 +109,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): 'expressions.tests.BasicExpressionsTests.test_object_create_with_f_expression_in_subquery', # JSONField queries with complex JSON parameters don't work: # https://github.com/Snowflake-Labs/django-snowflake/issues/58 + # Query: not analyzed + 'model_fields.test_jsonfield.TestQuerying.test_cast_with_key_text_transform', # Query: # WHERE "MODEL_FIELDS_NULLABLEJSONMODEL"."VALUE" = 'null' # needs to operate as: @@ -174,6 +177,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): 'model_fields.test_jsonfield.TestSaveLoad.test_bulk_update_custom_get_prep_value', # AssertionError: possibly a server bug that returns the array as a string? 'db_functions.json.test_json_array.JSONArrayTests.test_expressions', + # LISTAGG returns empty string rather than NULL + 'aggregation.tests.AggregateTestCase.test_stringagg_default_value', } django_test_skips = { @@ -185,6 +190,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): }, 'Snowflake does not enforce UNIQUE constraints.': { 'auth_tests.test_basic.BasicTestCase.test_unicode_username', + 'auth_tests.test_management.PermissionRenameOperationsTests.test_rename_permission_conflict', 'auth_tests.test_migrations.ProxyModelWithSameAppLabelTests.test_migrate_with_existing_target_permission', 'composite_pk.test_create.CompositePKCreateTests.test_save_default_pk_set', 'composite_pk.tests.CompositePKTests.test_error_on_comment_pk_conflict', @@ -231,6 +237,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): 'aggregation.tests.AggregateAnnotationPruningTests.test_referenced_subquery_requires_wrapping', 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation', 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values', + 'aggregation.tests.AggregateTestCase.test_string_agg_filter_in_subquery', 'annotations.tests.NonAggregateAnnotationTestCase.test_annotation_filter_with_subquery', 'annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_outerref_transform', 'composite_pk.test_filter.CompositePKFilterTests.test_outer_ref_pk', @@ -240,6 +247,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): 'expressions.tests.BasicExpressionsTests.test_annotation_with_deeply_nested_outerref', 'expressions.tests.BasicExpressionsTests.test_annotation_with_nested_outerref', 'expressions.tests.BasicExpressionsTests.test_annotation_with_outerref', + 'expressions.tests.BasicExpressionsTests.test_annotation_with_outerref_and_output_field', 'expressions.tests.BasicExpressionsTests.test_annotations_within_subquery', 'expressions.tests.BasicExpressionsTests.test_nested_outerref_with_function', 'expressions.tests.BasicExpressionsTests.test_nested_subquery', @@ -300,6 +308,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): 'transaction_hooks.tests.TestConnectionOnCommit.test_discards_hooks_from_rolled_back_savepoint', 'transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_rolled_back_with_outer', 'transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_does_not_affect_outer', + 'update_only_fields.tests.UpdateOnlyFieldsTests.test_update_fields_not_updated', }, 'Unused DatabaseIntrospection.get_sequences() not implemented.': { 'introspection.tests.IntrospectionTests.test_sequence_list', @@ -353,14 +362,12 @@ class DatabaseFeatures(BaseDatabaseFeatures): }, 'assertNumQueries is sometimes off because of the extra queries this ' 'backend uses to fetch an object\'s ID.': { - 'admin_utils.test_logentry.LogEntryTests.test_log_action_fallback', 'admin_utils.test_logentry.LogEntryTests.test_log_actions_single_object_param', 'contenttypes_tests.test_models.ContentTypesTests.test_get_for_models_creation', 'force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_diamond_mti', 'force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_false', 'force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_parent', 'force_insert_update.tests.ForceInsertInheritanceTests.test_force_insert_with_grandparent', - 'modeladmin.tests.ModelAdminTests.test_log_deletion_fallback', 'model_formsets_regress.tests.FormsetTests.test_extraneous_query_is_not_run', 'model_inheritance.tests.ModelInheritanceTests.test_create_child_no_update', 'model_inheritance.tests.ModelInheritanceTests.test_create_diamond_mti_common_parent', @@ -376,6 +383,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): }, "Snowflake: Unsupported feature 'Alter Column Set Default'.": { 'migrations.test_operations.OperationTests.test_alter_field_add_database_default', + 'migrations.test_operations.OperationTests.test_alter_field_add_database_default_func', 'migrations.test_operations.OperationTests.test_alter_field_change_default_to_database_default', 'migrations.test_operations.OperationTests.test_alter_field_change_nullable_to_database_default_not_null', 'migrations.test_operations.OperationTests.test_alter_field_change_nullable_to_decimal_database_default_not_null', # noqa @@ -385,6 +393,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): }, "Snowflake: Unsupported: Scalar subquery with multi-column SELECT clause.": { 'composite_pk.test_filter.CompositePKFilterTests.test_filter_comments_by_pk_exact_subquery', + 'composite_pk.test_filter.CompositePKFilterTests.test_outer_ref_pk_filter_on_pk_comparison', + 'composite_pk.test_filter.CompositePKFilterTests.test_outer_ref_pk_filter_on_pk_exact', } } diff --git a/django_snowflake/lookups.py b/django_snowflake/lookups.py index e691316..9a04a83 100644 --- a/django_snowflake/lookups.py +++ b/django_snowflake/lookups.py @@ -1,3 +1,4 @@ +from django.db import NotSupportedError from django.db.models.fields.json import ( HasKeyLookup, KeyTextTransform, KeyTransform, ) @@ -15,8 +16,14 @@ def compile_json_path(key_transforms): transform = transform.replace('"', '\\"') json_path += f'{separator}"{transform}"' else: - # An integer lookup is an array index. - json_path += f'[{idx}]' + if idx < 0: + raise NotSupportedError( + "Using negative JSON array indices is not supported on this " + "database backend." + ) + else: + # An integer lookup is an array index. + json_path += f'[{idx}]' # Escape percent literals since snowflake-connector-python uses # interpolation to bind parameters. return json_path.replace('%', '%%') @@ -51,7 +58,7 @@ def has_key_lookup(self, compiler, connection): rhs_key_transforms = [key] *rhs_key_transforms, final_key = rhs_key_transforms rhs_json_path = compile_json_path(rhs_key_transforms) - final_key = self.compile_json_path_final_key(final_key) + final_key = self.compile_json_path_final_key(connection, final_key) # If this is the only key, the separator must be a colon. if rhs_json_path == '': final_key = final_key.replace('.', ':', 1) diff --git a/setup.cfg b/setup.cfg index 3b15e5b..7fe1e6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,13 +11,11 @@ long_description_content_type = text/markdown classifiers = Development Status :: 5 - Production/Stable Framework :: Django - Framework :: Django :: 5.2 + Framework :: Django :: 6.0 License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.13 Programming Language :: Python :: 3.14 @@ -26,10 +24,10 @@ project_urls = Tracker = https://github.com/Snowflake-Labs/django-snowflake/issues [options] -python_requires = >=3.10 +python_requires = >=3.12 packages = find: install_requires = - django >= 5.2, < 6.0 + django >= 6.0, < 6.1 snowflake-connector-python >= 3.6.0 [flake8]