From b404a524375bf2086e3ba34ca8a1a836462a076f Mon Sep 17 00:00:00 2001 From: Arthur Vickers Date: Mon, 24 Aug 2026 15:56:13 +0100 Subject: [PATCH 001/186] Working: squash of EF-254, EF-295, EF-221, EF-218, EF-253, EF-359, EF-358, EF-376, EF-379, EF-380 and merged upstream PRs #339, #350, #353, #355, #357, #360, #366, #367, #368 --- docs/failing-spec-tests.md | 1 - ...ngoEFToLinqTranslatingExpressionVisitor.cs | 199 ++++++++++++++++ ...MongoProjectionBindingExpressionVisitor.cs | 33 +++ ...yableMethodTranslatingExpressionVisitor.cs | 120 +++++++++- .../DateTimeOffsetMemberProjectionTests.cs | 225 ++++++++++++++++++ .../Query/OwnedEntityTests.cs | 170 +++++++++++++ .../Mapping/BuiltInDataTypesMongoTest.cs | 14 +- .../NorthwindMiscellaneousQueryMongoTest.cs | 8 +- ...MethodTranslatingExpressionVisitorTests.cs | 203 ++++++++++++++++ 9 files changed, 954 insertions(+), 19 deletions(-) create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/DateTimeOffsetMemberProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitorTests.cs diff --git a/docs/failing-spec-tests.md b/docs/failing-spec-tests.md index 3dea4e0f3..606492f16 100644 --- a/docs/failing-spec-tests.md +++ b/docs/failing-spec-tests.md @@ -41,7 +41,6 @@ that currently lack a ticket. Counts are sourced from `tests/MongoDB.EntityFrame | [EF-202](https://jira.mongodb.org/browse/EF-202) | `Entity equality issue EF-202` | Comparing two entities (`entity1 == entity2` / `Contains(entity)`) is not lowered to a key-equality comparison. | 4 | | [EF-216](https://jira.mongodb.org/browse/EF-216) | `Cross-document navigation access issue EF-216` / `Navigations issue EF-216` | Navigations that cross collection boundaries cannot be translated; surfaces as `Unsupported cross-DbSet query between ...`. Documented at the helper `AssertNoMultiCollectionQuerySupport`. | 211 | | [EF-217](https://jira.mongodb.org/browse/EF-217) | `Call ToString on DateTimeOffset EF-217` | `DateTimeOffset.ToString()` cannot be translated. | 2 | -| [EF-218](https://jira.mongodb.org/browse/EF-218) | `Projecting DateTimeOffset members EF-218` | Projecting individual members of a `DateTimeOffset` (e.g. `.Year`, `.Hour`) is not supported. | 2 | | [EF-220](https://jira.mongodb.org/browse/EF-220) | `Multiple query roots issue EF-220` | Queries that reference more than one `DbSet<>` (Cartesian product / cross-join) are not translatable. Includes `SelectMany` across DbSets and tautology-predicate cross-joins. | 10 | | [EF-221](https://jira.mongodb.org/browse/EF-221) | _(no remaining `// Fails:` tags)_ | `Equals` calls with genuinely mismatched, non-convertible simple types (e.g. `int?.Equals((ulong)x)`) are now folded to the correct constant result in `MongoEFToLinqTranslatingExpressionVisitor` instead of reaching the driver's `Equals` translator, which threw `InvalidCastException`. The four tests formerly tagged here now pass. | 0 | | [EF-222](https://jira.mongodb.org/browse/EF-222) | `translation of Like issue EF-222` | `EF.Functions.Like(...)` is not translated. | 9 | diff --git a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoEFToLinqTranslatingExpressionVisitor.cs b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoEFToLinqTranslatingExpressionVisitor.cs index 94d1b8266..065db51aa 100644 --- a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoEFToLinqTranslatingExpressionVisitor.cs +++ b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoEFToLinqTranslatingExpressionVisitor.cs @@ -44,6 +44,29 @@ internal sealed partial class MongoEFToLinqTranslatingExpressionVisitor : System typeof(Mql).GetMethods(BindingFlags.Public | BindingFlags.Static) .Single(m => m.Name == nameof(Mql.Field) && m.GetParameters().Length == 3); + // DateTimeOffset members whose translation is rewritten below to work around the driver not + // implementing IBsonDocumentSerializer on DateTimeOffsetSerializer (CSHARP-5296 / EF-218). + private static readonly HashSet DateTimeOffsetComponentMembers = + [ + nameof(DateTimeOffset.DateTime), + nameof(DateTimeOffset.LocalDateTime), + nameof(DateTimeOffset.UtcDateTime), + nameof(DateTimeOffset.Date), + nameof(DateTimeOffset.TimeOfDay), + nameof(DateTimeOffset.Year), + nameof(DateTimeOffset.Month), + nameof(DateTimeOffset.Day), + nameof(DateTimeOffset.Hour), + nameof(DateTimeOffset.Minute), + nameof(DateTimeOffset.Second), + nameof(DateTimeOffset.Millisecond), + nameof(DateTimeOffset.DayOfWeek), + nameof(DateTimeOffset.DayOfYear) + ]; + + private static readonly MethodInfo DateTimeAddMinutesMethodInfo = + typeof(DateTime).GetMethod(nameof(DateTime.AddMinutes), [typeof(double)])!; + private readonly QueryContext _queryContext; private readonly Expression _source; private readonly BsonSerializerFactory _bsonSerializerFactory; @@ -429,6 +452,49 @@ when _innerSources.Count > 0 return navCall.ConvertIfRequired(memberExpression.Type); } + // Rewrite DateTimeOffset.DateTime/.Year/.Date/etc member access into a UTC-field + + // Offset-field reconstruction that the driver's own DateTime member translator can + // then handle natively. Works around CSHARP-5296: the driver's DateTimeOffsetSerializer + // does not implement IBsonDocumentSerializer, so member access directly off a + // DateTimeOffset-typed expression fails in the driver's LINQ translator. See EF-218. + case MemberExpression { Expression: { } dateTimeOffsetSource } dateTimeOffsetMember + when dateTimeOffsetSource.Type == typeof(DateTimeOffset) + && DateTimeOffsetComponentMembers.Contains(dateTimeOffsetMember.Member.Name): + + if (TryResolveDateTimeOffsetElementAccess(dateTimeOffsetSource, out var dtoDocSource, out var dtoElementName)) + { + var dtoDocMethod = MqlFieldMethodInfo.MakeGenericMethod(dtoDocSource.Type, typeof(BsonValue)); + var dtoDoc = Expression.Call(null, dtoDocMethod, dtoDocSource, + Expression.Constant(dtoElementName), Expression.Constant(BsonValueSerializer.Instance)); + + var utcFieldMethod = MqlFieldMethodInfo.MakeGenericMethod(typeof(BsonValue), typeof(DateTime)); + var utcField = Expression.Call(null, utcFieldMethod, dtoDoc, + Expression.Constant("DateTime"), Expression.Constant(DateTimeSerializer.Instance)); + + if (dateTimeOffsetMember.Member.Name == nameof(DateTimeOffset.UtcDateTime)) + { + return utcField; + } + + var offsetFieldMethod = MqlFieldMethodInfo.MakeGenericMethod(typeof(BsonValue), typeof(int)); + var offsetField = Expression.Call(null, offsetFieldMethod, dtoDoc, + Expression.Constant("Offset"), Expression.Constant(Int32Serializer.Instance)); + + var localDateTime = Expression.Call(utcField, DateTimeAddMinutesMethodInfo, + Expression.Convert(offsetField, typeof(double))); + + // NOTE: .LocalDateTime is deliberately translated identically to .DateTime (both use the + // value's stored Offset) — true .NET semantics need the *executing machine's* time zone, + // which isn't available in server-side aggregation. Also: the "DateTime" sub-field is + // millisecond-truncated (sub-ms ticks lost vs. client-side eval via "Ticks"), and the + // reconstructed Kind is always Utc, not Unspecified/Local — don't call .ToLocalTime() on it. + return dateTimeOffsetMember.Member.Name is nameof(DateTimeOffset.DateTime) or nameof(DateTimeOffset.LocalDateTime) + ? localDateTime + : Expression.MakeMemberAccess(localDateTime, typeof(DateTime).GetProperty(dateTimeOffsetMember.Member.Name)!); + } + + break; + // Handle method call to VectorQuery case MethodCallExpression methodCallExpression when methodCallExpression.IsVectorSearch(): @@ -605,6 +671,72 @@ void ThrowForBadOptions(string reason) } } + /// + /// Resolves a DateTimeOffset-typed sub-expression down to the document and stored element name it + /// reads from. Returns false for shapes it doesn't recognize, so the caller falls back to default + /// translation (and the original driver error). + /// + private bool TryResolveDateTimeOffsetElementAccess(Expression expression, out Expression doc, out string elementName) + { + doc = null!; + elementName = null!; + + // Unwrap Nullable.Value. + if (expression is MemberExpression { Member.Name: "Value" } valueMember + && Nullable.GetUnderlyingType(valueMember.Expression!.Type) == typeof(DateTimeOffset)) + { + expression = valueMember.Expression!; + } + + Expression source; + string propertyName; + switch (expression) + { + case MethodCallExpression methodCall + when methodCall.Method.IsEFPropertyMethod() + && methodCall.Arguments[1] is ConstantExpression { Value: string name }: + source = methodCall.Arguments[0]; + propertyName = name; + break; + + case MemberExpression { Expression: { } memberSource } memberExpression: + source = memberSource; + propertyName = memberExpression.Member.Name; + break; + + default: + return false; + } + + var entityType = _queryContext.Context.Model.FindEntityType(source.Type); + var property = entityType?.FindProperty(propertyName); + if (property == null) + { + return false; + } + + if (property.FindTypeMapping() is { Converter: not null }) + { + throw new NotSupportedException( + $"Projecting a member of '{property.DeclaringType.DisplayName()}.{property.Name}' is not supported " + + "because the property has a value converter configured. Member access on DateTimeOffset " + + "(e.g. '.DateTime', '.Year') is only supported for the default document representation."); + } + + if (property.GetBsonRepresentation() is { BsonType: not BsonType.Document } representation) + { + throw new NotSupportedException( + $"Projecting a member of '{property.DeclaringType.DisplayName()}.{property.Name}' is not supported " + + $"because the property uses a non-default BSON representation ('{representation.BsonType}'). " + + "Member access on DateTimeOffset (e.g. '.DateTime', '.Year') is only supported for the default " + + "document representation."); + } + + doc = Visit(source)!; + elementName = property.GetElementName(); + return true; + } + private static readonly MethodInfo EFPropertyMethodInfo = typeof(EF).GetMethod(nameof(EF.Property))!; @@ -661,6 +793,15 @@ protected override Expression VisitMethodCall(MethodCallExpression node) return sizeRewrite; } + // A bare/unfiltered embedded (owned) collection-navigation Count, e.g. `b.Posts.Count`, lowered by + // EF Core to Queryable.Count(Queryable.AsQueryable(EF.Property(shaper, "Posts"))). See + // TryRewriteEmbeddedCollectionNavigationCount for why this needs a null/missing-safe normalization + // the driver's own rendering of a bare Count (a server-side $size) doesn't provide. + if (TryRewriteEmbeddedCollectionNavigationCount(node, out var embeddedCountRewrite)) + { + return embeddedCountRewrite; + } + var zeroTakeRewrite = TryRewriteZeroTake(node); if (zeroTakeRewrite != null) { @@ -670,6 +811,64 @@ protected override Expression VisitMethodCall(MethodCallExpression node) return base.VisitMethodCall(node); } + /// + /// Rewrites a bare embedded (owned) collection-navigation Count/LongCount (e.g. + /// b.Posts.Count) into Enumerable.Count over a ??-normalized array read — needed + /// because the driver's bare-Count translation is a server-side $size, which throws on a + /// missing or null array (unlike a predicated Count, translated as null-tolerant $map/$sum). + /// + private bool TryRewriteEmbeddedCollectionNavigationCount(MethodCallExpression node, out Expression result) + { + result = null!; + + if (node.Method.DeclaringType != typeof(Queryable) + || node.Arguments.Count != 1 + || node.Method.Name is not (nameof(Queryable.Count) or nameof(Queryable.LongCount))) + { + return false; + } + + if (node.Arguments[0] is not MethodCallExpression + { + Method: { Name: nameof(Queryable.AsQueryable), DeclaringType: var asQueryableDeclaring } + } asQueryableCall + || asQueryableDeclaring != typeof(Queryable)) + { + return false; + } + + // Narrow to genuinely embedded (owned) collection navigations only: AsQueryable(...) also shows + // up wrapping other enumerable sources (e.g. an IGrouping in a GroupBy/Union pipeline) that are + // not a document field read at all, so rewriting unconditionally miscompiles those shapes. + if (asQueryableCall.Arguments[0] is not MethodCallExpression efPropertyCall + || !efPropertyCall.Method.IsEFPropertyMethod() + || efPropertyCall.Arguments[1] is not ConstantExpression { Value: string propertyName } + || _queryContext.Context.Model.FindEntityType(efPropertyCall.Arguments[0].Type) is not { } sourceEntityType + || sourceEntityType.FindNavigation(propertyName) is not { } navigation + || !navigation.IsEmbedded()) + { + return false; + } + + var fieldAccess = Visit(asQueryableCall.Arguments[0]); + var elementType = fieldAccess?.Type.TryGetItemType(); + if (elementType == null) + { + return false; + } + + var countMethod = (node.Method.Name == nameof(Queryable.LongCount) + ? EnumerableLongCountMethod + : EnumerableCountMethod) + .MakeGenericMethod(elementType); + + var emptyCollection = Expression.Constant(Activator.CreateInstance(fieldAccess!.Type), fieldAccess.Type); + var normalizedFieldAccess = Expression.Coalesce(fieldAccess, emptyCollection); + + result = Expression.Call(null, countMethod, normalizedFieldAccess); + return true; + } + /// /// Rewrites source.Take(0) (constant or parameterized) into source.Where(_ => false). /// The driver's AstLimitStage rejects a limit of 0 (EF-254) — a MongoDB $limit stage of 0 diff --git a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoProjectionBindingExpressionVisitor.cs b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoProjectionBindingExpressionVisitor.cs index 7443b8ca1..02b25fd22 100644 --- a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoProjectionBindingExpressionVisitor.cs +++ b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoProjectionBindingExpressionVisitor.cs @@ -451,6 +451,39 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp EnumerableMethods.Select.MakeGenericMethod(method.GetGenericArguments()), shaper, lambda); + + // A Count/LongCount over an embedded/owned collection navigation in a projection (e.g. + // `b.Posts.Count(p => p.Rank > 0)`). Kept bound to the original method-call subtree, not + // the shaper, so it push-downs to the driver's native $size/$filter — routing through the + // shaper would materialize owned Post entities, which the tracking-materializer rejects. + case nameof(Queryable.Count) when genericMethod == QueryableMethods.CountWithoutPredicate: + case nameof(Queryable.LongCount) when genericMethod == QueryableMethods.LongCountWithoutPredicate: + case nameof(Queryable.Count) when genericMethod == QueryableMethods.CountWithPredicate: + case nameof(Queryable.LongCount) when genericMethod == QueryableMethods.LongCountWithPredicate: + if (visitedSource is not CollectionShaperExpression + || methodCallExpression.Arguments[0] is not MethodCallExpression + { + Method: { Name: nameof(Queryable.AsQueryable), DeclaringType: var asQueryableDeclaring } + } asQueryableCall + || asQueryableDeclaring != typeof(Queryable)) + { + return null; + } + + // A missing/null stored array materializes to a null reference, not an empty + // collection (same gap as `.Select(e => e.Foo)`), so Count/LongCount must tolerate a + // null source ($size/$filter both reject null) — guard on the raw field access, not + // the AsQueryable-wrapped source. + var countSource = asQueryableCall.Arguments[0]; + Expression nullSafeCount = Expression.Condition( + Expression.Equal(countSource, Expression.Constant(null, countSource.Type)), + Expression.Default(methodCallExpression.Type), + methodCallExpression); + + var countProjectionMember = GetCurrentProjectionMember(); + _projectionMapping[countProjectionMember] = nullSafeCount; + + return new ProjectionBindingExpression(_queryExpression, countProjectionMember, methodCallExpression.Type); } } diff --git a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitor.cs b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitor.cs index 5b252ae50..cdf977e83 100644 --- a/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitor.cs @@ -71,6 +71,124 @@ TranslationErrorDetails is null private static readonly Type[] AllowedQueryableExtensions = [typeof(Queryable), typeof(MongoQueryableExtensions), typeof(Driver.Linq.MongoQueryable)]; + private static readonly HashSet OrderingMethodNames = + [ + nameof(Queryable.OrderBy), nameof(Queryable.OrderByDescending), + nameof(Queryable.ThenBy), nameof(Queryable.ThenByDescending) + ]; + + /// + /// Elides a ThenBy/ThenByDescending ordering whose key selector matches an earlier ordering + /// already established in the same OrderBy/ThenBy chain (see + /// for what "matches" means here). Once a key fully determines the sort order, re-ordering by it again - + /// in either direction - is a no-op; left untouched, the driver's LINQ provider renders both orderings + /// into a single $sort stage and MongoDB rejects the resulting document for its duplicate field + /// name (EF-253 / CSHARP-5690). Relational EF providers already drop this kind of redundant ordering from + /// the generated SQL, so this mirrors their behavior rather than merely working around the driver + /// limitation. Orderings that supply an explicit are left untouched (in either + /// role - as a candidate for elision, or as prior state a later ordering could match) since a custom + /// comparer can make an otherwise-identical key selector not actually redundant. + /// + internal static Expression ElideRedundantOrderings(Expression expression) + { + if (expression is not MethodCallExpression { Arguments.Count: > 0 } methodCall + || !AllowedQueryableExtensions.Contains(methodCall.Method.DeclaringType)) + { + return expression; + } + + if (!OrderingMethodNames.Contains(methodCall.Method.Name)) + { + var visitedSource = ElideRedundantOrderings(methodCall.Arguments[0]); + if (ReferenceEquals(visitedSource, methodCall.Arguments[0])) + { + return methodCall; + } + + var updatedArgs = methodCall.Arguments.ToArray(); + updatedArgs[0] = visitedSource; + return methodCall.Update(methodCall.Object, updatedArgs); + } + + // Collect the contiguous ordering chain, outer (last-applied) to inner (first-applied), then + // reverse it so it can be replayed in chronological order. + var chain = new List(); + var node = methodCall; + while (AllowedQueryableExtensions.Contains(node.Method.DeclaringType) && OrderingMethodNames.Contains(node.Method.Name)) + { + chain.Add(node); + if (node.Arguments[0] is not MethodCallExpression next) + { + break; + } + + node = next; + } + + chain.Reverse(); + + Expression current = ElideRedundantOrderings(chain[0].Arguments[0]); + var seenKeys = new List(); + foreach (var call in chain) + { + // OrderBy/OrderByDescending establish a brand new ordering - even mid-chain (e.g. + // .OrderBy(k).OrderByDescending(k)) - superseding whatever came before, so only ThenBy/ + // ThenByDescending calls continue an existing chain for duplicate-detection purposes. + if (call.Method.Name is nameof(Queryable.OrderBy) or nameof(Queryable.OrderByDescending)) + { + seenKeys.Clear(); + } + + var keySelector = call.Arguments.Count == 2 ? call.Arguments[1].UnwrapLambdaFromQuote() : null; + var isDuplicate = keySelector is not null && seenKeys.Any(seen => KeySelectorsMatch(seen, keySelector)); + + if (keySelector is not null) + { + seenKeys.Add(keySelector); + } + + if (isDuplicate) + { + continue; + } + + if (ReferenceEquals(current, call.Arguments[0])) + { + current = call; + continue; + } + + var newArgs = call.Arguments.ToArray(); + newArgs[0] = current; + current = call.Update(call.Object, newArgs); + } + + return current; + } + + /// + /// Whether two ordering key selectors are both a *direct*, single-hop access to the same member of the + /// lambda parameter (e.g. x => x.CustomerId), modulo the identity of their (distinct) lambda + /// parameters. Deliberately narrow to a single hop: the driver's LINQ provider renders a direct property + /// access as-is against its raw field path, so two orderings on the *same* directly-accessed member are + /// what collide into a single $sort document with a duplicate field name (EF-253 / CSHARP-5690). + /// Anything requiring computation - a further member hop off that property (x.Name.Length), a + /// method call (Math.Truncate(x.Amount)), etc. - instead gets materialized by EF into its own + /// uniquely-named projected field even when repeated, so it never collides and must not be elided here. + /// + internal static bool KeySelectorsMatch(LambdaExpression a, LambdaExpression b) + { + if (a.Parameters.Count != 1 || b.Parameters.Count != 1) + { + return false; + } + + var membersA = a.Parameters[0].MatchMemberAccess(a.Body); + var membersB = b.Parameters[0].MatchMemberAccess(b.Body); + + return membersA is [var memberA] && membersB is [var memberB] && memberA == memberB; + } + /// /// Visit the to capture the cardinality and final expression /// when found on a method. @@ -80,7 +198,7 @@ TranslationErrorDetails is null /// otherwise . protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression) { - _finalExpression ??= methodCallExpression; + _finalExpression ??= ElideRedundantOrderings(methodCallExpression); var method = methodCallExpression.Method; #if !EF8 diff --git a/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/DateTimeOffsetMemberProjectionTests.cs b/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/DateTimeOffsetMemberProjectionTests.cs new file mode 100644 index 000000000..07b4f8907 --- /dev/null +++ b/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/DateTimeOffsetMemberProjectionTests.cs @@ -0,0 +1,225 @@ +/* Copyright 2023-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.EntityFrameworkCore.Diagnostics; + +namespace MongoDB.EntityFrameworkCore.FunctionalTests.Query; + +[XUnitCollection("QueryTests")] +public class DateTimeOffsetMemberProjectionTests(TemporaryDatabaseFixture database) + : IClassFixture +{ + // A non-zero, non-round offset so UTC-vs-local math is actually exercised by the test. + private static readonly DateTimeOffset TestValue = + new(2024, 3, 15, 13, 45, 30, 250, TimeSpan.FromHours(-5)); + + private static readonly DateTimeOffsetEntity SeedEntity = new() + { + Id = ObjectId.GenerateNewId(), + DateTimeOffset = TestValue, + OptionalDateTimeOffset = TestValue + }; + + private IMongoCollection CreateSeededCollection([System.Runtime.CompilerServices.CallerMemberName] string? name = null) + { + var collection = database.CreateCollection(name); + collection.InsertOne(SeedEntity); + return collection; + } + + [Fact] + public void Select_DateTimeOffset_DateTime_component() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities.Select(e => e.DateTimeOffset.DateTime).Single(); + + Assert.Equal(TestValue.DateTime, result); + } + + [Fact] + public void Select_DateTimeOffset_Date_component() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities.Select(e => e.DateTimeOffset.Date).Single(); + + Assert.Equal(TestValue.Date, result); + } + + [Fact] + public void Select_DateTimeOffset_Year_component() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities.Select(e => e.DateTimeOffset.Year).Single(); + + Assert.Equal(TestValue.Year, result); + } + + [Fact] + public void Where_DateTimeOffset_DateTime_component() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities.Where(e => e.DateTimeOffset.DateTime == TestValue.DateTime).Single(); + + Assert.Equal(SeedEntity.Id, result.Id); + } + + [Fact] + public void Select_optional_DateTimeOffset_Value_DateTime_Date_component() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities + .Select(e => e.OptionalDateTimeOffset == null ? (DateTime?)null : e.OptionalDateTimeOffset.Value.DateTime.Date) + .Single(); + + Assert.Equal(TestValue.Date, result); + } + + [Fact] + public void Select_optional_DateTimeOffset_null_short_circuits() + { + var entityWithNull = new DateTimeOffsetEntity + { + Id = ObjectId.GenerateNewId(), + DateTimeOffset = TestValue, + OptionalDateTimeOffset = null + }; + var collection = database.CreateCollection(); + collection.InsertOne(entityWithNull); + using var db = SingleEntityDbContext.Create(collection); + + var result = db.Entities + .Select(e => e.OptionalDateTimeOffset == null ? (DateTime?)null : e.OptionalDateTimeOffset.Value.DateTime.Date) + .Single(); + + Assert.Null(result); + } + + [Fact] + public void Select_DateTimeOffset_DateTime_component_with_string_representation_throws() + { + var collection = database.CreateCollection(); + collection.InsertOne(SeedEntity); + using var db = SingleEntityDbContext.Create(collection, modelBuilderAction: mb => + mb.Entity().Property(e => e.DateTimeOffset).HasBsonRepresentation(BsonType.String)); + + Assert.Throws(() => db.Entities.Select(e => e.DateTimeOffset.DateTime).Single()); + } + + [Fact] + public void Select_DateTimeOffset_DateTime_component_with_value_converter_throws() + { + var collection = database.CreateCollection(); + collection.InsertOne(SeedEntity); + using var db = SingleEntityDbContext.Create(collection, modelBuilderAction: mb => + mb.Entity().Property(e => e.DateTimeOffset) + .HasConversion(v => v.Ticks, v => new DateTimeOffset(v, TimeSpan.Zero))); + + Assert.Throws(() => db.Entities.Select(e => e.DateTimeOffset.DateTime).Single()); + } + + [Fact] + public void Select_static_DateTimeOffset_UtcNow_Year_does_not_throw_NullReferenceException() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + // DateTimeOffset.UtcNow.Year reaches TryResolveDateTimeOffsetElementAccess as a static MemberExpression + // with a null Expression. A prior version null-forgave that and threw NullReferenceException; the fix + // falls through to `return false` instead, letting the query translate/evaluate normally. The + // regression guarded against is specifically the NullReferenceException, not any particular exception. + var exception = Record.Exception(() => db.Entities.Select(e => DateTimeOffset.UtcNow.Year).Single()); + + Assert.False(exception is NullReferenceException, + $"Expected no NullReferenceException; got {exception?.GetType().FullName ?? "no exception"}."); + } + + [Fact] + public void Select_DateTimeOffset_DateTime_component_renders_as_server_side_mql() + { + var (loggerFactory, spyLogger) = SpyLoggerProvider.Create(); + var collection = CreateSeededCollection(); + using var db = SingleEntityDbContext.Create(collection, loggerFactory, + optionsBuilderAction: b => b.EnableSensitiveDataLogging()); + + var result = db.Entities.Select(e => e.DateTimeOffset.DateTime).Single(); + + Assert.Equal(TestValue.DateTime, result); + + // Confirms the rewrite actually renders as a server-side aggregation expression (reading the + // stored DateTime/Offset sub-fields and reconstructing via $dateAdd) rather than silently + // falling back to client-side evaluation and coincidentally producing the right value. + var message = spyLogger.GetLogMessageByEventId(MongoEventId.ExecutedMqlQuery); + Assert.Contains("Executed MQL query", message); + Assert.Contains("\"$dateAdd\"", message); + Assert.Contains("\"startDate\" : \"$DateTimeOffset.DateTime\"", message); + Assert.Contains("\"unit\" : \"minute\"", message); + Assert.Contains("\"amount\" : \"$DateTimeOffset.Offset\"", message); + } + + [Fact] + public void Select_DateTimeOffset_remaining_components() + { + using var db = SingleEntityDbContext.Create(CreateSeededCollection()); + + var result = db.Entities.Select(e => new + { + e.DateTimeOffset.Month, + e.DateTimeOffset.Day, + e.DateTimeOffset.Hour, + e.DateTimeOffset.Minute, + e.DateTimeOffset.Second, + e.DateTimeOffset.Millisecond, + e.DateTimeOffset.DayOfWeek, + e.DateTimeOffset.DayOfYear, + e.DateTimeOffset.TimeOfDay, + e.DateTimeOffset.LocalDateTime, + e.DateTimeOffset.UtcDateTime + }).Single(); + + // Scalar member components work correctly + Assert.Equal(TestValue.Month, result.Month); + Assert.Equal(TestValue.Day, result.Day); + Assert.Equal(TestValue.Hour, result.Hour); + Assert.Equal(TestValue.Minute, result.Minute); + Assert.Equal(TestValue.Second, result.Second); + Assert.Equal(TestValue.Millisecond, result.Millisecond); + Assert.Equal(TestValue.DayOfWeek, result.DayOfWeek); + Assert.Equal(TestValue.DayOfYear, result.DayOfYear); + Assert.Equal(TestValue.TimeOfDay, result.TimeOfDay); + + Assert.Equal(TestValue.UtcDateTime, result.UtcDateTime); + + // LocalDateTime is translated identically to DateTime (using the value's own stored offset), + // not the query-executing machine's system time zone — see the doc comment on that translation + // in MongoEFToLinqTranslatingExpressionVisitor.cs. Assert against TestValue.DateTime (not + // TestValue.LocalDateTime, which would be flaky/machine-timezone-dependent). + Assert.Equal(TestValue.DateTime, result.LocalDateTime); + } +} + +public class DateTimeOffsetEntity +{ + public ObjectId Id { get; set; } + public DateTimeOffset DateTimeOffset { get; set; } + public DateTimeOffset? OptionalDateTimeOffset { get; set; } +} diff --git a/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/OwnedEntityTests.cs b/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/OwnedEntityTests.cs index 4d6e6b0b9..79e98e804 100644 --- a/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/OwnedEntityTests.cs +++ b/tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/OwnedEntityTests.cs @@ -15,7 +15,9 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.Extensions.Logging; using MongoDB.Bson; +using MongoDB.EntityFrameworkCore.Diagnostics; using MongoDB.EntityFrameworkCore.Extensions; namespace MongoDB.EntityFrameworkCore.FunctionalTests.Query; @@ -1573,6 +1575,174 @@ public void OwnedEntity_can_go_multiple_levels_deep_querying_reference() } } + [Fact] + public void OwnedEntity_filtered_count_in_projection_translates() + { + var collection = database.CreateCollection(); + + { + using var db = SingleEntityDbContext.Create(collection); + db.Entities.Add(new CountBlog + { + _id = "1", + Title = "Blog1", + Posts = [new CountPost { Rank = 1 }, new CountPost { Rank = 0 }, new CountPost { Rank = 2 }] + }); + db.SaveChanges(); + } + + { + // Assert the count is evaluated server-side (a $map/$sum over the array, equivalent to + // filter+size) rather than by materializing the owned CountPost entities and counting client-side. + var (loggerFactory, spyLogger) = SpyLoggerProvider.Create(); + using var db = SingleEntityDbContext.Create(collection, loggerFactory, + optionsBuilderAction: o => o.EnableSensitiveDataLogging()); + var result = Assert.Single(db.Entities.Select(b => new { b.Title, N = b.Posts.Count(p => p.Rank > 0) })); + + Assert.Equal("Blog1", result.Title); + Assert.Equal(2, result.N); + + var message = spyLogger.GetLogMessageByEventId(MongoEventId.ExecutedMqlQuery); + Assert.Contains("$map", message); + Assert.Contains("$sum", message); + } + } + + [Fact] + public void OwnedEntity_unfiltered_count_in_projection_translates() + { + var collection = database.CreateCollection(); + + { + using var db = SingleEntityDbContext.Create(collection); + db.Entities.Add(new CountBlog + { + _id = "1", + Title = "Blog1", + Posts = [new CountPost { Rank = 1 }, new CountPost { Rank = 0 }] + }); + db.SaveChanges(); + } + + { + using var db = SingleEntityDbContext.Create(collection); + var result = Assert.Single(db.Entities.Select(b => new { b.Title, N = b.Posts.Count() })); + + Assert.Equal("Blog1", result.Title); + Assert.Equal(2, result.N); + } + } + + [Fact] + public void OwnedEntity_collection_bare_count_projection_returns_element_count() + { + var collection = database.CreateCollection(); + + { + using var db = SingleEntityDbContext.Create(collection); + db.Entities.AddRange( + new A { _id = "1", children = [new B { name = "child1" }, new B { name = "child2" }] }, + new A { _id = "2", children = [] }, + new A { _id = "3", children = null! }); + db.SaveChanges(); + } + + // Deliberately a default *tracking* query (unlike EF-357's own tests, which required + // AsNoTracking): the bare Count must stay a server-side scalar, not a materialized owned-entity + // shaper, or EF Core's tracking-materializer rejects it ("owned entity without a corresponding + // owner"). + var (loggerFactory, spyLogger) = SpyLoggerProvider.Create(); + using var db2 = SingleEntityDbContext.Create(collection, loggerFactory, + optionsBuilderAction: o => o.EnableSensitiveDataLogging()); + + var counts = db2.Entities.OrderBy(e => e._id).Select(e => e.children.Count).ToList(); + Assert.Equal([2, 0, 0], counts); + + // A null stored array must report 0, not throw: the driver renders a bare Count as a server-side + // $size, which rejects a null array, so this needs an $ifNull normalization first. + var message = spyLogger.GetLogMessageByEventId(MongoEventId.ExecutedMqlQuery); + Assert.Contains("$ifNull", message); + Assert.Contains("$size", message); + + var longCounts = db2.Entities.OrderBy(e => e._id).Select(e => e.children.LongCount()).ToList(); + Assert.Equal([2L, 0L, 0L], longCounts); + + var filteredLongCounts = db2.Entities.OrderBy(e => e._id) + .Select(e => e.children.LongCount(c => c.name == "child1")) + .ToList(); + Assert.Equal([1L, 0L, 0L], filteredLongCounts); + } + + [Fact] + public void OwnedEntity_collection_bare_count_projection_over_missing_element_returns_zero() + { + // A document where the array element is OMITTED entirely (not stored as BSON null) is a distinct + // case from an explicit null: in an aggregation expression a missing field is not equal to BSON + // null, so a plain null-equality guard does not catch it and $size still throws on "missing". Only + // $ifNull (which normalizes missing the same as null) handles both. + var collection = database.CreateCollection(); + database.GetCollection(collection.CollectionNamespace).InsertOne(new BsonDocument("_id", "1")); + + using var db = SingleEntityDbContext.Create(collection); + var counts = db.Entities.Select(e => e.children.Count).ToList(); + + Assert.Equal([0], counts); + } + + [Fact] + public void OwnedEntity_collection_count_projection_wrapped_in_arithmetic_returns_element_count() + { + var collection = database.CreateCollection(); + + { + using var db = SingleEntityDbContext.Create(collection); + db.Entities.AddRange( + new A { _id = "1", children = [new B { name = "child1" }, new B { name = "child2" }] }, + new A { _id = "2", children = null! }); + db.SaveChanges(); + } + + using var db2 = SingleEntityDbContext.Create(collection); + var doubled = db2.Entities.OrderBy(e => e._id) + .Select(e => new { e._id, N = e.children.Count * 2 }) + .ToList(); + + Assert.Equal([4, 0], doubled.Select(r => r.N).ToArray()); + } + + [Fact] + public void OwnedEntity_collection_filtered_count_projection_with_null_children_returns_zero() + { + var collection = database.CreateCollection(); + + { + using var db = SingleEntityDbContext.Create(collection); + db.Entities.AddRange( + new A { _id = "1", children = [new B { name = "child1" }, new B { name = "child2" }] }, + new A { _id = "2", children = null! }); + db.SaveChanges(); + } + + using var db2 = SingleEntityDbContext.Create(collection); + var counts = db2.Entities.OrderBy(e => e._id) + .Select(e => e.children.Count(c => c.name == "child1")) + .ToList(); + + Assert.Equal([1, 0], counts); + } + + record CountBlog + { + public string _id { get; set; } + public string Title { get; set; } + public List Posts { get; set; } + } + + record CountPost + { + public int Rank { get; set; } + } + record A { public string _id { get; set; } diff --git a/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Mapping/BuiltInDataTypesMongoTest.cs b/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Mapping/BuiltInDataTypesMongoTest.cs index eeacfb9d8..0e7111e66 100644 --- a/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Mapping/BuiltInDataTypesMongoTest.cs +++ b/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Mapping/BuiltInDataTypesMongoTest.cs @@ -19,7 +19,6 @@ using Microsoft.EntityFrameworkCore.TestUtilities; using MongoDB.Driver; using MongoDB.Driver.Core.Misc; -using MongoDB.Driver.Linq; using MongoDB.EntityFrameworkCore.Diagnostics; using MongoDB.EntityFrameworkCore.FunctionalTests.Utilities; using Xunit.Sdk; @@ -53,12 +52,8 @@ public override async Task Object_to_string_conversion() "Actual: \"97\"", (await Assert.ThrowsAsync(() => base.Object_to_string_conversion())).Message); - // Fails: Projecting DateTimeOffset members EF-218 - public override async Task Optional_datetime_reading_null_from_database() - => Assert.Contains( - "d.DateTimeOffset.Value.DateTime", - (await Assert.ThrowsAsync(() => base.Optional_datetime_reading_null_from_database())) - .Message); + public override Task Optional_datetime_reading_null_from_database() + => base.Optional_datetime_reading_null_from_database(); #else public override void Can_insert_and_read_back_with_string_key() => base.Can_insert_and_read_back_with_string_key(); @@ -78,11 +73,8 @@ public override void Object_to_string_conversion() "Unsupported conversion from object to string in $convert with no onError value.", Assert.Throws(() => base.Object_to_string_conversion()).Message); - // Fails: Projecting DateTimeOffset members EF-218 public override void Optional_datetime_reading_null_from_database() - => Assert.Contains( - "d.DateTimeOffset.Value.DateTime.Date", - Assert.Throws(() => base.Optional_datetime_reading_null_from_database()).Message); + => base.Optional_datetime_reading_null_from_database(); #endif private static void AssertTranslationFailed(Action query) diff --git a/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/NorthwindMiscellaneousQueryMongoTest.cs b/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/NorthwindMiscellaneousQueryMongoTest.cs index 0b3e88dcd..b05f33f2c 100644 --- a/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/NorthwindMiscellaneousQueryMongoTest.cs +++ b/tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/NorthwindMiscellaneousQueryMongoTest.cs @@ -3456,15 +3456,11 @@ public override async Task Compare_two_collection_navigations_with_different_pro public override async Task OrderBy_ThenBy_same_column_different_direction(bool async) { - // Fails: Multiple ordering issue EF-253 - Assert.Contains( - "Duplicate element name '_id'.", - (await Assert.ThrowsAsync(() => base.OrderBy_ThenBy_same_column_different_direction(async))) - .Message); + await base.OrderBy_ThenBy_same_column_different_direction(async); AssertMql( """ - Customers. + Customers.{ "$match" : { "_id" : { "$regularExpression" : { "pattern" : "^A", "options" : "s" } } } }, { "$sort" : { "_id" : 1 } }, { "$project" : { "_v" : "$_id", "_id" : 0 } } """); } diff --git a/tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitorTests.cs b/tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitorTests.cs new file mode 100644 index 000000000..dae90210b --- /dev/null +++ b/tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitorTests.cs @@ -0,0 +1,203 @@ +/* Copyright 2023-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Linq; +using System.Linq.Expressions; +using MongoDB.EntityFrameworkCore.Query.Visitors; + +namespace MongoDB.EntityFrameworkCore.UnitTests.Query.Visitors; + +public class MongoQueryableMethodTranslatingExpressionVisitorTests +{ + private class Customer + { + public string Id { get; set; } = ""; + public string Name { get; set; } = ""; + public string City { get; set; } = ""; + } + + private static readonly IQueryable Source = new Customer[0].AsQueryable(); + + // Reads the OrderBy/ThenBy/... spine of an expression tree, outermost-in, and returns it in + // chronological (source-to-outermost) order as (method name, argument count) pairs. Argument + // count distinguishes the 2-arg (key selector only) and 3-arg (key selector + IComparer) overloads. + private static List<(string Name, int ArgCount)> GetOrderingChainShape(Expression expression) + { + var shape = new List<(string, int)>(); + while (expression is MethodCallExpression { Arguments.Count: > 0 } call + && call.Method.DeclaringType == typeof(Queryable)) + { + shape.Insert(0, (call.Method.Name, call.Arguments.Count)); + expression = call.Arguments[0]; + } + + return shape; + } + + [Fact] + public void Elides_ThenByDescending_duplicating_the_OrderBy_key() + { + var query = Source.OrderBy(c => c.Name).ThenByDescending(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal([(nameof(Queryable.OrderBy), 2)], GetOrderingChainShape(result)); + } + + [Fact] + public void Elides_ThenBy_duplicating_the_OrderBy_key_in_the_same_direction() + { + var query = Source.OrderBy(c => c.Name).ThenBy(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal([(nameof(Queryable.OrderBy), 2)], GetOrderingChainShape(result)); + } + + [Fact] + public void Keeps_two_independent_OrderBy_calls_on_the_same_key() + { + // .OrderBy().OrderByDescending() (as opposed to .OrderBy().ThenByDescending()) is two + // independent orderings - the second entirely supersedes the first - so both must be kept. + var query = Source.OrderBy(c => c.Name).OrderByDescending(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 2), (nameof(Queryable.OrderByDescending), 2)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Mid_chain_OrderBy_resets_duplicate_tracking() + { + // The second OrderByDescending starts a fresh ordering, so the ThenBy(Name) directly under it + // is a duplicate of *that* ordering (and is elided), while the first ThenBy(Name) - a duplicate + // of the original OrderBy - is also elided; the two survivors are the resetting pair. + var query = Source.OrderBy(c => c.Name).ThenBy(c => c.Name) + .OrderByDescending(c => c.Name).ThenBy(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 2), (nameof(Queryable.OrderByDescending), 2)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Elides_only_the_redundant_ordering_in_a_longer_chain() + { + // ThenBy(Name) duplicates OrderBy(Name) and is elided; ThenByDescending(City) is a genuinely + // new key and survives; the final ThenBy(City) duplicates it and is elided. + var query = Source.OrderBy(c => c.Name).ThenBy(c => c.Name) + .ThenByDescending(c => c.City).ThenBy(c => c.City); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 2), (nameof(Queryable.ThenByDescending), 2)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Does_not_elide_ordering_with_an_explicit_comparer() + { + var query = Source.OrderBy(c => c.Name).ThenByDescending(c => c.Name, System.StringComparer.Ordinal); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 2), (nameof(Queryable.ThenByDescending), 3)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Does_not_treat_an_earlier_comparer_ordering_as_a_match_for_a_later_plain_ordering() + { + var query = Source.OrderBy(c => c.Name, System.StringComparer.Ordinal).ThenByDescending(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 3), (nameof(Queryable.ThenByDescending), 2)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Does_not_elide_a_repeated_computed_key_selector() + { + // c.Name.Length is a two-hop computed key (Name, then Length) - EF materializes each such + // ordering into its own uniquely-named projected field, so repeating it never collides and + // must not be elided, unlike a direct single-hop property access. + var query = Source.OrderBy(c => c.Name.Length).ThenBy(c => c.Name.Length); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Equal( + [(nameof(Queryable.OrderBy), 2), (nameof(Queryable.ThenBy), 2)], + GetOrderingChainShape(result)); + } + + [Fact] + public void Leaves_a_non_ordering_expression_unchanged() + { + Expression> predicate = c => c.Name == "A"; + var query = Source.Where(predicate); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + Assert.Same(query.Expression, result); + } + + [Fact] + public void Elides_through_an_intervening_Where_below_the_ordering_chain() + { + var query = Source.Where(c => c.City == "X").OrderBy(c => c.Name).ThenByDescending(c => c.Name); + + var result = MongoQueryableMethodTranslatingExpressionVisitor.ElideRedundantOrderings(query.Expression); + + var call = Assert.IsAssignableFrom(result); + Assert.Equal(nameof(Queryable.OrderBy), call.Method.Name); + var whereCall = Assert.IsAssignableFrom(call.Arguments[0]); + Assert.Equal(nameof(Queryable.Where), whereCall.Method.Name); + } + + [Fact] + public void KeySelectorsMatch_returns_true_for_the_same_property_with_differently_named_parameters() + { + Expression> a = c => c.Name; + Expression> b = x => x.Name; + + Assert.True(MongoQueryableMethodTranslatingExpressionVisitor.KeySelectorsMatch(a, b)); + } + + [Fact] + public void KeySelectorsMatch_returns_false_for_different_properties() + { + Expression> a = c => c.Name; + Expression> b = c => c.City; + + Assert.False(MongoQueryableMethodTranslatingExpressionVisitor.KeySelectorsMatch(a, b)); + } + + [Fact] + public void KeySelectorsMatch_returns_false_for_a_computed_multi_hop_key() + { + Expression> a = c => c.Name.Length; + Expression> b = c => c.Name.Length; + + Assert.False(MongoQueryableMethodTranslatingExpressionVisitor.KeySelectorsMatch(a, b)); + } +} From 1d6089365a72929e6b9441056e0713227b0a5082 Mon Sep 17 00:00:00 2001 From: Arthur Vickers Date: Mon, 24 Aug 2026 15:59:23 +0100 Subject: [PATCH 002/186] EF-322: Native LINQ overhaul Squashes 3 commits: - EF-322: Native LINQ Overhaul (squashed: native overhaul + rebase fallout fix + $limit:0 test update) - Delete docs/failing-spec-tests.md - EF-322: drop machine-specific git pull allow-rule from .claude/settings.json --- .claude/settings.json | 3 +- AGENTS.md | 1 + BREAKING-CHANGES.md | 157 + .../HeadlineBenchmarks.cs | 113 +- .../Program.cs | 82 + .../results/perf-baseline.md | 92 + docs/failing-spec-tests.md | 437 -- .../EnumerableMethods.cs | 90 + .../ExpressionExtensionMethods.cs | 16 + .../MongoDbContextOptionsBuilder.cs | 15 + .../Infrastructure/MongoOptionsExtension.cs | 33 +- .../Infrastructure/MongoQueryMode.cs | 29 + .../Query/AGENTS.md | 114 +- .../ArrayAliasProjectionExpression.cs | 90 + .../Expressions/IArrayProjectionExpression.cs | 43 + .../Query/Expressions/LookupExpression.cs | 49 +- .../Expressions/MongoBinaryExpression.cs | 72 + .../Query/Expressions/MongoCardinality.cs | 100 + .../Expressions/MongoConstantExpression.cs | 53 + .../Expressions/MongoConvertExpression.cs | 85 + .../Expressions/MongoElemMatchExpression.cs | 79 + .../Expressions/MongoElementRefExpression.cs | 36 + .../Query/Expressions/MongoExpression.cs | 69 + .../Query/Expressions/MongoFieldExpression.cs | 48 + .../MongoFilteredSizeExpression.cs | 79 + .../Query/Expressions/MongoGrouping.cs | 41 + .../Query/Expressions/MongoInExpression.cs | 51 + .../Query/Expressions/MongoOrdering.cs | 24 + .../Expressions/MongoParameterExpression.cs | 55 + .../Query/Expressions/MongoProjection.cs | 25 + .../MongoQueryExpression.Lookup.cs | 101 +- .../Query/Expressions/MongoQueryExpression.cs | 19 +- .../Query/Expressions/MongoRegexExpression.cs | 64 + .../Expressions/MongoSelectDefinition.cs | 718 +++ .../Query/Expressions/MongoSelectOp.cs | 39 + .../Query/Expressions/MongoSetOperation.cs | 65 + .../Query/Expressions/MongoSizeExpression.cs | 80 + .../Query/Expressions/MongoUnaryExpression.cs | 63 + .../Query/Expressions/MongoUnwindSource.cs | 103 + .../Query/Expressions/MongoVectorSearch.cs | 68 + .../ObjectArrayProjectionExpression.cs | 7 +- .../MongoQueryCompilationContextFactory.cs | 30 +- .../Query/MongoExecutableQuery.cs | 20 + .../Query/MongoQueryCompilationContext.cs | 18 +- .../NativeTranslation/BsonValueSerializer.cs | 84 + .../MongoAggregationExpressionRenderer.cs | 182 + .../MongoEntityMaterializerSerializer.cs | 51 + .../MongoExpressionNegator.cs | 203 + .../MongoExpressionTranslator.Members.cs | 331 ++ .../MongoExpressionTranslator.MethodCalls.cs | 466 ++ .../MongoExpressionTranslator.cs | 1055 +++++ .../MongoFieldPrefixRewriter.cs | 60 + .../MongoNativeBuildContext.cs | 58 + .../NativeTranslation/MongoPipelineFactory.cs | 724 +++ .../MongoQueryLanguageRenderer.cs | 561 +++ .../NativeTranslation/MongoSelectLowerer.cs | 444 ++ ...ongoStreamingEntityMaterializerRewriter.cs | 1352 ++++++ .../NativeTranslation/MongoValueRenderer.cs | 83 + .../NativeCardinalityBinder.cs | 201 + .../NativeCorrelationMatcher.cs | 173 + .../NativeTranslation/NativeGroupByBinder.cs | 396 ++ .../NativeProjectionBinder.cs | 1479 ++++++ .../NativeTranslation/NativeQueryParameter.cs | 57 + .../NativeSelectManyBinder.cs | 827 ++++ .../NativeTranslation/NativeSlotPopulator.cs | 416 ++ .../NativeTranslationNotSupportedException.cs | 39 + .../NativeVectorSearchBinder.cs | 105 + .../NativeTranslation/PlaceholderTable.cs | 121 + .../Stages/MongoAddFieldsStage.cs | 46 + .../Stages/MongoCountStage.cs | 29 + .../Stages/MongoGroupAccumulatorStage.cs | 47 + .../Stages/MongoGroupStage.cs | 25 + .../Stages/MongoLimitStage.cs | 38 + .../Stages/MongoLookupStage.cs | 38 + .../Stages/MongoMatchStage.cs | 38 + .../Stages/MongoPipelineStage.cs | 25 + .../Stages/MongoProjectStage.cs | 39 + .../Stages/MongoReplaceRootStage.cs | 51 + .../Stages/MongoSetDifferenceStage.cs | 42 + .../Stages/MongoSkipStage.cs | 38 + .../Stages/MongoSortStage.cs | 39 + .../Stages/MongoUnionWithStage.cs | 37 + .../Stages/MongoUnsetStage.cs | 45 + .../Stages/MongoUnwindFieldStage.cs | 40 + .../Stages/MongoUnwindStage.cs | 55 + .../Stages/MongoVectorSearchScoreStage.cs | 43 + .../Stages/MongoVectorSearchStage.cs | 41 + .../NativeTranslation/StreamingEligibility.cs | 103 + .../Query/QueryingEnumerable.cs | 47 +- .../Query/VectorSearchStageBuilder.cs | 234 + .../BsonDocumentInjectingExpressionVisitor.cs | 13 +- ...nqTranslatingExpressionVisitor.LeftJoin.cs | 477 +- ...ngoEFToLinqTranslatingExpressionVisitor.cs | 229 +- ...jectionBindingRemovingExpressionVisitor.cs | 55 +- ...ojectionBindingExpressionVisitor.Lookup.cs | 23 +- ...MongoProjectionBindingExpressionVisitor.cs | 574 ++- ...jectionBindingRemovingExpressionVisitor.cs | 167 +- ...yableMethodTranslatingExpressionVisitor.cs | 1332 +++++- ...goShapedQueryCompilingExpressionVisitor.cs | 849 +++- .../Storage/BsonBinding.cs | 38 +- .../Storage/MongoClientWrapper.cs | 44 +- .../StoredDataStillReadableTests.cs | 16 +- .../Mapping/DiscriminatorTests.cs | 14 + .../Query/CrossCollectionIncludeTests.cs | 21 + .../Query/CrossCollectionRelationshipTests.cs | 96 +- .../Ef362OwnedHopArrayProjectionTests.cs | 337 ++ .../Query/Ef372DeepReferenceIncludeTests.cs | 1221 +++++ .../Query/Ef373InterleavedPagingTests.cs | 389 ++ .../Ef379RootNavigationMisroutingTests.cs | 948 ++++ .../Query/FirstSingleTests.cs | 16 +- .../Query/NativeArrayProjectionTests.cs | 1949 ++++++++ .../Query/NativeBareProjectionTests.cs | 817 ++++ .../Query/NativeCardinalityTests.cs | 563 +++ .../Query/NativeCastTests.cs | 2498 +++++++++++ .../Query/NativeCombinationQueryTests.cs | 183 + .../Query/NativeCompositionSeamAuditTests.cs | 290 ++ .../NativeComputedBareProjectionTests.cs | 1329 ++++++ .../Query/NativeComputedProjectionTests.cs | 382 ++ .../Query/NativeComputedSortTests.cs | 844 ++++ .../Query/NativeDistinctTests.cs | 501 +++ .../Query/NativeEfPropertyLeafTests.cs | 343 ++ .../Query/NativeExprComparisonTests.cs | 420 ++ .../Query/NativeGateRoutingTests.cs | 597 +++ .../Query/NativeGroupByTests.cs | 853 ++++ .../Query/NativeJoinInnerDeclineTests.cs | 239 + .../NativeMaterializerNullabilityTests.cs | 300 ++ .../Query/NativeMaterializerOnePassTests.cs | 474 ++ .../Query/NativeNullableMemberTests.cs | 645 +++ .../Query/NativeOfTypeTests.cs | 461 ++ .../Query/NativeOwnedCollectionAllTests.cs | 801 ++++ .../Query/NativeOwnedCollectionCountTests.cs | 1449 ++++++ ...NativeOwnedCollectionFilteredCountTests.cs | 1384 ++++++ .../NativeOwnedCollectionPredicateTests.cs | 732 +++ .../NativeOwnedCollectionWholeEntityTests.cs | 314 ++ .../NativeOwnedReferenceWholeEntityTests.cs | 697 +++ .../Query/NativeOwnedSubPropertyTests.cs | 330 ++ .../Query/NativePipelineExecutionTests.cs | 87 + .../Query/NativeReferenceIncludeTests.cs | 1437 ++++++ .../Query/NativeSelectManyTests.cs | 3970 +++++++++++++++++ .../Query/NativeSetOpsTests.cs | 1751 ++++++++ .../NativeTransactionAndCancellationTests.cs | 288 ++ .../Query/NativeVectorSearchTests.cs | 689 +++ .../Query/OwnedEntityTests.cs | 29 +- .../ProjectedCollectionNormalizationTests.cs | 382 ++ .../Query/ProjectionTests.cs | 78 +- .../Query/QueryModeGateIncludeTests.cs | 454 ++ .../Query/QueryModeGateTests.cs | 631 +++ .../Query/QueryModeOptionTests.cs | 107 + .../Query/RequiredNavigationUnwindTests.cs | 43 +- .../StreamingCollectionNormalizationTests.cs | 194 + .../Query/UnsupportedQueryTests.cs | 22 +- .../AGENTS.md | 6 + .../Query/MongoSpecTestHelpers.cs | 68 + ...rthwindAggregateOperatorsQueryMongoTest.cs | 99 +- .../Query/NorthwindBulkUpdatesMongoTest.cs | 25 +- .../NorthwindChangeTrackingQueryMongoTest.cs | 8 +- .../Query/NorthwindCompiledQueryMongoTest.cs | 49 +- ...orthwindEFPropertyIncludeQueryMongoTest.cs | 38 +- .../Query/NorthwindFunctionsQueryMongoTest.cs | 33 +- .../Query/NorthwindGroupByQueryMongoTest.cs | 866 ++-- ...orthwindIncludeNoTrackingQueryMongoTest.cs | 44 +- .../Query/NorthwindIncludeQueryMongoTest.cs | 44 +- .../Query/NorthwindJoinQueryMongoTest.cs | 60 +- .../NorthwindKeylessEntitiesQueryMongoTest.cs | 16 +- .../NorthwindMiscellaneousQueryMongoTest.cs | 197 +- .../NorthwindNavigationsQueryMongoTest.cs | 6 +- .../NorthwindQueryFiltersQueryMongoTest.cs | 4 +- .../NorthwindQueryTaggingQueryMongoTest.cs | 4 +- .../Query/NorthwindSelectQueryMongoTest.cs | 80 +- .../NorthwindSetOperationsQueryMongoTest.cs | 61 +- .../NorthwindStringIncludeQueryMongoTest.cs | 44 +- .../Query/NorthwindWhereQueryMongoTest.cs | 36 +- .../Query/VectorSearchExactMongoTest.cs | 2 +- .../Query/VectorSearchMongoTest.cs | 2 +- .../Utilities/MongoTestStore.cs | 14 +- .../Query/DeserializeScalarOverflowTests.cs | 81 + .../Query/Ef373DependenciesPrecedeTests.cs | 167 + .../ArrayAliasProjectionExpressionTests.cs | 115 + ...ongoQueryExpressionApplyProjectionTests.cs | 195 + .../AggregateStageRenderingTests.cs | 81 + .../Ef362ArrayLeafPathTests.cs | 173 + ...MongoAggregationExpressionRendererTests.cs | 362 ++ .../MongoCardinalityRouteTests.cs | 106 + .../MongoExpressionNegatorTests.cs | 486 ++ .../NativeTranslation/MongoExpressionTests.cs | 86 + .../MongoExpressionTranslatorTests.cs | 2438 ++++++++++ .../MongoFieldPrefixRewriterTests.cs | 127 + .../MongoPipelineFactoryDeferredSlotTests.cs | 250 ++ .../MongoPipelineFactoryTests.cs | 760 ++++ .../MongoPipelineStageTests.cs | 55 + .../MongoQueryLanguageRendererTests.cs | 984 ++++ ...ngoSelectDefinitionProjectionAliasTests.cs | 167 + .../MongoSelectDefinitionTests.cs | 379 ++ .../MongoSelectDefinitionTrailingOpsTests.cs | 93 + .../MongoSelectLowererTests.cs | 835 ++++ .../MongoSetOperationRouteTests.cs | 84 + .../MongoValueRendererTests.cs | 74 + .../NativeCorrelationMatcherTests.cs | 296 ++ .../NativeDispositionTests.cs | 103 + .../NativeGroupByBinderTests.cs | 313 ++ .../NativeProjectionBinderBareBodyTests.cs | 837 ++++ .../NativeSelectManyBinderTests.cs | 1084 +++++ .../NativeVectorSearchStageTests.cs | 344 ++ .../ReferenceIncludeRecognizerTests.cs | 154 + .../NativeTranslation/SlotPopulationTests.cs | 397 ++ .../StreamingEligibilityTests.cs | 244 + .../StreamingReferenceLookupsTests.cs | 176 + .../Step3aBareProjectionFallbackStripTests.cs | 155 + 208 files changed, 62946 insertions(+), 1661 deletions(-) delete mode 100644 docs/failing-spec-tests.md create mode 100644 src/MongoDB.EntityFrameworkCore/Infrastructure/MongoQueryMode.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/ArrayAliasProjectionExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/IArrayProjectionExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoBinaryExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoCardinality.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoConstantExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoConvertExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoElemMatchExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoElementRefExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoFieldExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoFilteredSizeExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoGrouping.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoInExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoOrdering.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoParameterExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoProjection.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoRegexExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoSelectDefinition.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoSelectOp.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoSetOperation.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoSizeExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoUnaryExpression.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoUnwindSource.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoVectorSearch.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/BsonValueSerializer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoAggregationExpressionRenderer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoEntityMaterializerSerializer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoExpressionNegator.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoExpressionTranslator.Members.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoExpressionTranslator.MethodCalls.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoExpressionTranslator.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoFieldPrefixRewriter.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoNativeBuildContext.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoPipelineFactory.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoQueryLanguageRenderer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoSelectLowerer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoStreamingEntityMaterializerRewriter.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/MongoValueRenderer.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeCardinalityBinder.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeCorrelationMatcher.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeGroupByBinder.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeProjectionBinder.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeQueryParameter.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeSelectManyBinder.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeSlotPopulator.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeTranslationNotSupportedException.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/NativeVectorSearchBinder.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/PlaceholderTable.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoAddFieldsStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoCountStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoGroupAccumulatorStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoGroupStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoLimitStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoLookupStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoMatchStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoPipelineStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoProjectStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoReplaceRootStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoSetDifferenceStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoSkipStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoSortStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoUnionWithStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoUnsetStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoUnwindFieldStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoUnwindStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoVectorSearchScoreStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/Stages/MongoVectorSearchStage.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/NativeTranslation/StreamingEligibility.cs create mode 100644 src/MongoDB.EntityFrameworkCore/Query/VectorSearchStageBuilder.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/Ef362OwnedHopArrayProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/Ef372DeepReferenceIncludeTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/Ef373InterleavedPagingTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/Ef379RootNavigationMisroutingTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeArrayProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeBareProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeCardinalityTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeCastTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeCombinationQueryTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeCompositionSeamAuditTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeComputedBareProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeComputedProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeComputedSortTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeDistinctTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeEfPropertyLeafTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeExprComparisonTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeGateRoutingTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeGroupByTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeJoinInnerDeclineTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeMaterializerNullabilityTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeMaterializerOnePassTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeNullableMemberTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOfTypeTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedCollectionAllTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedCollectionCountTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedCollectionFilteredCountTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedCollectionPredicateTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedCollectionWholeEntityTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedReferenceWholeEntityTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeOwnedSubPropertyTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativePipelineExecutionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeReferenceIncludeTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeSelectManyTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeSetOpsTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeTransactionAndCancellationTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/NativeVectorSearchTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/ProjectedCollectionNormalizationTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/QueryModeGateIncludeTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/QueryModeGateTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/QueryModeOptionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/StreamingCollectionNormalizationTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/MongoSpecTestHelpers.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/DeserializeScalarOverflowTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Ef373DependenciesPrecedeTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Expressions/ArrayAliasProjectionExpressionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Expressions/MongoQueryExpressionApplyProjectionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/AggregateStageRenderingTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/Ef362ArrayLeafPathTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoAggregationExpressionRendererTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoCardinalityRouteTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoExpressionNegatorTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoExpressionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoExpressionTranslatorTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoFieldPrefixRewriterTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoPipelineFactoryDeferredSlotTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoPipelineFactoryTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoPipelineStageTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoQueryLanguageRendererTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoSelectDefinitionProjectionAliasTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoSelectDefinitionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoSelectDefinitionTrailingOpsTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoSelectLowererTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoSetOperationRouteTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/MongoValueRendererTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeCorrelationMatcherTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeDispositionTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeGroupByBinderTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeProjectionBinderBareBodyTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeSelectManyBinderTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/NativeVectorSearchStageTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/ReferenceIncludeRecognizerTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/SlotPopulationTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/StreamingEligibilityTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/NativeTranslation/StreamingReferenceLookupsTests.cs create mode 100644 tests/MongoDB.EntityFrameworkCore.UnitTests/Query/Step3aBareProjectionFallbackStripTests.cs diff --git a/.claude/settings.json b/.claude/settings.json index b409bad5d..7c6bb7fd0 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -4,8 +4,7 @@ "Bash(dotnet test:*)", "Bash(dotnet build:*)", "Bash(dotnet --list-sdks)", - "Bash(docker info)", - "Bash(git -C /Users/damieng/Source/mongo-efcore-provider pull)" + "Bash(docker info)" ] } } diff --git a/AGENTS.md b/AGENTS.md index 4556751eb..ecd010e87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ What does **not** count as a break: - Changes to anything `internal` — regardless of `InternalsVisibleTo` (which exists only to expose internals to the test assemblies). Internal code is never part of the public surface. - Changes to the exception type thrown for an **unsupported** feature (a not-yet-implemented operator, an unsupported mapping, a guard rejecting something the provider doesn't support). Only the exception type of a supported, documented operation is part of the contract. +- **Which internal execution path a supported LINQ query takes** — the driver's LINQ provider vs. the provider's own native MQL translator — and the **exact aggregation-pipeline (MQL) emitted** for a given query. These are implementation details, not contract. In particular, the native query translator (`MongoQueryMode`) becoming the **default** (`Native`) execution path is **not** a breaking change: query *results* are unchanged, any shape the native path does not support falls back to driver-LINQ automatically, and `UseQueryMode(MongoQueryMode.DriverLinq)` restores the previous path. (Recorded so this stops being re-raised in reviews — the native default and changed emitted MQL are intended and non-breaking.) ## Async conventions The provider follows EF Core's pattern, not the C# driver's. There is **no enforced sync/async pairing** of public methods — async surfaces exist where EF Core defines them (`*Async` variants) and where the underlying driver call is async. Library code uses `ConfigureAwait(false)` consistently. `CancellationToken` flows from EF Core through to driver calls without substitution; new async methods must take a `CancellationToken` and pass it on. diff --git a/BREAKING-CHANGES.md b/BREAKING-CHANGES.md index 57aae65b5..ce5fa9ea8 100644 --- a/BREAKING-CHANGES.md +++ b/BREAKING-CHANGES.md @@ -4,6 +4,122 @@ Please note that this provider **does not follow traditional semantic versioning In order to evolve the provider as we introduce new features, we will be using the minor version number for breaking and significant changes to our EF Core provider. Please bear this in mind when upgrading to newer versions of the MongoDB EF Core Provider and ensure you read the release notes and this document for the latest in breaking change information. +### Projecting a required property whose stored element is absent or `null` now throws + +#### Old behavior + +Projecting a non-nullable property returned the CLR default — `0`, `null`, `false`, … — for any document in which that element was absent, or was present but stored as BSON `null`. This applied to both spellings of a projection: + +```c# +context.Blogs.Select(b => b.Rank); // returned 0 for a document with no "Rank" element +context.Blogs.Select(b => new { b.Rank }); // the same +``` + +The projected value was deserialized by the MongoDB C# driver, whose serializers are lenient about a missing element and substitute the default. + +A *whole-entity* query over those same documents already behaved differently: `context.Blogs.ToList()` (and `Select(b => b)`) threw `InvalidOperationException: Document element 'Rank' is missing for required non-nullable property 'Rank'`. So two read paths over one document disagreed about whether it was acceptable. + +#### New behavior + +Projections are now materialized by the provider's own shaper, which applies the same required-property rule as a whole-entity read. Projecting a non-nullable property whose stored element is absent, or is explicitly BSON `null`, throws: + +``` +System.InvalidOperationException: Document element 'Rank' is missing for required non-nullable property 'Rank'. +``` + +Both spellings above are affected, and the change applies under the default query mode with no configuration change. + +Nullable properties are unaffected: `int?`, `string?` and friends still read back as `null` for both of those stored states, in every spelling. Documents that do store the element are unaffected, whatever its value. + +#### Why + +The two read paths over the same document disagreed — a whole-entity query refused it, a projection quietly substituted a default — and only one of those can be right for a property the model declares required. A projection now agrees with the whole-entity read that was already rejecting these documents, rather than silently answering `0` for data that is not there. + +#### Mitigations + +In preference order: + +1. **Make the property nullable** if the element is genuinely optional. This is the recommended fix: it makes the intent explicit in the model, both read paths then agree on `null`, and nothing else in the application has to change. + +2. **Backfill the documents** so the element is present, if the property really is required: + + ```c# + var collection = client.GetDatabase("mydb").GetCollection("Blogs"); + + collection.UpdateMany( + Builders.Filter.Exists("Rank", false), + Builders.Update.Set("Rank", 0)); + + collection.UpdateMany( + Builders.Filter.Eq("Rank", BsonNull.Value), + Builders.Update.Set("Rank", 0)); + ``` + + Use the same `IMongoClient` your `DbContext` is configured with, or a new `MongoClient` against the same connection string. To find the affected documents before changing anything, run the same two filters through `Find` instead. + +3. **As a temporary measure**, restore the previous values by opting the context back onto the driver's LINQ provider: + + ```c# + optionsBuilder.UseMongoDB(connectionString, databaseName, o => o.UseQueryMode(MongoQueryMode.DriverLinq)); + ``` + + This restores the old projection results only. A whole-entity read of the same documents throws in every query mode, as it did in previous versions — so this is a way to keep a projection working while you apply mitigation 1 or 2, not a way to make those documents readable in general. + +### A missing or `null` embedded array now materializes as an empty collection, not `null` + +#### Old behavior + +When a stored document had no field at all for an embedded (owned) collection, or had that field explicitly set to BSON `null`, the provider created no collection for the navigation. What you observed then depended on your own class: a navigation declared as a plain `public List Posts { get; set; }` read back as `null`, while one declared `public List Posts { get; set; } = []` read back as an empty collection, because the field initializer had already supplied one. + +Projecting the collection directly made the `null` unambiguous and usable, since no field initializer was involved: `Select(b => b.Posts)` returned `null` for those documents, so `posts is null` was a reliable test for "the stored array was absent or `null`", as distinct from "the stored array was present but empty". + +#### New behavior + +A missing or explicitly-`null` embedded array now materializes as an **empty** collection on every read path — whole-entity queries, `Include`, and projections alike — regardless of how the navigation property is declared. The collection is created through the navigation's own collection accessor, so a `HashSet` or custom collection navigation gets its declared type. + +The distinction between "the stored array was absent or `null`" and "the stored array was present but empty" is therefore no longer observable through a materialized collection navigation. + +Nullable primitive collection *properties* (for example `List? Tags`) are unaffected and still read back as `null`, because they map through a property serializer with its own nullability semantics rather than as a collection navigation. + +**This can change stored documents, through a read-modify-write cycle.** The write path itself is unchanged: assigning `null` to a collection navigation and saving still persists `null`. But if you load an entity, change anything about it, and call `SaveChanges`, what gets written back for the collection navigation is now the empty collection that materialization produced, where it used to be the `null` that materialization produced. Measured, for a tracked load of one document, an edit to an unrelated scalar property, and `SaveChanges` — identical in both query modes: + +| Stored `Posts` before the cycle | Written back, previous versions | Written back, this version | +|---|---|---| +| field absent | `"Posts": null` | `"Posts": []` | +| `"Posts": null` | `"Posts": null` | `"Posts": []` | +| `"Posts": []` | `"Posts": []` | `"Posts": []` | + +So loading, mutating and saving entities whose embedded arrays are ragged will normalize those arrays **in the database**. Note this is not new *document-rewriting* behavior — the previous versions already replaced an absent field with `"Posts": null` on the same cycle; what changed is the value written. It follows from the read change described above, and nothing about how a collection is serialized changed. + +#### Why + +Empty-not-null is Entity Framework Core's contract for a collection navigation, independent of the CLR type's nullability or its field initializer. The previous behavior made the materialized result depend on the application's field initializer, which meant two models over the same documents disagreed, and different read paths over the same document could disagree with each other. It also made a projected count of such a collection throw `ArgumentNullException` instead of returning `0`. + +#### Mitigations + +If you need to tell an absent or `null` stored array apart from a present-but-empty one, ask the database rather than the materialized collection. A LINQ predicate cannot express it (`!b.Posts.Any()` is true for all three states), so query the documents through the driver, where the stored shape is visible: + +```c# +var collection = client.GetDatabase("mydb").GetCollection("Blogs"); +var absentOrNull = collection.Find( + Builders.Filter.Or( + Builders.Filter.Exists("Posts", false), + Builders.Filter.Eq("Posts", BsonNull.Value))) + .ToList(); +``` + +Use the same `IMongoClient` your `DbContext` is configured with, or a new `MongoClient` against the same connection string. If you were relying on `Select(b => b.Posts)` returning `null`, replace that test with the above. + +Do that check **before** the documents pass through a read-modify-write cycle, because that cycle normalizes them (see *New behavior*). If you need the ragged stored shape preserved, avoid round-tripping those entities through the change tracker: update only the fields you actually want to change, either with `ExecuteUpdate` (whose setter API differs between EF Core versions — see the EF Core docs for the form your version takes) or with a driver update, which leaves the array field untouched: + +```c# +collection.UpdateOne( + Builders.Filter.Eq("_id", id), + Builders.Update.Set("Title", "New title")); +``` + +Otherwise, treat the normalization as a one-time migration of the documents you save, and take whatever record of the distinction you need — a backup, or an audit query using the `Find` above — before saving. + ### Entity types with their own `DbSet` are no longer embedded when reached by a navigation #### Old behavior @@ -50,6 +166,47 @@ A password-only redaction still discloses the account and deployment topology, a Use `DatabaseName` to distinguish contexts, or log non-sensitive parts of the connection string yourself if you need it. `PopulateDebugInfo`'s connection-string hash (`Mongo:ConnectionString`) is unchanged. +### Query results can differ for a numeric cast in a `Where` clause + +#### Old behavior + +A numeric cast applied to a mapped property inside a `Where` clause — `(int)x.D`, `(double)x.Weight` — was translated by the MongoDB C# driver's LINQ provider, which for many shapes **silently dropped the cast** and filtered on the raw stored value instead: + +```c# +context.Blogs.Where(x => (int)x.Score > 0); // ran as though it were: x.Score > 0 +``` + +#### New behavior + +The provider's own MQL translator now renders the cast explicitly, as `$toInt` / `$toLong` / `$toDouble` / `$toDecimal`, instead of handing the query to the driver's LINQ provider. Two observable consequences for a query that ran successfully in `10.0.2` / `9.1.2` / `8.4.2`: + +* **A narrowing cast compared against a constant now returns the C#-correct rows.** `Where(x => (int)x.D > 0)` over a document with `D = 0.5` previously matched (the cast was dropped, so `0.5 > 0` was evaluated); it no longer does, because `(int)0.5` is `0`. This is a deliberate correctness fix — the new result is the one C# itself produces for the same expression. + +* **A value outside the target type's range now raises a server error** where rows were previously returned. `Where(x => (int)x.D > 0)` over a document storing `D = 1e30` fails with `MongoDB.Driver.MongoCommandException: … Conversion would overflow target type in $convert with no onError value: 1e+30`. **The failure aborts the whole query, not just the offending document** — the conversion is evaluated for every document the query scans, including documents that would never have matched the predicate — so no rows are returned at all. Unchecked C# would instead have produced an unspecified truncated value for that document. + +#### Why + +Dropping a cast answers a different question from the one the query asked, and does so silently. Rendering it explicitly makes a filtered query agree with what the same expression means in C#. + +The overflow case is a deliberate choice between three answers that all differ: the previous behavior (rows, from a comparison the query did not ask for), unchecked C# (an unspecified value), and suppressing the error (a converted-to-`null` operand, which then participates in a BSON-ordering comparison and quietly moves the document into or out of the result depending on the operator). A loud failure is the only one of the three that cannot be mistaken for an answer, and a value that does not fit the cast's target type indicates a defect in the query or in the stored data. + +#### Mitigations + +In preference order: + +1. **Fix the expression or the data.** If the cast was incidental, remove it (`Where(x => x.D > 0)`) — that is the query the old behavior actually ran. If a stored value genuinely cannot fit the cast's target type, either widen the target (`(long)`, `(double)`) or correct the document. + +2. **Restore the previous behavior for the whole context** by opting out of the native translator: + + ```c# + optionsBuilder.UseMongoDB(connectionString, databaseName) + .UseQueryMode(MongoQueryMode.DriverLinq); + ``` + + This routes every query through the driver's LINQ provider, as in earlier versions, and restores the old result for both consequences above. + +Queries with no numeric cast in their filter are unaffected, as are casts whose stored values all fit the target type and whose truncation does not change the comparison's outcome. A relational comparison (`<`, `<=`, `>`, `>=`) whose cast is applied to a **nullable** property is also unaffected: that shape still routes through the driver's LINQ provider, so it keeps the old behavior described above rather than either new consequence. + ## Breaking changes in 8.4.0 / 9.1.0 / 10.0.0 ### The element name for discriminators may have changed diff --git a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/HeadlineBenchmarks.cs b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/HeadlineBenchmarks.cs index 43ef0710b..36285f818 100644 --- a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/HeadlineBenchmarks.cs +++ b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/HeadlineBenchmarks.cs @@ -1,23 +1,29 @@ using BenchmarkDotNet.Attributes; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.Linq; +using MongoDB.EntityFrameworkCore.Infrastructure; namespace MongoDB.EntityFrameworkCore.Benchmarks; -// Two-config headline benchmarks (on main, before the native path exists): -// DriverOnly - raw MongoDB C# driver LINQ / aggregation, no EF Core (perf floor). -// EF - the current EF provider (driver-LINQ delegation == main baseline). -// A third config (EF-Native) is added in sub-project 1. All configs read the SAME documents -// seeded once in [GlobalSetup], via one shared MongoClient (fairness: a per-context client would -// charge connection-pool + topology startup to the EF numbers). +// Three-config headline benchmarks (EF-323: native query path): +// DriverOnly - raw MongoDB C# driver LINQ / aggregation, no EF Core (perf floor). +// EF_DriverLinq - EF provider pinned to UseQueryMode(MongoQueryMode.DriverLinq) == the +// pre-rebuild "EF-current" baseline (matches the numbers in perf-baseline.md). +// EF_Native - EF provider in UseQueryMode(MongoQueryMode.Native) == the new native path. +// ReferenceInclude falls back to driver-LINQ (deferred — Include is not yet +// native), so EF_Native ≈ EF_DriverLinq for that shape. +// All configs read the SAME documents seeded once in [GlobalSetup], via one shared MongoClient +// (fairness: a per-context client would charge connection-pool + topology startup to EF numbers). [Config(typeof(BenchmarkConfig))] public class HeadlineBenchmarks { private const int N = 10_000; - private DbContextOptions _efOptions = null!; + private DbContextOptions _efOptionsDriverLinq = null!; + private DbContextOptions _efOptionsNative = null!; private IMongoCollection _flatColl = null!; private IMongoCollection _reviewColl = null!; private IMongoCollection _productColl = null!; @@ -31,10 +37,18 @@ public void Setup() _dbName = "ef_bench_headline_" + Guid.NewGuid().ToString("N"); _client = new MongoClient(conn); - _efOptions = new DbContextOptionsBuilder() - .UseMongoDB(_client, _dbName).Options; + _efOptionsDriverLinq = new DbContextOptionsBuilder() + .UseMongoDB(_client, _dbName, o => o.UseQueryMode(MongoQueryMode.DriverLinq)) + .ConfigureWarnings(w => w.Ignore(CoreEventId.ManyServiceProvidersCreatedWarning)) + .Options; - using (var ctx = new BenchmarkDbContext(_efOptions)) + _efOptionsNative = new DbContextOptionsBuilder() + .UseMongoDB(_client, _dbName, o => o.UseQueryMode(MongoQueryMode.Native)) + .ConfigureWarnings(w => w.Ignore(CoreEventId.ManyServiceProvidersCreatedWarning)) + .Options; + + // Seed using DriverLinq options (seeding is writes, unaffected by query mode). + using (var ctx = new BenchmarkDbContext(_efOptionsDriverLinq)) { BenchmarkSeeder.Seed(ctx, flatCount: N, productCount: 100, reviewCount: N); } @@ -53,17 +67,35 @@ private void Validate() var driverWhere = _flatColl.AsQueryable().Where(f => f.Active).ToList().Count; var driverReview = DriverReviewInclude(); - using var ef = new BenchmarkDbContext(_efOptions); - var efAll = ef.FlatItems.AsNoTracking().ToList().Count; - var efWhere = ef.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; - var efReview = ef.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count(r => r.Product != null); - - if (driverAll != N || efAll != N) - throw new InvalidOperationException($"FlatItem count mismatch: driver={driverAll}, ef={efAll}, expected {N}."); - if (driverWhere != efWhere) - throw new InvalidOperationException($"Where count mismatch: driver={driverWhere}, ef={efWhere}."); - if (driverReview != N || efReview != N) - throw new InvalidOperationException($"Review+Product mismatch: driver={driverReview}, ef={efReview}, expected {N}."); + // Validate EF_DriverLinq + using (var efDl = new BenchmarkDbContext(_efOptionsDriverLinq)) + { + var efAll = efDl.FlatItems.AsNoTracking().ToList().Count; + var efWhere = efDl.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; + var efReview = efDl.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count(r => r.Product != null); + + if (driverAll != N || efAll != N) + throw new InvalidOperationException($"FlatItem count mismatch (DriverLinq): driver={driverAll}, ef={efAll}, expected {N}."); + if (driverWhere != efWhere) + throw new InvalidOperationException($"Where count mismatch (DriverLinq): driver={driverWhere}, ef={efWhere}."); + if (driverReview != N || efReview != N) + throw new InvalidOperationException($"Review+Product mismatch (DriverLinq): driver={driverReview}, ef={efReview}, expected {N}."); + } + + // Parity: EF_Native must return the same counts as EF_DriverLinq (correctness gate). + using (var efN = new BenchmarkDbContext(_efOptionsNative)) + { + var efAllNative = efN.FlatItems.AsNoTracking().ToList().Count; + var efWhereNative = efN.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; + var efReviewNative = efN.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count(r => r.Product != null); + + if (driverAll != efAllNative) + throw new InvalidOperationException($"FlatItem count mismatch (Native): driver={driverAll}, efNative={efAllNative}, expected {N}."); + if (driverWhere != efWhereNative) + throw new InvalidOperationException($"Where count mismatch (Native): driver={driverWhere}, efNative={efWhereNative}."); + if (efReviewNative != N) + throw new InvalidOperationException($"Review+Product mismatch (Native): efNative={efReviewNative}, expected {N}."); + } } // Hand-written $lookup + $unwind equivalent of Reviews.Include(r => r.Product). @@ -97,7 +129,7 @@ private sealed class ReviewWithProduct [GlobalCleanup] public void Cleanup() { - using var ctx = new BenchmarkDbContext(_efOptions); + using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); ctx.Database.EnsureDeleted(); } @@ -105,30 +137,47 @@ public void Cleanup() [Benchmark] public int WhereToList_DriverOnly() => _flatColl.AsQueryable().Where(f => f.Active).ToList().Count; - [Benchmark] public int WhereToList_EF() - { using var ctx = new BenchmarkDbContext(_efOptions); return ctx.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; } + [Benchmark] public int WhereToList_EF_DriverLinq() + { using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); return ctx.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; } + + [Benchmark] public int WhereToList_EF_Native() + { using var ctx = new BenchmarkDbContext(_efOptionsNative); return ctx.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; } // ----- whole-entity ToList() ----- [Benchmark] public int WholeEntityToList_DriverOnly() => _flatColl.AsQueryable().ToList().Count; - [Benchmark] public int WholeEntityToList_EF_NoTracking() - { using var ctx = new BenchmarkDbContext(_efOptions); return ctx.FlatItems.AsNoTracking().ToList().Count; } + [Benchmark] public int WholeEntityToList_EF_DriverLinq_NoTracking() + { using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); return ctx.FlatItems.AsNoTracking().ToList().Count; } + + [Benchmark] public int WholeEntityToList_EF_DriverLinq_Tracked() + { using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); return ctx.FlatItems.ToList().Count; } + + [Benchmark] public int WholeEntityToList_EF_Native_NoTracking() + { using var ctx = new BenchmarkDbContext(_efOptionsNative); return ctx.FlatItems.AsNoTracking().ToList().Count; } - [Benchmark] public int WholeEntityToList_EF_Tracked() - { using var ctx = new BenchmarkDbContext(_efOptions); return ctx.FlatItems.ToList().Count; } + [Benchmark] public int WholeEntityToList_EF_Native_Tracked() + { using var ctx = new BenchmarkDbContext(_efOptionsNative); return ctx.FlatItems.ToList().Count; } // ----- OrderBy(x => x.Count).Take(100).ToList() ----- [Benchmark] public int OrderByTake_DriverOnly() => _flatColl.AsQueryable().OrderBy(f => f.Count).Take(100).ToList().Count; - [Benchmark] public int OrderByTake_EF() - { using var ctx = new BenchmarkDbContext(_efOptions); return ctx.FlatItems.AsNoTracking().OrderBy(f => f.Count).Take(100).ToList().Count; } + [Benchmark] public int OrderByTake_EF_DriverLinq() + { using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); return ctx.FlatItems.AsNoTracking().OrderBy(f => f.Count).Take(100).ToList().Count; } + + [Benchmark] public int OrderByTake_EF_Native() + { using var ctx = new BenchmarkDbContext(_efOptionsNative); return ctx.FlatItems.AsNoTracking().OrderBy(f => f.Count).Take(100).ToList().Count; } // ----- Reviews.Include(r => r.Product).ToList() ----- + // Note: ReferenceInclude falls back to driver-LINQ even in Native mode (Include is deferred — + // not yet native). EF_Native ≈ EF_DriverLinq for this shape. [Benchmark] public int ReferenceInclude_DriverOnly() => DriverReviewInclude(); - [Benchmark] public int ReferenceInclude_EF() - { using var ctx = new BenchmarkDbContext(_efOptions); return ctx.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count; } + [Benchmark] public int ReferenceInclude_EF_DriverLinq() + { using var ctx = new BenchmarkDbContext(_efOptionsDriverLinq); return ctx.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count; } + + [Benchmark] public int ReferenceInclude_EF_Native() + { using var ctx = new BenchmarkDbContext(_efOptionsNative); return ctx.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count; } } diff --git a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/Program.cs b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/Program.cs index 6ff7e8ff4..dd44a7af4 100644 --- a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/Program.cs +++ b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/Program.cs @@ -1,5 +1,87 @@ using Microsoft.EntityFrameworkCore; +using MongoDB.Driver; using MongoDB.EntityFrameworkCore.Benchmarks; +using MongoDB.EntityFrameworkCore.Infrastructure; + +if (args.Contains("--verify-native")) +{ + var conn = Environment.GetEnvironmentVariable("MONGODB_URI") ?? "mongodb://localhost:27017"; + var dbName = "ef_bench_nativeverify_" + Guid.NewGuid().ToString("N"); + var client = new MongoClient(conn); + + var seedOptions = new DbContextOptionsBuilder() + .UseMongoDB(client, dbName, o => o.UseQueryMode(MongoQueryMode.DriverLinq)).Options; + var nativeOnlyOptions = new DbContextOptionsBuilder() + .UseMongoDB(client, dbName, o => o.UseQueryMode(MongoQueryMode.NativeOnly)).Options; + + try + { + using (var ctx = new BenchmarkDbContext(seedOptions)) + BenchmarkSeeder.Seed(ctx, flatCount: 100, productCount: 20, reviewCount: 100); + + Console.WriteLine("=== NativeOnly verification ==="); + + static void RunShape(string label, Action test, bool expectFallback = false) + { + try + { + test(); + if (expectFallback) + Console.WriteLine($"{label}: NATIVE OK <-- UNEXPECTED, should fall back!"); + } + catch (Exception ex) when (ex.GetType().Name == "NativeTranslationNotSupportedException") + { + if (expectFallback) + Console.WriteLine($"{label}: FALLBACK CONFIRMED (NativeTranslationNotSupportedException)"); + else + Console.WriteLine($"{label}: FALLBACK (unexpected) — {ex.Message[..Math.Min(80, ex.Message.Length)]}"); + } + catch (Exception ex) + { + Console.WriteLine($"{label}: ERROR — {ex.GetType().Name}: {ex.Message[..Math.Min(80, ex.Message.Length)]}"); + } + } + + // WhereToList — should be NATIVE OK + RunShape("WhereToList: ", () => + { + using var ctx = new BenchmarkDbContext(nativeOnlyOptions); + var count = ctx.FlatItems.AsNoTracking().Where(f => f.Active).ToList().Count; + Console.WriteLine($"WhereToList: NATIVE OK (count={count})"); + }); + + // WholeEntityToList — should be NATIVE OK + RunShape("WholeEntityToList:", () => + { + using var ctx = new BenchmarkDbContext(nativeOnlyOptions); + var count = ctx.FlatItems.AsNoTracking().ToList().Count; + Console.WriteLine($"WholeEntityToList: NATIVE OK (count={count})"); + }); + + // OrderByTake — should be NATIVE OK + RunShape("OrderByTake: ", () => + { + using var ctx = new BenchmarkDbContext(nativeOnlyOptions); + var count = ctx.FlatItems.AsNoTracking().OrderBy(f => f.Count).Take(10).ToList().Count; + Console.WriteLine($"OrderByTake: NATIVE OK (count={count})"); + }); + + // ReferenceInclude — expected to throw NativeTranslationNotSupportedException (Include deferred) + RunShape("ReferenceInclude: ", () => + { + using var ctx = new BenchmarkDbContext(nativeOnlyOptions); + var count = ctx.Reviews.AsNoTracking().Include(r => r.Product).ToList().Count; + }, expectFallback: true); + + Console.WriteLine("=== verification done ==="); + } + finally + { + using var ctx = new BenchmarkDbContext(seedOptions); + ctx.Database.EnsureDeleted(); + } + return; +} if (args.Contains("--extended-smoke")) { diff --git a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/results/perf-baseline.md b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/results/perf-baseline.md index 658d5ef71..d8b8dac1d 100644 --- a/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/results/perf-baseline.md +++ b/benchmarks/MongoDB.EntityFrameworkCore.Benchmarks/results/perf-baseline.md @@ -47,3 +47,95 @@ ACCOUNT_N=200, ORDER_N=10,000, WIDE_N=5,000. Order shape: 3 lines × 2 discounts - Each EF op constructs a new DbContext (realistic per-unit-of-work usage) while the driver reuses one IMongoCollection — a deliberate asymmetry. - EF (tracked) measures change-tracking/snapshot overhead, not relationship fixup (FlatItem has no navigations). - Aborted runs leak GUID-named `ef_bench_*` databases (cleanup runs only in GlobalCleanup). + +--- + +## Native config (EF-323) — authoritative (quiet-machine pass) + +Run: 2026-06-29, **quiet Apple M4 Max** (14 cores), machine held idle during the run (no parallel work). BenchmarkDotNet v0.15.8, .NET 10.0.9 Arm64. Supersedes the earlier provisional containerized pass — both Mean and Allocated are authoritative here and directly comparable to the baseline table above. + +Three configs: +- **DriverOnly** — raw MongoDB C# driver LINQ, no EF Core (floor). +- **EF_DriverLinq** — EF provider pinned to `UseQueryMode(MongoQueryMode.DriverLinq)` (pre-rebuild baseline; matches the EF numbers in the table above). +- **EF_Native** — EF provider using `UseQueryMode(MongoQueryMode.Native)` (new path). + +N = 10,000. Config: Release EF10, InProcessEmitToolchain, 3 warmup / 10 iterations, MemoryDiagnoser. + +### Three-config table + +| Shape | Config | Mean (quiet M4 Max) | Allocated | +|---|---|---|---| +| WhereToList | DriverOnly | 6.084 ms | 1589.90 KB | +| WhereToList | EF_DriverLinq | 15.146 ms | 22683.02 KB | +| WhereToList | EF_Native | 8.269 ms | 9590.97 KB | +| WholeEntityToList | DriverOnly | 8.481 ms | 3133.01 KB | +| WholeEntityToList | EF_DriverLinq (no-track) | 32.373 ms | 45284.27 KB | +| WholeEntityToList | EF_DriverLinq (tracked) | 42.683 ms | 51369.10 KB | +| WholeEntityToList | EF_Native (no-track) | 15.813 ms | 19108.57 KB | +| WholeEntityToList | EF_Native (tracked) | 25.219 ms | 25196.83 KB | +| OrderByTake | DriverOnly | 2.326 ms | 59.42 KB | +| OrderByTake | EF_DriverLinq | 3.197 ms | 513.80 KB | +| OrderByTake | EF_Native | 2.608 ms | 243.13 KB | +| ReferenceInclude | DriverOnly | 37.310 ms | 7332.72 KB | +| ReferenceInclude | EF_DriverLinq | 136.710 ms | 51953.32 KB | +| ReferenceInclude | EF_Native | 138.487 ms | 51952.91 KB | + +### Native-vs-EF_DriverLinq allocation delta (authoritative) + +| Shape | EF_DriverLinq Allocated | EF_Native Allocated | Delta (KB) | Delta (%) | +|---|---|---|---|---| +| WhereToList | 22565.71 KB | 9590.87 KB | -12974.84 KB | **-57.5%** | +| WholeEntityToList (no-track) | 45049.07 KB | 19108.51 KB | -25940.56 KB | **-57.6%** | +| WholeEntityToList (tracked) | 51134.06 KB | 25195.16 KB | -25938.90 KB | **-50.7%** | +| OrderByTake | 511.44 KB | 243.16 KB | -268.28 KB | **-52.5%** | +| ReferenceInclude | 51952.73 KB | 51952.50 KB | ~0 KB | **0%** (Include falls back — deferred) | + +### Key observations + +- **WhereToList / WholeEntityToList / OrderByTake**: native path cuts allocations by ~50-58% **and** wall-clock time by ~18-51% versus EF_DriverLinq. Native-vs-EF_DriverLinq Mean: WhereToList 8.27 vs 15.15 ms (**-45%**); WholeEntity no-track 15.81 vs 32.37 ms (**-51%**); WholeEntity tracked 25.22 vs 42.68 ms (**-41%**); OrderByTake 2.61 vs 3.20 ms (**-18%**). +- **ReferenceInclude EF_Native ≈ EF_DriverLinq**: Include navigation is not yet native (deferred) — the query falls back to driver-LINQ automatically in `Native` mode, so both configs produce identical allocations and similar times. This is expected behavior. +- **EF_DriverLinq baseline vs prior-recorded EF numbers**: EF_DriverLinq allocations (22566 / 45049 / 51134 / 511 / 51953 KB) match the previously recorded EF numbers in the baseline table (22563 / 45048 / 51132 / 509 / 51956 KB) within normal run-to-run variance — confirming EF_DriverLinq is a faithful reproduction of the old EF baseline. +- **vs the spike's recorded headline (the SP1 "native ≥ spike" success bar): MET on the realized native slice.** The program design recorded (current → native): Where→ToList 15.1→8.3 ms / 22.7→9.6 MB; whole-entity no-track 33.4→15.6 ms / 45.3→19.1 MB. Our quiet-machine native matches almost exactly — Where 8.27 ms / 9.59 MB, whole-entity 15.81 ms / 19.11 MB — i.e. the B2 rebuild reproduces the spike's native performance (≥, within run-to-run noise), confirming no perf regression vs the spike for filter/sort/paging/whole-entity. +- **Reference Include is the one shape below the spike — by design (deferred).** The spike did Include natively (recorded 114.7 ms / 22.3 MB); here Include falls back to driver-LINQ (~138 ms / 52 MB ≈ EF_DriverLinq), so the spike's Include-native win is **not** realized yet — it is the payoff of the future Collection Includes sub-project, not an SP1 regression. + +### Genuine-native confirmation (NativeOnly mode) + +Verified by running each shape against `MongoQueryMode.NativeOnly` — a mode that throws `NativeTranslationNotSupportedException` instead of falling back: + +| Shape | Result | +|---|---| +| WhereToList | NATIVE OK | +| WholeEntityToList | NATIVE OK | +| OrderByTake | NATIVE OK | +| ReferenceInclude | FALLBACK CONFIRMED (NativeTranslationNotSupportedException thrown) | + +--- + +## SP7 — one-pass materializer (EF-322 sub-project 7) + +Run: 2026-07-26, Apple M4 Max host, Release EF10, InProcessEmitToolchain, 3 warmup / 10 iterations, MemoryDiagnoser, N=10,000, against a local replica-set mongod. `EF_Native` here is the **SP7 one-pass** path: a custom `IBsonSerializer` (EF's compiled materializer reading off the cursor's own `IBsonReader`) as the `Aggregate` output serializer (deserialize *is* materialize), plus allocation-free per-property typed reads (reused `BsonDeserializationContext`, generic `Deserialize`, no boxing). Compare the `EF_Native` **Allocated** column against the pre-SP7 native numbers in the "Native config (EF-323)" table above. Allocation is deterministic under MemoryDiagnoser; timing on this pass is directional (not the quiet-machine methodology). + +| Shape | Config | Mean | Allocated | +|---|---|---|---| +| WhereToList | DriverOnly | 5.27 ms | 1589.85 KB | +| WhereToList | EF_DriverLinq | 15.86 ms | 22565.86 KB | +| WhereToList | **EF_Native (SP7)** | 5.14 ms | **2751.28 KB** | +| WholeEntityToList | DriverOnly | 8.28 ms | 3133.07 KB | +| WholeEntityToList | EF_DriverLinq (no-track) | 35.42 ms | 45049.17 KB | +| WholeEntityToList | EF_DriverLinq (tracked) | 42.56 ms | 51134.51 KB | +| WholeEntityToList | **EF_Native (SP7, no-track)** | 8.78 ms | **5430.45 KB** | +| WholeEntityToList | **EF_Native (SP7, tracked)** | 17.64 ms | **11518.93 KB** | +| OrderByTake | DriverOnly | 2.00 ms | 59.42 KB | +| OrderByTake | EF_DriverLinq | 2.97 ms | 511.52 KB | +| OrderByTake | **EF_Native (SP7)** | 2.10 ms | **106.63 KB** | + +### SP7 native allocation delta + +| Shape | pre-SP7 native | SP7 native | Δ vs pre-SP7 | × driver-only floor | +|---|---|---|---|---| +| WhereToList | 9590.97 KB | 2751.28 KB | **−71.3%** | 1.73× (floor 1589.85) | +| WholeEntity no-track | 19108.57 KB | 5430.45 KB | **−71.6%** | 1.73× (floor 3133.07) | +| WholeEntity tracked | 25196.83 KB | 11518.93 KB | **−54.3%** | — (tracking overhead; no driver-only tracked floor) | +| OrderByTake | 243.13 KB | 106.63 KB | **−56.1%** | 1.79× (floor 59.42) | + +**Result.** The two-optimization one-pass materializer (custom output serializer collapsing the double-pass + allocation-free per-property reads) cuts native allocation ~54–72% and brings the streaming shapes to ~**1.73× the raw-driver floor** (from ~6.1× pre-SP7). Wall-clock also improves: WhereToList native (5.14 ms) is on par with / below DriverOnly (5.27 ms), and whole-entity no-track (8.78 ms) ≈ the driver-only floor (8.28 ms). The residual ~1.73× is EF's own materializer / tracking-buffer allocation — a separate axis SP7 does not target. Correctness: full EF8/EF9/EF10 suite green (7384/7745/7342, 0 failed); the `NativeOnly` EF10 spec pass-set is unchanged at 2192/2397/19 (materialization-only; no eligibility change). diff --git a/docs/failing-spec-tests.md b/docs/failing-spec-tests.md deleted file mode 100644 index 606492f16..000000000 --- a/docs/failing-spec-tests.md +++ /dev/null @@ -1,437 +0,0 @@ -# Failing specification tests — Jira inventory - -EF Core ships a "specification tests" suite that providers consume from the -`Microsoft.EntityFrameworkCore.Specification.Tests` NuGet package and override -to assert the provider's actual behavior. When the MongoDB provider does not -yet support the functionality exercised by a test, the override is updated to -**assert the current failure** rather than be skipped — so the test stays green, -and any change in behavior (different exception type, different error message, -or the feature beginning to work) is detected immediately. - -Each such override is annotated with a `// Fails: ` -comment. This document enumerates every ticket referenced by such comments -together with a one-line description, plus a section listing failure modes -that currently lack a ticket. Counts are sourced from `tests/MongoDB.EntityFrameworkCore.SpecificationTests/**/*.cs`. - -> **Counting basis — read before trusting a number in the `Count` column.** Where a row is restated it -> means: the number of distinct overridden test methods carrying a `// Fails: … ` tag in -> `tests/MongoDB.EntityFrameworkCore.SpecificationTests/**/*.cs` (a method split across `#if` arms counts -> once). Most figures in the column predate that rule and **do not reconcile with it**: measured at -> `58e05a0`, EF-216 read 265 where the rule yields 217, EF-X020 read 168 against 171, EF-149 246 against -> 292, and EF-X002's 46-against-45 discrepancy was recorded earlier and never explained. No rule tried -> (raw tag lines, per-method, per-method-name, whole-`tests/`-tree) reproduces the older figures, so their -> basis is unknown. Rows touched from 2026-08-03 onwards state the rule's figure; the rest are left as -> they stand rather than silently re-derived. - -> If you fix one of these bugs, search for the ticket id in the spec-tests -> project — the corresponding overrides need to be updated (drop the -> `AssertTranslationFailed` wrapper or the throws-asserting `Assert.Contains`, -> assert the new MQL baseline, and remove the `// Fails:` line). - ---- - -## MongoDB EF Core provider tickets — `EF-NNN` - -| Ticket | Comment subject | Description | Count | -| --- | --- | --- | --- | -| [EF-117](https://jira.mongodb.org/browse/EF-117) | _(no remaining `// Fails:` tags)_ | Cross-collection **Include**/`ThenInclude` is now implemented for the tested shapes. The five tests formerly tagged here were re-investigated: `Outer_identifier_correctly_determined_when_doing_include_on_right_side_of_left_join` (tracking + no-tracking) now **passes**; `Collection_include_over_result_of_single_non_scalar` and `Do_not_erase_projection_mapping_when_adding_single_projection` actually fail on cross-`DbSet` subquery translation (re-tagged **EF-X001**); `Included_one_to_many_query_with_client_eval` fails on driver client-evaluation (re-tagged **EF-X003**); and Include on a keyless entity (incl. multi-level) is a genuine PK-less `$lookup` gap (re-tagged **EF-X019**). EF-117 no longer has any active `// Fails:` tags. (Join/GroupJoin/SelectMany/RightJoin/subquery failures formerly tagged here were re-categorized — see EF-X001/EF-216/EF-220/EF-X016/EF-X017/EF-X018.) | 0 | -| [EF-149](https://jira.mongodb.org/browse/EF-149) | `GroupBy issue EF-149` | `GroupBy` translation is severely limited; most non-trivial group-by shapes fail to translate. | 246 | -| [EF-153](https://jira.mongodb.org/browse/EF-153) | `TagWith EF-153` | `TagWith(...)` content is silently dropped — does not appear in the emitted MQL. | 9 | -| [EF-164](https://jira.mongodb.org/browse/EF-164) | `Missing property values issue EF-164` / `Projections issue EF-164` | BSON documents that omit a required scalar (or required navigation) throw on materialization — `Project_root_with_missing_scalars`, `Project_root_entity_with_missing_required_navigation`, etc. | 3 | -| [EF-202](https://jira.mongodb.org/browse/EF-202) | `Entity equality issue EF-202` | Comparing two entities (`entity1 == entity2` / `Contains(entity)`) is not lowered to a key-equality comparison. | 4 | -| [EF-216](https://jira.mongodb.org/browse/EF-216) | `Cross-document navigation access issue EF-216` / `Navigations issue EF-216` | Navigations that cross collection boundaries cannot be translated; surfaces as `Unsupported cross-DbSet query between ...`. Documented at the helper `AssertNoMultiCollectionQuerySupport`. | 211 | -| [EF-217](https://jira.mongodb.org/browse/EF-217) | `Call ToString on DateTimeOffset EF-217` | `DateTimeOffset.ToString()` cannot be translated. | 2 | -| [EF-220](https://jira.mongodb.org/browse/EF-220) | `Multiple query roots issue EF-220` | Queries that reference more than one `DbSet<>` (Cartesian product / cross-join) are not translatable. Includes `SelectMany` across DbSets and tautology-predicate cross-joins. | 10 | -| [EF-221](https://jira.mongodb.org/browse/EF-221) | _(no remaining `// Fails:` tags)_ | `Equals` calls with genuinely mismatched, non-convertible simple types (e.g. `int?.Equals((ulong)x)`) are now folded to the correct constant result in `MongoEFToLinqTranslatingExpressionVisitor` instead of reaching the driver's `Equals` translator, which threw `InvalidCastException`. The four tests formerly tagged here now pass. | 0 | -| [EF-222](https://jira.mongodb.org/browse/EF-222) | `translation of Like issue EF-222` | `EF.Functions.Like(...)` is not translated. | 9 | -| [EF-227](https://jira.mongodb.org/browse/EF-227) | `Max over empty nullables issue EF-227` | `Min` / `Max` over an empty nullable sequence does not produce the EF-expected `null`. | 4 | -| [EF-228](https://jira.mongodb.org/browse/EF-228) | `Truncation data loss issue EF-228` | `Sum`/`Average` over `float` columns suffers precision/truncation loss when accumulated server-side. | 2 | -| [EF-232](https://jira.mongodb.org/browse/EF-232) | `Sum of empty set cast to nullable issue EF-232` | `Sum_with_no_data_cast_to_nullable` does not produce the EF-expected `null`. (The `Compiled_query_when_does_not_end_in_query_operator` failure that previously also cited EF-232 has been re-tagged as `EF-X011`.) | 1 | -| [EF-234](https://jira.mongodb.org/browse/EF-234) | `translation of Random issue EF-234` | `EF.Functions.Random()` is not translated. | 2 | -| [EF-235](https://jira.mongodb.org/browse/EF-235) | `Translate Convert methods issue EF-235` | `Convert.ToBoolean/Byte/Int*/Decimal/Double/String/...` calls are not translated. | 8 | -| [EF-237](https://jira.mongodb.org/browse/EF-237) | `MathF mapping issue EF-237` | `MathF.*` overloads (the `float` Math API) are not translated. | 25 | -| [EF-238](https://jira.mongodb.org/browse/EF-238) | `Math.Min/Math.Max mapping issue EF-238` | `Math.Min` / `Math.Max` are not translated. | 6 | -| [EF-239](https://jira.mongodb.org/browse/EF-239) | `Math.Sign mapping issue EF-239` | `Math.Sign` is not translated. | 1 | -| [EF-240](https://jira.mongodb.org/browse/EF-240) | `Double.RadiansToDegrees and Double.DegreesToRadians mapping issue EF-240` | `Double.RadiansToDegrees` / `Double.DegreesToRadians` are not translated. | 2 | -| [EF-241](https://jira.mongodb.org/browse/EF-241) | `Translate string.Trim methods issue EF-241` | `String.TrimStart(...)` / `String.TrimEnd(...)` with a `char[]` argument are not translated. | 6 | -| [EF-242](https://jira.mongodb.org/browse/EF-242) | `DateOnly support issue EF-242` | `DateOnly.FromDateTime(...)` (and related `DateOnly` conversions) are not translated. | 1 | -| [EF-243](https://jira.mongodb.org/browse/EF-243) | `StartsWith/Contains/EndsWith Ordinal/OrdinalIgnoreCase issue EF-243` | `string.StartsWith/Contains/EndsWith` with `StringComparison.Ordinal` or `OrdinalIgnoreCase` is not translated. | 9 | -| [EF-245](https://jira.mongodb.org/browse/EF-245) | `String.Join issue EF-245` | `String.Join(separator, source.Select(...))` is not translated. | 5 | -| [EF-246](https://jira.mongodb.org/browse/EF-246) | `DateTime subtraction issue EF-246` | `(dateA - dateB).TotalDays / TotalHours / TotalSeconds / TotalMilliseconds` is not translated. | 1 | -| [EF-247](https://jira.mongodb.org/browse/EF-247) | `Regex with non-constant pattern issue EF-247` | `Regex.IsMatch` with a non-constant pattern is not translated. | 1 | -| [EF-248](https://jira.mongodb.org/browse/EF-248) | `Translate String.FirstOrDefault and String.LastOrDefault issue EF-248` | `String.FirstOrDefault()` / `LastOrDefault()` (LINQ-on-string) is not translated. | 2 | -| [EF-249](https://jira.mongodb.org/browse/EF-249) | `checked issue EF-249` | `checked { ... }` arithmetic is not honored — the `Checked_context_with_arithmetic_does_not_fail` test sees a different exception than EF expects. | 1 | -| [EF-252](https://jira.mongodb.org/browse/EF-252) | `Concurrency detector tests broken EF-252` | `Throws_on_concurrent_query_first/list` — the concurrency detector does not fire as the EF base test expects. | 2 | -| [EF-253](https://jira.mongodb.org/browse/EF-253) | `Multiple ordering issue EF-253` | `OrderBy(x).ThenBy(x)` on the same column with different directions does not emit the expected MQL. | 1 | -| [EF-371](https://jira.mongodb.org/browse/EF-371) | `returns wrong data (0 rows instead of 6) EF-371` | A self-referencing two-hop reference navigation (`e.Manager.Manager`) collapses to a single join and hop 2 degrades to an inner `$unwind`, so the query returns 0 rows instead of 6. Baselined green on EF10 by asserting the wrong-data failure; the EF8/EF9 arm is a translation failure tagged EF-X020. | 1 | - -## MongoDB C# Driver tickets — `CSHARP-NNNN` - -| Ticket | Comment subject | Description | Count | -| --- | --- | --- | --- | -| [CSHARP-5296](https://jira.mongodb.org/browse/CSHARP-5296) | `DateTimeOffset issue CSHARP-5296` | Driver-level: `DateTimeOffset.Now / UtcNow` component access (`.Year`, `.Hour`, etc.) is not translated by the LINQ provider. | 2 | -| [CSHARP-5836](https://jira.mongodb.org/browse/CSHARP-5836) | `Reverse not supported CSHARP-5836` | Driver-level: `Queryable.Reverse()` is not implemented in the driver's LINQ provider. | 14 | - -## Upstream EF Core tickets — `dotnet/efcore#NNNNN` - -| Ticket | Comment subject | Description | Count | -| --- | --- | --- | --- | -| [dotnet/efcore#36412](https://github.com/dotnet/efcore/issues/36412) | `EF upstream issue--see https://github.com/dotnet/efcore/issues/36412` | Upstream EF Core test bug — provider-side override compensates while the upstream is fixed. | 1 | - -Two further GitHub references appear in the codebase but are not `// Fails:`-tagged (they are TODOs / notes, not failure annotations): [dotnet/efcore#36488](https://github.com/dotnet/efcore/issues/36488) (`NorthwindSetOperationsQueryMongoTest.cs:149`, upstream test has a bug), [dotnet/efcore#36521](https://github.com/dotnet/efcore/issues/36521) (`MongoApiConsistencyTest.cs:91`), [dotnet/efcore#36413](https://github.com/dotnet/efcore/issues/36413) (`Utilities/TestMqlLoggerFactory.cs:6`). - ---- - -## Failure modes lacking a Jira ticket — proposed new tickets - -These entries appear in `// Fails:` comments without an `EF-` or `CSHARP-` reference, or in test bodies as un-commented failure assertions. Each entry is assigned a **temporary** ticket id of the form `EF-X###` to be replaced with a real Jira number once filed; the `X` makes it obvious in `grep` results that the id is a placeholder. - -| Temp ticket | Subject | Count | -| --- | --- | --- | -| EF-X001 | Sub-query selection across DbSets is not translated | 144 | -| EF-X002 | Provider throws a different exception than the EF translation-failure message | 46 | -| EF-X003 | Driver-level feature gaps surfaced as test failures | 19 | -| EF-X004 | Float `Sum`/`Average` truncation (likely duplicate of EF-228) | 1 | -| EF-X005 | BSON document missing nested required reference (AdHoc JSON) | 2 | -| EF-X006 | MongoDB `DateTimeKind` round-trip handling | 1 | -| EF-X007 | Views / `HasDefiningQuery` semantics for MongoDB collections | 2 | -| EF-X008 | No support for nested JSON in AdHoc JSON tests | 2 | -| EF-X009 | Single uncategorized failure — needs triage | 1 | -| EF-X010 | Provider-specific Include error message differs from EF baseline | 4 | -| EF-X011 | ~~Compiled query with non-query operator — wrong exception~~ — fixed by EF-233 | 0 | -| EF-X012 | `OfType` translation unsupported | 2 | -| EF-X013 | MongoDB has no `$xor` operator (`Where_bitwise_xor`) | 1 | -| EF-X014 | Server-side projection conflict with cast-to-nullable | 1 | -| EF-X015 | Sub-second `DateTime` component translation (nanosecond/microsecond) | 1 | -| EF-X016 | GroupJoin shapes not translated | 9 | -| EF-X017 | Join shapes not translated | 5 | -| EF-X018 | RightJoin not supported | 1 | -| EF-X019 | Include on keyless entity not supported (no primary key for $lookup join) | 2 | -| EF-X020 | Cross-collection Include/join/navigation not translated on EF8/EF9 (works on EF10) | 176 | -| EF-X021 | Filtered Include / query filter on cross-collection target not translated | 0 | -| EF-X022 | Join/GroupJoin whose inner source is a filtered or ordered sub-query is rejected | 18 | -| EF-X024 | `Where`/entity materialization over a flattened multi-join chain onto ambiguous same-navigation or navigation-less siblings is not translated | 2 | -| EF-X016 | Bulk `ExecuteUpdate`/`ExecuteDelete` source restricted to a single collection scoped by `Where` | 47 | - -### EF-X001 — Sub-query selection across DbSets is not translated -Comment patterns: `// Fails: Subquery selection EF-X001`, `// Fails: Subqueries not supported EF-X001`, `// Fails: No subquery support EF-X001`. -Test-body patterns: `AssertTranslationFailed(() => base.X(...))`, `AssertNoMultiCollectionQuerySupport(() => base.X(...))`. -Affected: ~140 tests across `NorthwindAggregateOperators`, `NorthwindMiscellaneous`, `NorthwindWhere`, `NorthwindNavigations`, `NorthwindSetOperations`, `NorthwindJoin`, etc. Many overlap with [EF-216](https://jira.mongodb.org/browse/EF-216); a separate ticket lets cross-collection subquery support be tracked independently from raw navigation access. - -### EF-X002 — Provider throws a different exception than the EF translation-failure message -Comment patterns: `// Fails: Not throwing expected translation failed exception from EF, but still throws EF-X002`, `// Fails: Not throwing expected translation failed exception from EF. EF-X002`, `// Fails: Does not throw expected unable to translate exception EF-X002`, `// Fails: Does not use translation failed message EF-X002`, `// Fails: Throws different exception, but still throws EF-X002`. -Affected: ~36 tests. EF's base tests expect `InvalidOperationException` with EF's "could not be translated" message; the provider currently throws `ExpressionNotSupportedException` / `MongoCommandException` / `NotSupportedException` instead. Aligning the messages would let EF-level diagnostics work without provider-specific overrides. Two of these are bulk-update shapes in `NorthwindBulkUpdatesMongoTest` — `Update_Concat_set_constant` / `Update_Union_set_constant`, asserted with `Assert.ThrowsAnyAsync`. These are a **test-harness/transaction artifact rather than a bulk-path defect**: the conformance asserter's before/after snapshot query mirrors the `Concat`/`Union` source as `$unionWith`, which the server forbids inside the rollback transaction the fixture enlists (`MongoCommandException: Stage not supported inside of a multi-document transaction: $unionWith`) — so it fails before the provider's own clean bulk rejection. (The two GroupBy bulk shapes that previously sat here — `Delete_Where_predicate_with_GroupBy_aggregate`, `Delete_GroupBy_Where_Select_2` — now reject with the canonical translation-failure message via the bulk translation guard and are tracked under EF-X016.) Three further bulk shapes sit here for the same reason — `Update_with_join_set_constant` (EF9/EF10) and the EF10-only `Update_with_LeftJoin` / `Update_with_LeftJoin_via_flattened_GroupJoin`, asserted with `Assert.ThrowsAnyAsync`: these `Join`/`LeftJoin`/`GroupJoin` sources became EF-translatable with the cross-collection-join work (EF-117), so the asserter's before/after snapshot reads the join (whose inner is a filtered subquery, not a bare collection) and the driver's LINQ provider throws `ExpressionNotSupportedException` before the bulk path's own rejection runs. - -### EF-X003 — Driver-level feature gaps surfaced as test failures -Comment patterns: `// Fails: Unsupported by driver EF-X003`, `// Fails: Reverse not supported by driver EF-X003`, `// Fails: Limited support on client evaluation EF-X003`. -Affected: ~17 tests. These are MongoDB C# Driver gaps (the driver does not implement a particular LINQ-to-MQL translation). Many likely fold into existing `CSHARP-*` tickets; a single umbrella issue would let the provider track its dependency on driver work. - -### EF-X004 — Float `Sum`/`Average` truncation (likely duplicate of EF-228) -Comment pattern: `// Fails: Truncation resulted in data loss EF-X004`. -Affected: 1 test (`NorthwindAggregateOperatorsQueryMongoTest.cs`). Almost certainly the same failure mode as [EF-228](https://jira.mongodb.org/browse/EF-228); recommend re-tagging with `EF-228` and dropping this ticket once confirmed. - -### EF-X005 — BSON document missing nested required reference (AdHoc JSON) -Comment patterns: `// Fails: NestedRequiredReference is null in BsonDocument for entity id=6 EF-X005`, `// Fails: Entity id=5 has no RequiredReference field EF-X005`. -Affected: 2 tests in `AdHocJsonQueryMongoTest.cs`. Same family as [EF-164](https://jira.mongodb.org/browse/EF-164) but more specific — failure occurs when an owned navigation (rather than a scalar) is missing from the seeded BSON. - -### EF-X006 — MongoDB `DateTimeKind` round-trip handling -Comment pattern: `// Fails: MongoDB DateTimeKind handling EF-X006`. -Affected: 1 test. BSON cannot represent `DateTimeKind.Unspecified`, so the provider normalizes to UTC; tests that compare original vs round-tripped `DateTime.Kind` diverge. - -### EF-X007 — Views / `HasDefiningQuery` semantics for MongoDB collections -Comment patterns: `// Fails: Views are not supported, so this returns all entities from mapped collection. EF-X007`, `// Fails: Defining queries are not supported. EF-X007`. -Affected: 2 tests in `NorthwindKeylessEntitiesQueryMongoTest.cs`. The EF "view" / `HasDefiningQuery` notion doesn't map onto MongoDB; the provider returns the full collection instead of the view-filtered subset. - -### EF-X008 — No support for nested JSON in AdHoc JSON tests -Comment pattern: `// Fails: No support for nested JSON EF-X008`. -Affected: 2 tests in `AdHocJsonQueryMongoTest.cs`. The provider's JSON-column emulation does not nest deeply enough for these EF Core AdHoc cases. - -### EF-X009 — Single uncategorized failure — needs triage -Comment pattern: `// Fails: Unknown reasons EF-X009`. -Affected: 1 test. Author was unsure of root cause when adding the override. - -### EF-X010 — Provider-specific Include error message differs from EF baseline -Pattern: `Include_collection_with_client_filter` across all four Include variants asserts -`Assert.Contains("ExpressionNotSupportedException", (await Assert.ThrowsAsync(...)).Message)` -— the same shape as the 28 existing `EF-X002` sites. The base test expects EF's own -`InvalidOperationException` translation-failed message for a client-evaluated member on an -Include-d query; the provider instead throws the driver's `ExpressionNotSupportedException`, -so `Assert.ThrowsAsync` inside the base method itself fails with a -`ThrowsException` reporting the actual type. The mechanism is identical to `EF-X002` — see that -entry for the general case; this one's trigger is specifically a client-evaluated member on an -`Include`, not a plain unsupported operator. -Affected: 4 tests (`NorthwindEFPropertyIncludeQueryMongoTest.cs`, `NorthwindIncludeNoTrackingQueryMongoTest.cs`, `NorthwindIncludeQueryMongoTest.cs`, `NorthwindStringIncludeQueryMongoTest.cs`). - -Until this was fixed, each of these four suites shadowed the upstream `AssertTranslationFailed` -helper with a bare `try { await query(); } catch { return; }`, which — unlike the -`Assert.ThrowsAsync` pattern above — swallowed *any* exception, including an -assertion failure from wrong data returned by a query that didn't fail to translate at all. A -full audit of all 234 call sites of the shadowed helper (across the four suites) found this one -test as the only case actually masking a real failure; the other 230 call sites already threw a -genuine non-assertion exception and are unaffected. The shadowed helpers now delegate to a -corrected `MongoAssert.AssertTranslationFailed` (rejects any `XunitException`, so wrong data -still fails the test) instead of duplicating the swallow-everything logic in each file. - -### EF-X011 — Compiled query with non-query operator — wrong exception — **fixed by [EF-233](https://jira.mongodb.org/browse/EF-233)** -Comment pattern (historical): `// Fails: Compiled query with non-query operator issue EF-X011`. -Root cause: `MongoQueryableMethodTranslatingExpressionVisitor` captured the "final" query-chain expression in its generic `Visit` override (`_finalExpression ??= expression`), which fires on whatever node is visited first. For a compiled query ending in a non-query operator (e.g. `.Count() == 1`), the wrapping `BinaryExpression` was visited before the actual `Count()` call, so the captured expression included the trailing comparison and downstream translation broke (`ArgumentException("No ultimate source found")`, or the reported `InvalidCastException` for other shapes). Fixed by moving the capture into `VisitMethodCall` (`_finalExpression ??= methodCallExpression`), which only ever fires on the true queryable-chain nodes. `NorthwindCompiledQueryMongoTest.Compiled_query_when_does_not_end_in_query_operator` now calls `base.*` directly and asserts the correct MQL baseline. - -One incidental side effect: `NorthwindCompiledQueryMongoTest.Multiple_queries` (two separately-compiled queries against different `DbSet`s reusing translation state — a pre-existing [EF-216](https://jira.mongodb.org/browse/EF-216) cross-DbSet limitation) now surfaces as an `ArgumentException` from the driver-LINQ rebuild instead of the provider's own "Unsupported cross-DbSet query" guard. The test was updated to assert the new exception type; per this repo's versioning conventions, the exception type for an already-unsupported shape is not part of the public contract. - -### EF-X012 — `OfType` translation unsupported -Comment pattern: `// Fails: OfType translation EF-X012`. -Affected: 2 tests in `NorthwindAggregateOperatorsQueryMongoTest.cs` (`OfType_Select`, `OfType_Select_OfType_Select`). `Queryable.OfType()` is not translated; the failure surfaces as a generic `AssertTranslationFailed`. - -### EF-X013 — MongoDB has no `$xor` operator -Comment pattern: `// Fails: MongoDB does not have an xor operator EF-X013`. -Affected: 1 test (`NorthwindWhereQueryMongoTest.Where_bitwise_xor`). The provider throws `ExpressionNotSupportedException` with message `"because MongoDB does not have a boolean $xor operator"`. - -### EF-X014 — Server-side projection conflict with cast-to-nullable -Comment pattern: `// Fails: Server-side projection conflict with cast-to-nullable EF-X014`. -Affected: 1 test (`NorthwindSelectQueryMongoTest.Select_bool_closure_with_order_by_property_with_cast_to_nullable`). Server rejects the generated `$project` stage with `"Cannot do exclusion on field _key1 in inclusion projection"` — likely a translator bug where the projection stage mixes inclusion and exclusion. - -### EF-X015 — Sub-second `DateTime` component translation (nanosecond/microsecond) -Comment pattern: `// Fails: Sub-second DateTime component translation EF-X015`. -Affected: 1 test (`NorthwindMiscellaneousQueryMongoTest.Where_nanosecond_and_microsecond_component`). `DateTime.Nanosecond` and `DateTime.Microsecond` (added in .NET 7) are not translated; the provider throws `ExpressionNotSupportedException`. - -### EF-X016 — Bulk `ExecuteUpdate`/`ExecuteDelete` source restricted to a single collection scoped by `Where` -Comment pattern: `// Fails: ExecuteUpdate/ExecuteDelete source restricted to a Where predicate; unsupported EF-X016`. -Test-body pattern: `AssertTranslationFailed(() => base.X(...))`. -Affected: 47 tests in `NorthwindBulkUpdatesMongoTest.cs`. By design, the provider supports bulk `ExecuteUpdate`/`ExecuteDelete` only against a single collection scoped by `Where`, with constant / parameter / self-referencing scalar setters (see README "What is supported" and EF-107). The EF conformance suite exercises shapes outside that subset — joins / correlated subqueries, set operations (`Except`/`Intersect`), `GroupBy`, `SelectMany`, cross-document navigation predicates, non-entity projection sources, and multiple-collection updates — and the provider rejects them during translation with an `InvalidOperationException` whose message reports the LINQ expression could not be translated. This is a documented limitation rather than a bug to fix; the ticket tracks the boundary so any future expansion of the supported subset (or a behavior change) is detected. Most shapes are rejected by compile-time bulk-source validation; a few (e.g. a `GroupBy` subquery inside the `Where` predicate) only fail when the filter is built at execution time, where a translation guard (`TranslateBulkOrThrow` in `MongoShapedQueryCompilingExpressionVisitor`) converts the raw driver/expression exception into the same canonical non-query translation failure. A handful of sibling shapes surface differently and are tagged separately: cross-DbSet `GroupBy` rejections under [EF-216](https://jira.mongodb.org/browse/EF-216), and the `$unionWith`-in-transaction `Concat`/`Union` cases under `EF-X002`. - -**Remaining unsupported shapes — grouped by root cause.** The unifying cause: two-phase execution only works when the *source query is translatable as a read*, because phase 1 must be able to project the target `_id`s. Every group below fails because the source shape itself cannot be translated — so neither phase 1 nor a single-command filter can be built. (Delete has the analogous cases; the `Update_*` methods are enumerated here since they are the larger set.) - -- **Joins / correlated subqueries** — `RightJoin`, cross-apply, outer-apply, and the cross-/two-collection join shapes the read path cannot translate, so phase 1 cannot project `_id`s and the source is rejected at translation with EF's canonical message. Unblocking requires net-new read-side join translation (ultimately a driver `$lookup`/`$group` + `$merge` capability) and is out of scope for EF-107. Update cases: `Update_Where_Join_set_property_from_joined_single_result_scalar`, `Update_Where_Join_set_property_from_joined_single_result_table`, `Update_Where_Join_set_property_from_joined_table`, `Update_with_two_inner_joins`, `Update_with_left_join_set_constant`, `Update_with_cross_join_set_constant`, `Update_with_cross_apply_set_constant`, `Update_with_outer_apply_set_constant`, `Update_with_cross_join_cross_apply_set_constant`, `Update_with_cross_join_left_join_set_constant`, `Update_with_cross_join_outer_apply_set_constant`, `Update_with_PK_pushdown_and_join_and_multiple_setters`, plus EF10-only `Update_with_RightJoin`. (The simple single-key `Join` / `LeftJoin` / `GroupJoin` shapes — `Update_with_join_set_constant` and EF10-only `Update_with_LeftJoin` / `Update_with_LeftJoin_via_flattened_GroupJoin` — became read-translatable with the cross-collection-join work (EF-117), so they no longer fail at bulk translation; the conformance asserter's snapshot read of the filtered-inner join now throws a driver non-translation exception instead, tracked under EF-X002.) -- **Set operations — `Except` / `Intersect`** — MongoDB has no cross-collection intersect/except (only `$unionWith`), so the source cannot be expressed and is rejected at translation. Update cases: `Update_Except_set_constant`, `Update_Intersect_set_constant`. -- **`SelectMany`** — the read path has no cross-document flattening, so the source cannot be translated. Update cases: `Update_Where_SelectMany_set_null`, `Update_Where_SelectMany_subquery_set_null`. -- **Cross-document navigation predicates** — predicates traversing navigations across collections aren't translatable. Update cases: `Update_Where_using_navigation_2_set_constant`, `Update_Where_using_navigation_set_null`. -- **Multiple-collection update** — a relational table-sharing concept with no document-model meaning; rejected by design regardless of phase strategy. Update case: `Update_multiple_tables_throws`. - -**Related shapes that surface as a different failure mode** (cross-referenced, not under the canonical translation-failure pattern above): - -- **`Concat` / `Union` (`EF-X002`)** — these lower to `$unionWith`, which the server forbids inside the multi-document transaction the conformance asserter (and the two-phase path) run in, so they throw a runtime `MongoCommandException` *before* the provider's clean bulk rejection. Update cases: `Update_Concat_set_constant`, `Update_Union_set_constant`. Unblocking requires the server to allow `$unionWith` inside a transaction. -- **`GroupBy`-scoped (`EF-216`)** — `GroupBy` isn't translatable; these surface as the provider's `Unsupported cross-DbSet query` rejection and are asserted via `AssertNoMultiCollectionQuerySupport` rather than a `// Fails:` `AssertTranslationFailed`. Update cases: `Update_Where_GroupBy_First_set_constant`, `Update_Where_GroupBy_First_set_constant_2`, `Update_Where_GroupBy_First_set_constant_3`, `Update_Where_GroupBy_aggregate_set_constant`. - -**Two-phase bulk support (EF-107):** `OrderBy`/`Skip`/`Take`/`Distinct`-scoped bulk delete and update are now supported via a transactional two-phase execution strategy: phase 1 projects the matching `_id` values (including composite-key entities) into a temporary in-memory set; phase 2 issues the actual `deleteMany`/`updateMany` scoped to that `_id` set. The corresponding conformance tests (`Delete_Where_OrderBy*`, `Delete_Where_Skip*`, `Delete_Where_Take*`, `Delete_Where_Distinct`, `Update_Where_OrderBy*`, `Update_Where_Skip*`, `Update_Where_Take*`, `Update_Where_Distinct_set_constant`, `Delete_Where_Skip_Take_Skip_Take_causing_subquery`, `Update_Where_OrderBy_Skip_Take_Skip_Take_set_constant`) have been promoted from `AssertTranslationFailed` to `base.*` and are no longer tracked under this ticket. - -### EF-X016 — GroupJoin shapes not translated -Comment pattern: `// Fails: GroupJoin shape not translated EF-X016`. -Affected: 9 tests in `NorthwindJoinQueryMongoTest.cs` — `GroupJoin_aggregate_anonymous_key_selectors` (+`2`, +`_one_argument`, +`_nested`), `GroupJoin_DefaultIfEmpty_multiple`, `GroupJoin_DefaultIfEmpty2`, `GroupJoin_subquery_projection_outer_mixed`, `GroupJoin_on_true_equal_true` (EF9 only), `Unflattened_GroupJoin_composed_2`. These are `GroupJoin` shapes the flatten-to-`$lookup` pipeline does not yet handle (aggregate / anonymous or nested key selectors, multiple `DefaultIfEmpty`, on-`true == true`, mixed projection). Distinct from the `GroupJoin` *subquery* shapes, which are tracked under [EF-X001](#ef-x001--sub-query-selection-across-dbsets-is-not-translated), and from cross-collection `Include`, which is [EF-117](https://jira.mongodb.org/browse/EF-117). - -### EF-X017 — Join shapes not translated -Comment pattern: `// Fails: Join shape not translated EF-X017`. -Affected: 5 tests — `Join_composite_key`, `Join_complex_condition`, `Join_with_key_selectors_being_nested_anonymous_objects`, `Join_local_collection_int_closure_is_cached_correctly` (all in `NorthwindJoinQueryMongoTest.cs`) and `Join_with_default_if_empty_on_both_sources` (in `NorthwindMiscellaneousQueryMongoTest.cs`). The provider translates simple single-key `Join`/`GroupJoin` to `$lookup`, but composite keys, complex (non-equality) conditions, nested-anonymous key selectors, joins against a local in-memory collection, and `DefaultIfEmpty` on both sources are not yet supported. - -### EF-X018 — RightJoin not supported -Comment pattern: `// Fails: RightJoin not supported EF-X018`. -Affected: 1 test (`NorthwindJoinQueryMongoTest.RightJoin`, EF10+). `Queryable.RightJoin` (the EF9+/EF10 operator) is not translated; the provider fails translation rather than emitting the reversed `$lookup` pipeline. - -### EF-X019 — Include on keyless entity not supported (no primary key for $lookup join) -Comment pattern: `// Fails: Include on keyless entity not supported (no primary key for $lookup join) EF-X019`. -Affected: 2 tests (`NorthwindKeylessEntitiesQueryMongoTest.KeylessEntity_with_included_nav`, `KeylessEntity_with_included_navs_multi_level`). A keyless entity has no primary key, so the cross-collection `$lookup` join-key cannot be resolved and keyless entities are never tracked (no `InternalEntityEntry` is emitted into the shaper). The provider now detects this in `MongoProjectionBindingRemovingExpressionVisitor.AddInclude` and throws the standard `CoreStrings.TranslationFailed` (translation-failure) message instead of the internal `Sequence contains no matching element` error. Distinct from cross-collection `Include` on keyed entities, which is implemented (formerly [EF-117](https://jira.mongodb.org/browse/EF-117)). - -### EF-X020 — Cross-collection Include/join/navigation not translated on EF8/EF9 -Comment pattern: `// Fails: Cross-collection Include/join not translated on EF8/EF9 EF-X020`. -Test-body pattern: the override is wrapped in `#if EF8 || EF9` / `#else`. The `#if EF8 || EF9` branch asserts the translation failure (`AssertTranslationFailed(() => base.X(...))`); the `#else` branch keeps the working EF10 baseline (the real `base` call plus its `AssertMql(...)`). -Affected: 176 tests across `NorthwindEFPropertyIncludeQueryMongoTest`, `NorthwindStringIncludeQueryMongoTest`, `NorthwindIncludeQueryMongoTest`, `NorthwindIncludeNoTrackingQueryMongoTest`, `NorthwindNavigationsQueryMongoTest`, `NorthwindMiscellaneousQueryMongoTest`, `NorthwindJoinQueryMongoTest`, `NorthwindAggregateOperatorsQueryMongoTest`, `NorthwindAsNoTrackingQueryMongoTest`, `NorthwindKeylessEntitiesQueryMongoTest`, `NorthwindSelectQueryMongoTest`, `NorthwindSetOperationsQueryMongoTest`, `NorthwindWhereQueryMongoTest`, and `BuiltInDataTypesMongoTest`. These are the cross-collection Include/`ThenInclude`/join/navigation shapes implemented for the EF10-targeted query pipeline. On EF8/EF9 the upstream nav-expansion / query pipeline produces a different expression shape (e.g. an extra `.OrderBy(o => o.OrderID)` injected during navigation expansion) that the EF10-targeted translator does not handle, so translation fails with EF Core's `InvalidOperationException` "could not be translated" (a few also throw the provider's `ExpressionNotSupportedException`); the same query translates and runs on EF10. The provider's local `AssertTranslationFailed` helper swallows whichever exception is thrown, so both shapes are covered. Four `Include_reference_dependent_already_tracked` overrides (in the four Include suites) emit MQL from a first principal query *before* the Include sub-query fails to translate, so their `#if EF8 || EF9` branch asserts only the translation failure and omits the empty `AssertMql()`. `BuiltInDataTypesMongoTest.Can_read_back_bool_mapped_as_int_through_navigation` is split three ways (a nested `#if EF9` inside the file's `#if !EF8` branch, plus the `#else` EF8 branch) because that file uses async signatures on EF9/EF10 and sync `void` signatures on EF8. - -### EF-X022 — Join/GroupJoin whose inner source is a filtered or ordered sub-query is rejected -Comment pattern: `// Fails: Join/GroupJoin inner sub-query (filtered/ordered) not supported EF-X022`. -Test-body patterns: `Assert.ThrowsAsync` asserting -`"Expression not supported"`, or `Assert.ThrowsAnyAsync` where the EF8/EF9 failure mode differs. - -The provider lowers a single-key `Join`/`GroupJoin` to `$lookup`. When the **inner** source is not a bare -collection but a sub-query — `Orders.Where(...)`, `Orders.OrderBy(...)`, or a dependent-side query filter -that lowers to the same `Where` — the driver's LINQ provider rejects the whole expression with -`ExpressionNotSupportedException: ... because expression must be a MongoDB IQueryable against a collection`. -This is an **intentional rejection**: these shapes were previously folded into the correlated `$lookup` -sub-pipeline, which is not a faithful translation — the mis-folding formerly tracked as `CSHARP-6017`. -Failing loudly is preferred over returning wrong results, so this is a documented boundary rather than a -regression to fix in the test suite. - -Affected: 18 overrides, all asserting the rejection (none are `Skip`ped). - -- `NorthwindJoinQueryMongoTest` — `Join_customers_orders_with_subquery`, - `Join_customers_orders_with_subquery_predicate`, `Join_customers_orders_with_subquery_anonymous_property_method`, - `GroupJoin_customers_employees_subquery_shadow`, `GroupJoin_SelectMany_subquery_with_filter`, plus - EF10-only `GroupJoin_DefaultIfEmpty2` and `GroupJoin_SelectMany_subquery_with_filter_and_DefaultIfEmpty`. -- `NorthwindMiscellaneousQueryMongoTest` — `OrderBy_Join`, `Join_take_count_works`. -- `NorthwindQueryFiltersQueryMongoTest` (EF10-only) — `Entity_Equality`, `Included_many_to_one_query`, - `Included_many_to_one_query2`, where the *query filter* on the `$lookup` target is what makes the inner - a filtered sub-query. -- **Former `CSHARP-6017` skips**, now converted to the same assertion style rather than - `[ConditionalTheory(Skip = ...)]` — `Join_customers_orders_with_subquery_with_take`, - `Join_customers_orders_with_subquery_predicate_with_take`, - `Join_customers_orders_with_subquery_anonymous_property_method_with_take`, `GroupJoin_simple_subquery`, - `GroupJoin_Subquery_with_Take_Then_SelectMany_Where`, `GroupJoin_customers_employees_subquery_shadow_take`. - These were skipped while the driver silently mis-folded the `Take`/sub-query inner and returned wrong - results; the driver now rejects them, so they assert the rejection and no longer need a skip. - -Interaction with [EF-352](https://jira.mongodb.org/browse/EF-352): that fix (shadow/scalar `EF.Property` -read on a joined entity in a projection) re-enabled -`Join_customers_orders_with_subquery_anonymous_property_method` against driver 3.10, but the query's inner -is an ordered sub-query, so 3.11 rejects it and the override now asserts the rejection instead. The -underlying EF-352 fix is still covered functionally by -`ShadowPropertyJoinProjectionTests` and `ShadowPropertyFlatJoinProjectionTests`; the first of those had the -`orderby` dropped from its join inner (incidental to what it asserts) so it keeps exercising the non-root -joined-entity shaper on a shape the driver still supports. - -The EF10-only entries carry a second ticket for their EF8/EF9 failure mode (`EF-X016`, `EF-X001`, `EF-202`, -`EF-216`), because on those versions the query fails earlier in nav-expansion and logs no MQL at all — -hence the `#if EF8 || EF9` split on the `AssertMql` baseline. - -Lifting this ticket requires translating the inner sub-query into the `$lookup` sub-pipeline correctly -(correlated `let`/`pipeline` form), which overlaps with [EF-X021](#ef-x021--filtered-include--query-filter-on-cross-collection-target-not-translated). - -### EF-371 — self-referencing two-hop reference navigation collapses to one join - -**History (kept deliberately — the original diagnosis was wrong).** This entry replaces -"EF-216 — wrong-data on EF10 (cross-collection navigation), unsupported on EF8/EF9", which covered -**five** `NorthwindNavigationsQueryMongoTest` methods skipped with -`[ConditionalTheory(Skip = "EF-216: multi-hop cross-collection navigation returns wrong data")]`: -`Include_with_multiple_optional_navigations`, `Multiple_include_with_multiple_optional_navigations`, -`Navigation_from_join_clause_inside_contains`, `Navigation_inside_contains_nested` and -`Select_Where_Navigation_Null_Deep`. They were believed to share one root cause — "compound multi-hop -navigation lowering". They did not. - -**All five are now fixed.** Four via EF-369 / EF-370 (see -`docs/superpowers/specs/2026-08-03-required-nav-unwind-semantics-design.md`): the composed predicate / -`Contains` filter was being *discarded* when a multi-join Include chain was flattened to root-level -`_lookup_