CBL-7986: Remove "flags&1=0" in the result of QueryTranslator whereve…

C/tests/c4QueryTest.cc

@@ -1254,7 +1254,7 @@ N_WAY_TEST_CASE_METHOD(C4QueryTest, "C4Query refresh", "[Query][C][!throws]") {
     string explanationString = toString(c4query_explain(query));
     INFO("Explanation = " << explanationString);
     CHECK(litecore::hasPrefix(explanationString, "SELECT _doc.key FROM kv_default AS _doc WHERE fl_value(_doc.body, "
-                                                 "'contact.address.state') = 'CA' AND (_doc.flags & 1 = 0)"));
+                                                 "'contact.address.state') = 'CA'"));
 
     auto e = c4query_run(query, kC4SliceNull, ERROR_INFO(error));
     REQUIRE(e);

LiteCore/Query/Translator/ExprNodes.cc

@@ -19,6 +19,8 @@
 #include "TranslatorTables.hh"
 #include "TranslatorUtils.hh"
 #include <iostream>
+#include <vector>
+#include <ranges>
 
 namespace litecore::qt {
     using namespace fleece;
@@ -445,6 +447,21 @@ namespace litecore::qt {
 
     OpFlags OpNode::opFlags() const { return _op.flags; }
 
+    ExprNode* OpNode::swapArg(size_t i, ExprNode* newArg) {
+        auto count = argCount();
+        require(i < count, "OpNode::swapArg: is out of bounds");
+        std::vector<ExprNode*> nodes;
+        for ( size_t q = 0; q < i; ++q ) nodes.push_back(_operands.pop_front());
+        auto ith = _operands.pop_front();
+        ith->setParent(nullptr);
+        if ( newArg ) {
+            newArg->setParent(this);
+            _operands.push_front(newArg);
+        }
+        for ( auto expr : nodes | std::views::reverse ) _operands.push_front(expr);
+        return ith;
+    }
+
     void OpNode::visitChildren(ChildVisitor const& visitor) { visitor(_operands); }
 
 #if DEBUG

LiteCore/Query/Translator/ExprNodes.hh

@@ -202,6 +202,10 @@ namespace litecore::qt {
 
         void addArg(ExprNode* node) { addChild(_operands, node); }
 
+        size_t argCount() const { return _operands.size(); }
+
+        ExprNode* swapArg(size_t i, ExprNode* C4NULLABLE newArg);
+
         OpFlags opFlags() const override;
         void    visitChildren(ChildVisitor const& visitor) override;
         void    writeSQL(SQLWriter&) const override;

LiteCore/Query/Translator/Node.cc

@@ -39,6 +39,11 @@ namespace litecore::qt {
         _parent = p;
     }
 
+    void Node::setNext(Node* n) {
+        DebugAssert(!_next || !n);
+        _next = n;
+    }
+
     void Node::postprocess(ParseContext& ctx) {
         visitChildren(ChildVisitor{[&](Node& child) { child.postprocess(ctx); }});
     }

LiteCore/Query/Translator/Node.hh

@@ -79,6 +79,7 @@ namespace litecore::qt {
 #endif
         std::function<void(SourceNode*, ParseContext&)> assignTableNameToMainSource;
         std::function<string_view()>                    translatorDefaultCollection;
+        std::function<bool(string_view collection)>     isDeletedDocsFullyTracked;
     };
 
     /** State used during parsing, passed down through the recursive descent. */
@@ -129,9 +130,10 @@ namespace litecore::qt {
         void            operator delete(void* C4NULLABLE ptr, ParseContext& ctx) noexcept;
 
         /// The node's parent in the parse tree.
-        Node const* parent() const noexcept { return _parent; }
+        Node const* C4NULLABLE parent() const noexcept { return _parent; }
 
         void setParent(Node* C4NULLABLE);
+        void setNext(Node* C4NULLABLE);
 
         /// Next sibling in list.
         /// @note Only some parents organize children into lists! Some parents use multiple lists!

LiteCore/Query/Translator/QueryTranslator.cc

@@ -51,6 +51,10 @@ namespace litecore {
             mutableThis->assignTableNameToSource(source, ctx);
         };
         root.translatorDefaultCollection = [&]() -> string_view { return _defaultCollectionName; };
+        root.isDeletedDocsFullyTracked   = [&](string_view collection) {
+            if ( collection.empty() ) collection = _defaultCollectionName;
+            return collection != "_default" || _delegate.isDeletedDocsFullyTracked();
+        };
         return root;
     }
 
@@ -140,9 +144,14 @@ namespace litecore {
             if ( name.empty() ) name = _defaultCollectionName;
             if ( !source->scope().empty() ) name = string(source->scope()) + "." + name;
 
-            DeletionStatus delStatus = source->usesDeletedDocs() ? kLiveAndDeletedDocs : kLiveDocs;
-            //FIXME: Support kDeletedDocs
-
+            DeletionStatus delStatus{kLiveDocs};
+            if ( source->onlyDeletedDocs() && ctx.delegate.isDeletedDocsFullyTracked(name) ) {
+                // WHERE clause guarantees only deleted docs match, and the delegate
+                // confirms that all deleted docs are in the dedicated kv_del_ table.
+                delStatus = kDeletedDocs;
+            } else if ( source->usesDeletedDocs() ) {
+                delStatus = kLiveAndDeletedDocs;
+            }
             tableName = _delegate.collectionTableName(name, delStatus);
             if ( name != _defaultCollectionName && !_delegate.tableExists(tableName) )
                 fail("no such collection \"%s\"", name.c_str());

LiteCore/Query/Translator/QueryTranslator.hh

@@ -49,6 +49,7 @@ namespace litecore {
             [[nodiscard]] virtual string collectionTableName(const string& collection, DeletionStatus) const    = 0;
             [[nodiscard]] virtual string FTSTableName(const string& onTable, const string& property) const      = 0;
             [[nodiscard]] virtual string unnestedTableName(const string& onTable, const string& property) const = 0;
+            [[nodiscard]] virtual bool   isDeletedDocsFullyTracked() const                                      = 0;
 #ifdef COUCHBASE_ENTERPRISE
             [[nodiscard]] virtual string predictiveTableName(const string& onTable, const string& property) const = 0;
             [[nodiscard]] virtual string vectorTableName(const string& collection, const std::string& property,

LiteCore/Query/Translator/SelectNodes.cc

@@ -351,7 +351,10 @@ namespace litecore::qt {
             for ( WhatNode* what : _what ) what->parseChildExprs(ctx);
 
             // Parse the WHERE clause:
-            if ( Value where = getCaseInsensitive(select, "WHERE") ) { setChild(_where, ExprNode::parse(where, ctx)); }
+            if ( Value where = getCaseInsensitive(select, "WHERE") ) {
+                auto nodeWhere = reduceDeleted(ExprNode::parse(where, ctx), ctx);
+                if ( nodeWhere ) setChild(_where, nodeWhere);
+            }
 
             if ( Value order = getCaseInsensitive(select, "ORDER_BY") ) {
                 for ( Value orderItem : requiredArray(order, "ORDER BY") ) {
@@ -434,7 +437,7 @@ namespace litecore::qt {
         for ( SourceNode* source : _sources ) {
             string_view coll{source->collection()};
             if ( coll.empty() ) coll = ctx.delegate.translatorDefaultCollection();
-            if ( !source->_usesDeleted && coll == "_default" && source->isCollection() ) {
+            if ( !source->_usesDeleted && source->isCollection() && !ctx.delegate.isDeletedDocsFullyTracked(coll) ) {
                 // The default collection may contain deleted documents in its main table,
                 // so if the query didn't ask for deleted docs, add a condition to the WHERE
                 // or ON clause that only passes live docs:
@@ -495,4 +498,78 @@ namespace litecore::qt {
         visitor(_sources)(_what)(_where)(_groupBy)(_having)(_orderBy)(_limit)(_offset);
     }
 
+    ExprNode* SelectNode::reduceDeleted(ExprNode* expr, ParseContext& ctx) {
+        if ( !expr ) return nullptr;
+
+        // Find all branches following "AND" ending at MetaNode of property "deleted"
+        // and store them in deleted. They are leaf nodes.
+        std::vector<MetaNode*> delMetas;
+        [&](ExprNode* root) {
+            auto markDeleted = [&](auto self, ExprNode* expr) -> void {
+                if ( auto meta = dynamic_cast<MetaNode*>(expr) ) {
+                    if ( meta->property() == MetaProperty::deleted ) {
+                        meta->source()->setOnlyDeleted();
+                        delMetas.push_back(meta);
+                    }
+                } else if ( auto op = dynamic_cast<OpNode*>(expr); op && op->op().name == "AND"_sl ) {
+                    op->visitChildren({[self](Node& node) {
+                        if ( auto* operand = dynamic_cast<ExprNode*>(&node) ) self(self, operand);
+                    }});
+                }
+            };
+            markDeleted(markDeleted, root);
+        }(expr);
+
+        ExprNode* ret = expr;
+        for ( auto del : delMetas ) {
+            if ( auto source = del->source();
+                 source && source->onlyDeletedDocs() && ctx.delegate.isDeletedDocsFullyTracked(source->collection()) ) {
+                // reduce only if the source collection has the deleted table complete.
+                if ( !del->parent() ) ret = nullptr;
+                else if ( auto parentOp = dynamic_cast<OpNode*>(const_cast<Node*>(del->parent()));
+                          parentOp->op().name == "AND"_sl ) {
+                    // Must enter here because of how "onlyDeleted" are marked.
+                    auto argCount = parentOp->argCount();
+                    auto childIdx = [](const OpNode* parent, const Node* child, size_t childCount) -> int {
+                        for ( int i = 0; i < childCount; ++i )
+                            if ( parent->operand(i) == child ) return i;
+                        return -1;
+                    };
+                    auto delIdx = childIdx(parentOp, del, argCount);
+                    require(delIdx >= 0, "Internal error, child index not found for OpNode");
+
+                    // Reducing parentOp
+                    ExprNode* reduced = nullptr;
+                    if ( argCount == 2 ) {
+                        // typical case. <expr> AND ['._deleted'] => <expr>.
+                        reduced = parentOp->operand((delIdx + 1) % 2);
+                    } else {
+                        // ['AND', A, B, TRUE, X ] => ['AND', A, B, X]
+                        parentOp->swapArg(delIdx, nullptr);
+                        reduced = parentOp;
+                    }
+                    if ( !parentOp->parent() ) {
+                        // The parent "AND" node is the root.
+                        ret = reduced;
+                    } else if ( auto gParentOp = dynamic_cast<OpNode*>(const_cast<Node*>(parentOp->parent()));
+                                // Must enter here because of how "onlyDeleted" are marked.
+                                gParentOp->op().name == "AND"_sl ) {
+                        auto parentIdx = childIdx(gParentOp, parentOp, gParentOp->argCount());
+                        require(parentIdx >= 0, "Internal error, child index not found for OpNode");
+
+                        // grand parent to adopt reduced as direct child
+                        // reduced is to be adopted by grand-parent as new child.
+                        reduced->setParent(nullptr);
+                        reduced->setNext(nullptr);
+                        gParentOp->swapArg(parentIdx, reduced);
+                    }
+                }
+            }
+        }
+        if ( ret != nullptr && ret != expr ) {
+            ret->setParent(nullptr);
+            ret->setNext(nullptr);
+        }
+        return ret;
+    }
 }  // namespace litecore::qt

LiteCore/Query/Translator/SelectNodes.hh

@@ -87,6 +87,8 @@ namespace litecore::qt {
 
         bool usesDeletedDocs() const { return _usesDeleted; }  ///< True if exprs refer to deleted docs
 
+        bool onlyDeletedDocs() const { return _onlyDeleted; }  ///< True if WHERE guarantees only deleted docs
+
         string_view asColumnName() const { return _columnName; }  ///< Name to use, if used as result column
 
         bool isJoin() const { return _join != JoinType::none; }  ///< True if this is a JOIN
@@ -118,6 +120,11 @@ namespace litecore::qt {
             _tableName   = string_view{};
         }
 
+        void setOnlyDeleted() {
+            _onlyDeleted = true;  // WHERE guarantees only deleted docs; can use kv_del_ directly
+            _tableName   = string_view{};
+        }
+
       private:
         string_view          _scope;                  // Scope name, or empty for default
         string_view          _collection;             // Collection name, or empty for default
@@ -127,6 +134,7 @@ namespace litecore::qt {
         ExprNode* C4NULLABLE _joinOn{};               // "ON ..." predicate
         Value                _tempOn;                 // Temporarily holds source of _joinOn
         bool                 _usesDeleted = false;    // True if exprs refer to deleted docs
+        bool                 _onlyDeleted = false;    // True if WHERE guarantees only deleted docs
         SourceType const     _type;
     };
 
@@ -188,13 +196,14 @@ namespace litecore::qt {
         void writeSQL(SQLWriter&) const override;
 
       private:
-        void   parse(Value, ParseContext&);
-        void   registerAlias(AliasedNode*, ParseContext&);
-        void   addSource(SourceNode*, ParseContext&);
-        void   addIndexes(ParseContext&);
-        void   addIndexForNode(IndexedNode*, ParseContext&);
-        string makeIndexAlias() const;
-        void   writeFTSColumns(SQLWriter&, fleece::delimiter&) const;
+        void                 parse(Value, ParseContext&);
+        void                 registerAlias(AliasedNode*, ParseContext&);
+        void                 addSource(SourceNode*, ParseContext&);
+        void                 addIndexes(ParseContext&);
+        void                 addIndexForNode(IndexedNode*, ParseContext&);
+        string               makeIndexAlias() const;
+        void                 writeFTSColumns(SQLWriter&, fleece::delimiter&) const;
+        ExprNode* C4NULLABLE reduceDeleted(ExprNode* expr, ParseContext& ctx);
 
         List<SourceNode>     _sources;                      // The sources (FROM exprs)
         List<WhatNode>       _what;                         // The WHAT expressions

LiteCore/Storage/DataFile.cc

@@ -317,7 +317,7 @@ namespace litecore {
         for ( auto& ks : _keyStores ) fn(*ks.second);
     }
 
-    bool DataFile::isDeletedTableComplete() {
+    bool DataFile::isDeletedTableComplete() const {
         Record rec = getKeyStore(kInfoKeyStoreName, KeyStore::noSequences).get(kMaxRowidWithDeletedInDefault);
         return rec.exists() && rec.bodyAsUInt() == 0;
     }

LiteCore/Storage/DataFile.hh

@@ -181,7 +181,7 @@ namespace litecore {
         /** Permanently deletes a KeyStore. */
         virtual void deleteKeyStore(const std::string& name) = 0;
 
-        bool isDeletedTableComplete();
+        bool isDeletedTableComplete() const;
 
         // Redeclare logging methods as public, so Database can use them
         bool willLog(LogLevel level = LogLevel::Info) const { return Logging::willLog(level); }

LiteCore/Storage/SQLiteDataFile.cc

@@ -914,6 +914,8 @@ namespace litecore {
         }
     }
 
+    bool SQLiteDataFile::isDeletedDocsFullyTracked() const { return isDeletedTableComplete(); }
+
 #ifdef COUCHBASE_ENTERPRISE
     string SQLiteDataFile::predictiveTableName(const string& onTable, const std::string& property) const {
         return auxiliaryTableName(onTable, KeyStore::kPredictSeparator, property);

LiteCore/Storage/SQLiteDataFile.hh

@@ -117,6 +117,7 @@ namespace litecore {
         static string auxiliaryTableName(const string& onTable, slice typeSeparator, const string& property);
         std::string   FTSTableName(const string& collection, const std::string& property) const override;
         std::string   unnestedTableName(const string& collection, const std::string& property) const override;
+        bool          isDeletedDocsFullyTracked() const override;
 #ifdef COUCHBASE_ENTERPRISE
         std::string predictiveTableName(const string& collection, const std::string& property) const override;
         std::string vectorTableName(const string& collection, const std::string& property,

LiteCore/Storage/SQLiteKeyStore.cc

@@ -51,8 +51,11 @@ namespace litecore {
     }
 
     KeyStore& SQLiteDataFile::keyStoreFromTable(slice tableName) {
-        Assert(tableName == "kv_default" || tableName.hasPrefix("kv_."));
-        auto tableName_ = string(tableName.from(3));
+        Assert(tableName.hasPrefix("kv_"));
+        tableName = tableName.from(3);
+        if ( tableName.hasPrefix("del_") ) { tableName = tableName.from(4); }
+        Assert(tableName == "default" || tableName.hasPrefix("."));
+        auto tableName_ = string(tableName);
         return getKeyStore(SQLiteKeyStore::transformCollectionName(tableName_, false));
     }