Optimize out degree count query

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Lbug VERSION 0.18.0 LANGUAGES CXX C)
+project(Lbug VERSION 0.19.0 LANGUAGES CXX C)
 
 option(SINGLE_THREADED "Single-threaded mode" FALSE)
 if(SINGLE_THREADED)

scripts/antlr4/Cypher.g4

@@ -196,6 +196,8 @@ SEQUENCE : ( 'S' | 's' ) ( 'E' | 'e' ) ( 'Q' | 'q' ) ( 'U' | 'u' ) ( 'E' | 'e' )
 
 SET : ( 'S' | 's' ) ( 'E' | 'e' ) ( 'T' | 't' ) ;
 
+SORTED : ( 'S' | 's' ) ( 'O' | 'o' ) ( 'R' | 'r' ) ( 'T' | 't' ) ( 'E' | 'e' ) ( 'D' | 'd' ) ;
+
 SHORTEST : ( 'S' | 's' ) ( 'H' | 'h' ) ( 'O' | 'o' ) ( 'R' | 'r' ) ( 'T' | 't' ) ( 'E' | 'e' ) ( 'S' | 's' ) ( 'T' | 't' ) ;
 
 START : ( 'S' | 's' ) ( 'T' | 't' ) ( 'A' | 'a' ) ( 'R' | 'r' ) ( 'T' | 't' ) ;
@@ -461,7 +463,8 @@ iC_AlterOptions
         | iC_RenameTable
         | iC_RenameProperty
         | iC_AddFromToConnection
-        | iC_DropFromToConnection;
+        | iC_DropFromToConnection
+        | iC_SetSortedBy;
 
 iC_AddProperty
     : ADD SP (iC_IfNotExists SP)? oC_PropertyKeyName SP iC_DataType ( SP iC_Default )? ;
@@ -484,6 +487,12 @@ iC_AddFromToConnection
 iC_DropFromToConnection
     : DROP SP (iC_IfExists SP)? iC_FromToConnection ;
 
+iC_SetSortedBy
+    : SET SP SORTED SP BY SP? '(' SP? iC_SortedByItem ( SP? ',' SP? iC_SortedByItem )* SP? ')' ;
+
+iC_SortedByItem
+    : oC_PropertyKeyName SP ( ASC | DESC ) ;
+
 iC_ColumnDefinitions: iC_ColumnDefinition ( SP? ',' SP? iC_ColumnDefinition )* ;
 
 iC_ColumnDefinition : oC_PropertyKeyName SP iC_DataType ;

scripts/antlr4/hash.md5

@@ -1 +1 @@
-b940f6324ab2c15b0367f4cbd26040fb
+1c6e4420671a8ce9be258805fc58ff5a

src/antlr4/Cypher.g4

@@ -206,7 +206,8 @@ iC_AlterOptions
         | iC_RenameTable
         | iC_RenameProperty
         | iC_AddFromToConnection
-        | iC_DropFromToConnection;
+        | iC_DropFromToConnection
+        | iC_SetSortedBy;
 
 iC_AddProperty
     : ADD SP (iC_IfNotExists SP)? oC_PropertyKeyName SP iC_DataType ( SP iC_Default )? ;
@@ -229,6 +230,12 @@ iC_AddFromToConnection
 iC_DropFromToConnection
     : DROP SP (iC_IfExists SP)? iC_FromToConnection ;
 
+iC_SetSortedBy
+    : SET SP SORTED SP BY SP? '(' SP? iC_SortedByItem ( SP? ',' SP? iC_SortedByItem )* SP? ')' ;
+
+iC_SortedByItem
+    : oC_PropertyKeyName SP ( ASC | DESC ) ;
+
 iC_ColumnDefinitions: iC_ColumnDefinition ( SP? ',' SP? iC_ColumnDefinition )* ;
 
 iC_ColumnDefinition : oC_PropertyKeyName SP iC_DataType ;

src/antlr4/keywords.txt

@@ -89,6 +89,7 @@ ROLLBACK
 ROLLBACK_SKIP_CHECKPOINT
 SEQUENCE
 SET
+SORTED
 SHORTEST
 START
 STARTS

src/binder/bind/bind_ddl.cpp

@@ -655,6 +655,9 @@ std::unique_ptr<BoundStatement> Binder::bindAlter(const Statement& statement) {
     case AlterType::COMMENT: {
         return bindCommentOn(statement);
     }
+    case AlterType::SET_SORTED_BY: {
+        return bindSetSortedBy(statement);
+    }
     case AlterType::ADD_FROM_TO_CONNECTION:
     case AlterType::DROP_FROM_TO_CONNECTION: {
         return bindAlterFromToConnection(statement);
@@ -738,6 +741,27 @@ std::unique_ptr<BoundStatement> Binder::bindCommentOn(const Statement& statement
     return std::make_unique<BoundAlter>(std::move(boundInfo));
 }
 
+std::unique_ptr<BoundStatement> Binder::bindSetSortedBy(const Statement& statement) const {
+    auto& alter = statement.constCast<Alter>();
+    auto info = alter.getInfo();
+    auto extraInfo = info->extraInfo->constPtrCast<ExtraSetSortedByInfo>();
+    auto tableName = info->tableName;
+    auto catalog = Catalog::Get(*clientContext);
+    auto transaction = transaction::Transaction::Get(*clientContext);
+    auto tableEntry = catalog->getTableCatalogEntry(transaction, tableName);
+    validateNodeTableType(tableEntry);
+    std::vector<BoundSortedByProperty> properties;
+    properties.reserve(extraInfo->properties.size());
+    for (auto& property : extraInfo->properties) {
+        validateColumnExistence(tableEntry, property.propertyName);
+        properties.push_back(BoundSortedByProperty{property.propertyName, property.ascending});
+    }
+    auto boundExtraInfo = std::make_unique<BoundExtraSetSortedByInfo>(std::move(properties));
+    auto boundInfo = BoundAlterInfo(AlterType::SET_SORTED_BY, tableName, std::move(boundExtraInfo),
+        info->onConflict);
+    return std::make_unique<BoundAlter>(std::move(boundInfo));
+}
+
 std::unique_ptr<BoundStatement> Binder::bindAlterFromToConnection(
     const Statement& statement) const {
     auto& alter = statement.constCast<Alter>();

src/binder/ddl/bound_alter_info.cpp

@@ -35,6 +35,19 @@ std::string BoundAlterInfo::toString() const {
         result += "Comment on Table " + tableName;
         break;
     }
+    case common::AlterType::SET_SORTED_BY: {
+        auto sortedByInfo =
+            common::dynamic_cast_checked<BoundExtraSetSortedByInfo*>(extraInfo.get());
+        result += "Set Table " + tableName + " Sorted By ";
+        for (auto i = 0u; i < sortedByInfo->properties.size(); ++i) {
+            if (i > 0) {
+                result += ", ";
+            }
+            auto& property = sortedByInfo->properties[i];
+            result += property.propertyName + (property.ascending ? " ASC" : " DESC");
+        }
+        break;
+    }
     default:
         break;
     }

src/catalog/catalog_entry/node_table_catalog_entry.cpp

@@ -14,6 +14,18 @@ using namespace lbug::common;
 namespace lbug {
 namespace catalog {
 
+void SortedByProperty::serialize(common::Serializer& serializer) const {
+    serializer.write(propertyName);
+    serializer.write(ascending);
+}
+
+SortedByProperty SortedByProperty::deserialize(common::Deserializer& deserializer) {
+    SortedByProperty result;
+    deserializer.deserializeValue(result.propertyName);
+    deserializer.deserializeValue(result.ascending);
+    return result;
+}
+
 static void upgradeLegacyStorageFormat(const std::string& storage,
     common::StorageFormat& storageFormat) {
     const auto lowerStorage = common::StringUtils::getLower(storage);
@@ -28,12 +40,20 @@ void NodeTableCatalogEntry::renameProperty(const std::string& propertyName,
     if (common::StringUtils::caseInsensitiveEquals(propertyName, primaryKeyName)) {
         primaryKeyName = newName;
     }
+    for (auto& sortedByProperty : sortedByProperties) {
+        if (common::StringUtils::caseInsensitiveEquals(propertyName,
+                sortedByProperty.propertyName)) {
+            sortedByProperty.propertyName = newName;
+        }
+    }
 }
 
 void NodeTableCatalogEntry::serialize(common::Serializer& serializer) const {
     TableCatalogEntry::serialize(serializer);
     serializer.writeDebuggingInfo("primaryKeyName");
     serializer.write(primaryKeyName);
+    serializer.writeDebuggingInfo("sortedByProperties");
+    serializer.serializeVector(sortedByProperties);
     serializer.writeDebuggingInfo("storage");
     serializer.write(storage);
     serializer.writeDebuggingInfo("storageFormat");
@@ -44,10 +64,16 @@ std::unique_ptr<NodeTableCatalogEntry> NodeTableCatalogEntry::deserialize(
     common::Deserializer& deserializer) {
     std::string debuggingInfo;
     std::string primaryKeyName;
+    std::vector<SortedByProperty> sortedByProperties;
     std::string storage;
     auto storageFormat = StorageFormat::NONE;
     deserializer.validateDebuggingInfo(debuggingInfo, "primaryKeyName");
     deserializer.deserializeValue(primaryKeyName);
+    if (deserializer.getStorageVersion() >=
+        ::lbug::storage::StorageVersionInfo::STORAGE_VERSION_43) {
+        deserializer.validateDebuggingInfo(debuggingInfo, "sortedByProperties");
+        deserializer.deserializeVector(sortedByProperties);
+    }
     deserializer.validateDebuggingInfo(debuggingInfo, "storage");
     deserializer.deserializeValue(storage);
     if (deserializer.getStorageVersion() >=
@@ -59,6 +85,7 @@ std::unique_ptr<NodeTableCatalogEntry> NodeTableCatalogEntry::deserialize(
     }
     auto nodeTableEntry = std::make_unique<NodeTableCatalogEntry>();
     nodeTableEntry->primaryKeyName = primaryKeyName;
+    nodeTableEntry->sortedByProperties = std::move(sortedByProperties);
     nodeTableEntry->storage = storage;
     nodeTableEntry->storageFormat = storageFormat;
     return nodeTableEntry;
@@ -88,6 +115,7 @@ std::unique_ptr<binder::BoundTableScanInfo> NodeTableCatalogEntry::getBoundScanI
 std::unique_ptr<TableCatalogEntry> NodeTableCatalogEntry::copy() const {
     auto other = std::make_unique<NodeTableCatalogEntry>();
     other->primaryKeyName = primaryKeyName;
+    other->sortedByProperties = sortedByProperties;
     other->storage = storage;
     other->storageFormat = storageFormat;
     other->scanFunction = scanFunction;

src/catalog/catalog_entry/table_catalog_entry.cpp

@@ -37,6 +37,15 @@ std::unique_ptr<TableCatalogEntry> TableCatalogEntry::alter(transaction_t timest
         auto& commentInfo = *alterInfo.extraInfo->constPtrCast<BoundExtraCommentInfo>();
         newEntry->setComment(commentInfo.comment);
     } break;
+    case AlterType::SET_SORTED_BY: {
+        auto& sortedByInfo = *alterInfo.extraInfo->constPtrCast<BoundExtraSetSortedByInfo>();
+        std::vector<SortedByProperty> properties;
+        properties.reserve(sortedByInfo.properties.size());
+        for (auto& property : sortedByInfo.properties) {
+            properties.push_back(SortedByProperty{property.propertyName, property.ascending});
+        }
+        newEntry->ptrCast<NodeTableCatalogEntry>()->setSortedByProperties(std::move(properties));
+    } break;
     case AlterType::ADD_FROM_TO_CONNECTION: {
         auto& connectionInfo =
             *alterInfo.extraInfo->constPtrCast<BoundExtraAlterFromToConnection>();

src/catalog/catalog_set.cpp

@@ -195,6 +195,7 @@ void CatalogSet::alterTableEntry(Transaction* transaction,
     case AlterType::ADD_PROPERTY:
     case AlterType::DROP_PROPERTY:
     case AlterType::RENAME_PROPERTY:
+    case AlterType::SET_SORTED_BY:
     case AlterType::ADD_FROM_TO_CONNECTION:
     case AlterType::DROP_FROM_TO_CONNECTION: {
         emplaceNoLock(std::move(newEntry));

src/common/serializer/deserializer.cpp

@@ -8,7 +8,7 @@ void Deserializer::deserializeValue(std::string& value) {
     uint64_t valueLength = 0;
     deserializeValue(valueLength);
     value.resize(valueLength);
-    reader->read(reinterpret_cast<uint8_t*>(value.data()), valueLength);
+    this->read(reinterpret_cast<uint8_t*>(value.data()), valueLength);
 }
 
 void Deserializer::validateDebuggingInfo(std::string& value, const std::string& expectedVal) {

src/include/binder/binder.h

@@ -112,6 +112,7 @@ class Binder {
     std::unique_ptr<BoundStatement> bindDropProperty(const parser::Statement& statement) const;
     std::unique_ptr<BoundStatement> bindRenameProperty(const parser::Statement& statement) const;
     std::unique_ptr<BoundStatement> bindCommentOn(const parser::Statement& statement) const;
+    std::unique_ptr<BoundStatement> bindSetSortedBy(const parser::Statement& statement) const;
     std::unique_ptr<BoundStatement> bindAlterFromToConnection(
         const parser::Statement& statement) const;
 

src/include/binder/ddl/bound_alter_info.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vector>
+
 #include "binder/ddl/property_definition.h"
 #include "binder/expression/expression.h"
 #include "common/enums/alter_type.h"
@@ -8,6 +10,11 @@
 namespace lbug {
 namespace binder {
 
+struct BoundSortedByProperty {
+    std::string propertyName;
+    bool ascending;
+};
+
 struct BoundExtraAlterInfo {
     virtual ~BoundExtraAlterInfo() = default;
 
@@ -121,5 +128,18 @@ struct BoundExtraAlterFromToConnection final : BoundExtraAlterInfo {
     }
 };
 
+struct BoundExtraSetSortedByInfo final : BoundExtraAlterInfo {
+    std::vector<BoundSortedByProperty> properties;
+
+    explicit BoundExtraSetSortedByInfo(std::vector<BoundSortedByProperty> properties)
+        : properties{std::move(properties)} {}
+    BoundExtraSetSortedByInfo(const BoundExtraSetSortedByInfo& other)
+        : properties{other.properties} {}
+
+    std::unique_ptr<BoundExtraAlterInfo> copy() const override {
+        return std::make_unique<BoundExtraSetSortedByInfo>(*this);
+    }
+};
+
 } // namespace binder
 } // namespace lbug

src/include/catalog/catalog_entry/node_table_catalog_entry.h

@@ -4,6 +4,9 @@
 #include <optional>
 
 #include "common/enums/storage_format.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
+#include "common/string_utils.h"
 #include "function/table/table_function.h"
 #include "table_catalog_entry.h"
 
@@ -14,6 +17,14 @@ class Transaction;
 
 namespace catalog {
 
+struct SortedByProperty {
+    std::string propertyName;
+    bool ascending;
+
+    void serialize(common::Serializer& serializer) const;
+    static SortedByProperty deserialize(common::Deserializer& deserializer);
+};
+
 // Callback to create bind data for foreign tables
 // This allows extensions to provide bind data creation without core needing to know extension types
 using CreateBindDataFunc =
@@ -59,6 +70,17 @@ class LBUG_API NodeTableCatalogEntry final : public TableCatalogEntry {
     }
     const std::string& getStorage() const { return storage; }
     common::StorageFormat getStorageFormat() const { return storageFormat; }
+    const std::vector<SortedByProperty>& getSortedByProperties() const {
+        return sortedByProperties;
+    }
+    bool isLeadingSortPrimaryKeyAsc() const {
+        return !sortedByProperties.empty() && sortedByProperties[0].ascending &&
+               common::StringUtils::caseInsensitiveEquals(sortedByProperties[0].propertyName,
+                   primaryKeyName);
+    }
+    void setSortedByProperties(std::vector<SortedByProperty> properties) {
+        sortedByProperties = std::move(properties);
+    }
     std::optional<function::TableFunction> getScanFunction() const override;
     const CreateBindDataFunc& getCreateBindDataFunc() const { return createBindDataFunc; }
     const std::string& getForeignDatabaseName() const { return foreignDatabaseName; }
@@ -84,6 +106,7 @@ class LBUG_API NodeTableCatalogEntry final : public TableCatalogEntry {
 
 private:
     std::string primaryKeyName;
+    std::vector<SortedByProperty> sortedByProperties;
     std::string storage;
     common::StorageFormat storageFormat = common::StorageFormat::NONE;
     std::optional<function::TableFunction> scanFunction;

src/include/common/enums/alter_type.h

@@ -13,6 +13,7 @@ enum class AlterType : uint8_t {
     RENAME_PROPERTY = 12,
     ADD_FROM_TO_CONNECTION = 13,
     DROP_FROM_TO_CONNECTION = 14,
+    SET_SORTED_BY = 15,
 
     COMMENT = 201,
     INVALID = 255

src/include/common/serializer/deserializer.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstring>
 #include <functional>
 #include <limits>
 #include <map>
@@ -28,10 +29,25 @@ class LBUG_API Deserializer {
     template<typename T>
         requires std::is_trivially_destructible_v<T> || std::is_same_v<std::string, T>
     void deserializeValue(T& value) {
-        reader->read(reinterpret_cast<uint8_t*>(&value), sizeof(T));
+        this->read(reinterpret_cast<uint8_t*>(&value), sizeof(T));
     }
 
-    void read(uint8_t* data, uint64_t size) const { reader->read(data, size); }
+    void read(uint8_t* data, uint64_t size) const {
+        if (readLimit.has_value()) {
+            const auto remaining = *readLimit - getReadOffset();
+            if (size > remaining) {
+                // Don't read past the declared record boundary.
+                // Read what's available and zero-fill the rest so that fields added
+                // in a newer format default to zero when reading older WAL records.
+                if (remaining > 0) {
+                    reader->read(data, remaining);
+                }
+                std::memset(data + remaining, 0, size - remaining);
+                return;
+            }
+        }
+        reader->read(data, size);
+    }
 
     Reader* getReader() const { return reader.get(); }
 

src/include/optimizer/count_rel_table_optimizer.h

@@ -58,6 +58,8 @@ class CountRelTableOptimizer : public LogicalOperatorVisitor {
     bool canOptimize(planner::LogicalOperator* aggregate) const;
     std::shared_ptr<planner::LogicalOperator> tryRewriteActiveBoundCount(
         std::shared_ptr<planner::LogicalOperator> op);
+    std::shared_ptr<planner::LogicalOperator> tryRewriteSortedOffsetCount(
+        std::shared_ptr<planner::LogicalOperator> op);
     std::shared_ptr<planner::LogicalOperator> tryRewriteDegreeTopK(
         std::shared_ptr<planner::LogicalOperator> op);