Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions query/AST/FunctionDecls.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -988,6 +988,20 @@ void FunctionDecls::initCollectionFunctions() {
indexOf->setReturnTypes({{EvaluatedType::Integer}});
indexOf->setSearchesItsList(true);
}

const PossibleArgTypes listOrCell {EvaluatedType::List, EvaluatedType::Any};
const PossibleArgTypes integerOrCell {EvaluatedType::Integer, EvaluatedType::Any};

std::vector<FunctionSignature*> sortMultis;
coll->createOverloads("sortMulti", {listOrCell, listOrCell, integerOrCell, integerOrCell}, sortMultis);

for (FunctionSignature* sortMulti : sortMultis) {
const bool sortsAList = sortMulti->argumentTypes().front().getType() == EvaluatedType::List;

sortMulti->setRequiredArgCount(1);
sortMulti->setReturnTypes({{EvaluatedType::List}});
sortMulti->setReturnsItsArgumentShape(sortsAList);
}
}

void FunctionDecls::initSpatialFunctions() {
Expand Down
26 changes: 26 additions & 0 deletions query/ir/codegen/DBProgramGenerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10445,6 +10445,9 @@ void DBProgramGenerator::translateFunctionExpr(const Expr* expr,
} else if (funcName == "replace") {
translateReplace(expr, args);
return;
} else if (funcName == "coll.sortMulti") {
translateSortMulti(expr, args);
return;
} else if (funcName == "point.distance") {
translatePointDistance(expr, args);
return;
Expand Down Expand Up @@ -10623,6 +10626,29 @@ void DBProgramGenerator::translateSubstring(const Expr* expr, const ExprChain* a
_part._exprMap[expr] = substring.getResult();
}

void DBProgramGenerator::translateSortMulti(const Expr* expr, const ExprChain* args) {
bioassert(args && !args->empty() && args->size() <= 4, "coll.sortMulti() takes 1 to 4 arguments.");

const ExprChain::ExprVector& argExprs = args->getExprs();
const size_t argCount = argExprs.size();

const mlir::Value list = translateArg(argExprs[0]);
const mlir::Value fields = argCount > 1 ? translateArg(argExprs[1]) : mlir::Value {};
const mlir::Value limit = argCount > 2 ? translateArg(argExprs[2]) : mlir::Value {};
const mlir::Value skip = argCount > 3 ? translateArg(argExprs[3]) : mlir::Value {};

const mlir::db::ColumnType noneType = allocColumnType(mlir::NoneType::get(_mlirCtxt));

mlir::db::SortMulti sortMulti = _opBuilder.create<mlir::db::SortMulti>(_opBuilder.getUnknownLoc(),
noneType,
list,
fields,
limit,
skip);

_part._exprMap[expr] = sortMulti.getResult();
}

void DBProgramGenerator::translateSplit(const Expr* expr, const ExprChain* args) {
bioassert(args && args->size() == 2, "split() takes 2 arguments.");

Expand Down
1 change: 1 addition & 0 deletions query/ir/codegen/DBProgramGenerator.h
Original file line number Diff line number Diff line change
Expand Up @@ -1391,6 +1391,7 @@ class DBProgramGenerator {

void translateRange(const Expr* expr, const ExprChain* args);
void translateSubstring(const Expr* expr, const ExprChain* args);
void translateSortMulti(const Expr* expr, const ExprChain* args);
void translateSplit(const Expr* expr, const ExprChain* args);
void translateReplace(const Expr* expr, const ExprChain* args);
void translatePointDistance(const Expr* expr, const ExprChain* args);
Expand Down
44 changes: 36 additions & 8 deletions query/ir/dialect/db/DBHeldTags.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "mlir/IR/BuiltinTypes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"

#include "DBColumnElement.h"
Expand All @@ -29,6 +30,15 @@ TagMask tagBit(CellTag tag) {
return TagMask {1} << static_cast<uint8_t>(tag);
}

TagMask tagMaskOf(std::initializer_list<CellTag> tags) {
TagMask mask = 0;
for (const CellTag tag : tags) {
mask |= tagBit(tag);
}

return mask;
}

CellTag integerTag(mlir::IntegerType integer) {
if (integer.getWidth() == 1) {
return CellTag::Bool;
Expand Down Expand Up @@ -133,8 +143,13 @@ class HeldTagWalk {
_steps.push_back(ReadStep {});
}

bool holdsOnlyTags(mlir::Value column) {
reach(column, NO_READ);
bool holdsOnlyTags(mlir::Value column, llvm::ArrayRef<mlir::StringAttr> reads) {
size_t path = NO_READ;
for (const mlir::StringAttr read : llvm::reverse(reads)) {
path = prepend(read, path);
}

reach(column, path);

while (!_worklist.empty()) {
const Reach next = _worklist.pop_back_val();
Expand Down Expand Up @@ -246,6 +261,8 @@ class HeldTagWalk {
return walkMadeMap(map, path);
} else if (mlir::db::MergeMaps merge = mlir::dyn_cast_or_null<mlir::db::MergeMaps>(made)) {
return walkMergedMaps(merge, path);
} else if (mlir::db::SortMulti sort = mlir::dyn_cast_or_null<mlir::db::SortMulti>(made)) {
return walkSortedList(sort, path);
} else {
return walkType(element, path);
}
Expand All @@ -270,6 +287,16 @@ class HeldTagWalk {
return true;
}

// A sort holds some of the elements of the list it sorts
bool walkSortedList(mlir::db::SortMulti sort, size_t path) {
if (path == NO_READ) {
return holds(CellTag::ListView);
}

reach(sort.getList(), path);
return true;
}

// A key missing from the map reads null, and the map iterated is a list of one
bool walkMadeMap(mlir::db::MakeMap map, size_t path) {
if (path == NO_READ) {
Expand Down Expand Up @@ -377,12 +404,13 @@ class HeldTagWalk {
}

bool db::holdsOnlyNullOr(mlir::Value column, std::initializer_list<CellTag> tags) {
TagMask mask = 0;
for (const CellTag tag : tags) {
mask |= tagBit(tag);
}
return readsHoldOnlyNullOr(column, {}, tags);
}

HeldTagWalk walk(mask);
bool db::readsHoldOnlyNullOr(mlir::Value column,
llvm::ArrayRef<mlir::StringAttr> reads,
std::initializer_list<CellTag> tags) {
HeldTagWalk walk(tagMaskOf(tags));

return walk.holdsOnlyTags(column);
return walk.holdsOnlyTags(column, reads);
}
9 changes: 9 additions & 0 deletions query/ir/dialect/db/DBHeldTags.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

#include <initializer_list>

#include "llvm/ADT/ArrayRef.h"

#include "CellTag.h"

namespace mlir {
class Value;
class StringAttr;
}

namespace db {
Expand All @@ -15,4 +18,10 @@ namespace db {
// carry it, and from a key or an element read off it to the value it was read out of.
bool holdsOnlyNullOr(mlir::Value column, std::initializer_list<CellTag> tags);

// The same of what @param reads, taken in order off each row's value, arrive at: the value
// under a key of a map, or each element of a list where the key is null
bool readsHoldOnlyNullOr(mlir::Value column,
llvm::ArrayRef<mlir::StringAttr> reads,
std::initializer_list<CellTag> tags);

}
73 changes: 73 additions & 0 deletions query/ir/dialect/db/DBOps.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2547,6 +2547,24 @@ LogicalResult Range::verify() {
return success();
}

LogicalResult SortMulti::verify() {
const Value fields = getFields();
const Value limit = getLimit();
const Value skip = getSkip();

const bool readsLists = isColumnOf<storage::ListType>(getList().getType())
&& (!fields || isColumnOf<storage::ListType>(fields.getType()));
const bool readsIntegerBounds = (!limit || isIntegerColumn(limit)) && (!skip || isIntegerColumn(skip));

if (!readsLists) {
return emitOpError("sorts a column of lists on a column of lists of fields");
} else if (!readsIntegerBounds) {
return emitOpError("limit and skip must be integer columns");
}

return success();
}

LogicalResult Replace::verify() {
const bool readsStrings = llvm::all_of(getOperation()->getOperands(), [](Value operand) {
return isColumnOf<storage::StringType>(operand.getType());
Expand Down Expand Up @@ -3287,6 +3305,61 @@ Speculation::Speculatability ListSlice::getSpeculatability() {
return speculatableWhen(holdsOnlyLists(getList()) && readsIntegerBounds);
}

// A key is sorted on in the order <=> gives cells, which raises on an embedding, and a list
// or a map may hold one at any depth. With no field the list is kept as it is.
Speculation::Speculatability SortMulti::getSpeculatability() {
const Value list = getList();
const Value fields = getFields();
const Value limit = getLimit();
const Value skip = getSkip();

int64_t skipped = 0;
const bool skipsALiteralCount = !skip || (matchIntegerLiteral(skip, skipped) && skipped >= 0);
const bool readsIntegerLimit = !limit || holdsOnlyIntegers(limit);

ConstantOp fieldsLiteral = fields ? fields.getDefiningOp<ConstantOp>() : ConstantOp {};
const ArrayAttr names = fieldsLiteral ? dyn_cast<ArrayAttr>(fieldsLiteral.getValue()) : ArrayAttr {};

if (!skipsALiteralCount || !readsIntegerLimit || !holdsOnlyLists(list)) {
return Speculation::NotSpeculatable;
} else if (!fields || (names && names.empty())) {
return Speculation::Speculatable;
} else if (!names || !::db::readsHoldOnlyNullOr(list, {StringAttr {}}, {::db::CellTag::MapView})) {
return Speculation::NotSpeculatable;
}

for (const Attribute name : names) {
const StringAttr field = dyn_cast<StringAttr>(name);
if (!field) {
return Speculation::NotSpeculatable;
}

const StringRef text = field.getValue();
const StringAttr key = StringAttr::get(getContext(), text.starts_with("^") ? text.drop_front() : text);
const StringAttr keyOfEachElement[] = {StringAttr {}, key};

const bool keyHasAnOrder = ::db::readsHoldOnlyNullOr(list,
keyOfEachElement,
{::db::CellTag::Int,
::db::CellTag::UInt,
::db::CellTag::Double,
::db::CellTag::String,
::db::CellTag::Bool,
::db::CellTag::DateTime,
::db::CellTag::Duration,
::db::CellTag::Point,
::db::CellTag::NodeID,
::db::CellTag::EdgeID,
::db::CellTag::Path});

if (!keyHasAnOrder) {
return Speculation::NotSpeculatable;
}
}

return Speculation::Speculatable;
}

Speculation::Speculatability ListIndex::getSpeculatability() {
return speculatableWhen(holdsOnlyLists(getLhs()));
}
Expand Down
37 changes: 37 additions & 0 deletions query/ir/dialect/db/DBOps.td
Original file line number Diff line number Diff line change
Expand Up @@ -684,6 +684,43 @@ def ListSlice : TuringOp<"list_slice", [MayRaise, ConstantThroughOperands, AttrS
}];
}

/**
* SortMulti
*/
def SortMulti : TuringOp<"sort_multi", [MayRaise, ConstantThroughOperands, AttrSizedOperandSegments, DeclareOpInterfaceMethods<ConditionallySpeculatable>]> {
let summary = "Sort each row's list of maps on the values of some keys";

let description = [{
`coll.sortMulti(maps, fields, limit, skip)`, APOC's `apoc.coll.sortMulti`. The op reads
one list column and produces one list cell per row:

%sorted = db.sort_multi(%maps) fields %fields limit %limit skip %skip
: (!db.column<none>, !db.column<!storage.list<!storage.string>>,
!db.column<i64>, !db.column<i64>) -> !db.column<none>

Row r's maps are sorted on the keys row r's fields name, the first one first: a field
starting with `^` sorts ascending on the key after it, any other descending. Two values
compare as ORDER BY compares them, and a missing key is a null, which APOC orders before
every value: first ascending, last descending. Maps that tie keep their order. The sorted maps past the first `skip` are
kept, at most `limit` of them, all where `limit` is negative.

An operand the query left out is absent: no field keeps the list as it is, whatever it
holds, no limit keeps every map and no skip keeps the first. A row with a null operand
gets a null cell. Sorted on a field, a null element holds no key, as `null.a` is null,
and an element that is no map raises, as do a field that is no string and a negative skip.
The result element type is the input's and is resolved during lowering.
}];

let arguments = (ins Column:$list, Optional<Column>:$fields, Optional<Column>:$limit, Optional<Column>:$skip);
let results = (outs Column:$result);
let hasVerifier = 1;

let assemblyFormat = [{
`(` $list `)` (`fields` $fields^)? (`limit` $limit^)? (`skip` $skip^)? attr-dict `:`
functional-type(operands, results)
}];
}

/**
* PercentileCont / PercentileDisc
*/
Expand Down
33 changes: 33 additions & 0 deletions query/ir/dialect/nl/NLOps.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1013,6 +1013,39 @@ LogicalResult DynamicMapKey::verify() {
return success();
}

LogicalResult SortMulti::verify() {
const Value fields = getFields();
const Value limit = getLimit();
const Value skip = getSkip();

const auto readsAList = [](Value chunk) {
return isa<storage::ListType, storage::AnyType>(valueElementOf(chunk.getType()));
};

const auto readsAnInteger = [](Value chunk) {
return isa<IntegerType, storage::AnyType>(valueElementOf(chunk.getType()));
};

const Type list = valueElementOf(getList().getType());
const Type sorted = isa<storage::AnyType>(list) ? storage::ListType::get(getContext(), list) : list;
const Type result = cast<ChunkType>(getResult().getType()).getElementType();
const auto nullableResult = dyn_cast<storage::NullableType>(result);

const bool readsLists = readsAList(getList()) && (!fields || readsAList(fields));
const bool readsIntegerBounds = (!limit || readsAnInteger(limit)) && (!skip || readsAnInteger(skip));
const bool answersTheInputList = nullableResult && nullableResult.getValueType() == sorted;

if (!readsLists) {
return emitOpError("sorts a chunk of lists or of tagged cells on one of lists of fields, optionally nullable");
} else if (!readsIntegerBounds) {
return emitOpError("reads its limit and its skip out of chunks of integers or of tagged cells");
} else if (!answersTheInputList) {
return emitOpError("result must be a chunk of nullable lists of the input list's elements");
}

return success();
}

LogicalResult ContainerIndex::verify() {
const Type base = valueElementOf(getBase().getType());
if (!isa<storage::ListType, storage::MapType, storage::AnyType>(base)) {
Expand Down
34 changes: 34 additions & 0 deletions query/ir/dialect/nl/NLOps.td
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,40 @@ def ListSlice : NLOp<"list_slice", [Pure, RowAlignedThroughOperands, AttrSizedOp
}];
}

/**
* SortMulti
*/
def SortMulti : NLOp<"sort_multi", [Pure, RowAlignedThroughOperands, AttrSizedOperandSegments]> {
let summary = "Row-wise sort of a list of maps on the values of some keys";

let description = [{
Imperative counterpart of db.sort_multi. Row r of the result holds the maps of row r's
list in the order row r's fields sort them, cut by its skip and its limit. A sorted list
lives in the step's own buffer, which the next run of the step clears; with no field the
row is a slice of its input list, which copies nothing.

%sorted = nl.sort_multi %maps fields %fields limit %limit skip %skip
: (!nl.chunk<!storage.nullable<!storage.list<!storage.any>>>,
!nl.chunk<!storage.nullable<!storage.list<!storage.string>>>,
!nl.chunk<!storage.nullable<i64>>,
!nl.chunk<!storage.nullable<i64>>)
-> !nl.chunk<!storage.nullable<!storage.list<!storage.any>>>

Every operand carries rows - lowering lays a constant one out over them first. The
limit and the skip are read as nullable value chunks, or as chunks of type-erased
cells whose integer each row reads out of its cell.
}];

let arguments = (ins NLChunk:$list, Optional<NLChunk>:$fields, Optional<NLChunk>:$limit, Optional<NLChunk>:$skip);
let results = (outs NLChunk:$result);
let hasVerifier = 1;

let assemblyFormat = [{
$list (`fields` $fields^)? (`limit` $limit^)? (`skip` $skip^)? attr-dict `:`
functional-type(operands, results)
}];
}

/**
* PercentileCont / PercentileDisc
*/
Expand Down
Loading
Loading