From 444bc7536619784a713d01685307e9ee376b07e0 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 6 Feb 2026 17:57:09 +0000 Subject: [PATCH 1/4] HPCC-35827 Add LogicalFileListFiltered filtering and field projection Introduces new file listing API with user-friendly filter syntax and field selection. LogicalFileList is marked DEPRECATED, users should use new function for efficiency and capability. Features: - Filter syntax: wildcards (owner:jsmith), numeric comparisons (size>1000000), date ranges (modified>2024-01-01), property checks (has:description), file type filtering (is:superfile, is:normal) - Field projection to select specific result fields - Configurable file limits with breach detection - Overflow/underflow protection for numeric filters Implementation: - FileListResultFieldSource for IFieldSource pattern - parseUserFilterSyntax converts user syntax to internal DFUQFilter - Field validation via dfuFieldAttrPathMap - Default fields: name, modified, size, rowcount, cluster, superfile, owner Testing: - 12 regression test cases covering filtering, projection, limits, edge cases Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 160 +++-- dali/base/dadfs.hpp | 1 + ecllibrary/std/File.ecl | 103 ++- plugins/fileservices/fileservices.cpp | 656 +++++++++++++++++- plugins/fileservices/fileservices.hpp | 1 + plugins/proxies/lib_fileservices.ecllib | 18 + .../ecl/key/logicalfilelist_filtered_test.xml | 186 +++++ .../ecl/logicalfilelist_filtered_test.ecl | 285 ++++++++ 8 files changed, 1355 insertions(+), 55 deletions(-) create mode 100644 testing/regress/ecl/key/logicalfilelist_filtered_test.xml create mode 100644 testing/regress/ecl/logicalfilelist_filtered_test.ecl diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 7ca096d8e14..a906ab73c53 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -2191,51 +2191,52 @@ struct DFUQFieldInfo DFUQResultField field; std::string name; DFUQResultFieldType type; + std::string attrPath; // Full attribute path (e.g., "Attr/@name" or "@directory") }; -// enum field, name, type +// enum field, name, type, attrPath static const DFUQFieldInfo dfuqFieldInfos[] = { - {DFUQResultField::name, "@name", DFUQResultFieldType::stringType}, - {DFUQResultField::description, "@description", DFUQResultFieldType::stringType}, - {DFUQResultField::nodegroups, "@group", DFUQResultFieldType::stringType}, - {DFUQResultField::kind, "@kind", DFUQResultFieldType::stringType}, - {DFUQResultField::timemodified, "@modified", DFUQResultFieldType::stringType}, - {DFUQResultField::job, "@job", DFUQResultFieldType::stringType}, - {DFUQResultField::owner, "@owner", DFUQResultFieldType::stringType}, - {DFUQResultField::recordcount, "@DFUSFrecordCount", DFUQResultFieldType::numericType}, - {DFUQResultField::origrecordcount, "@recordCount", DFUQResultFieldType::numericType}, - {DFUQResultField::recordsize, "@recordSize", DFUQResultFieldType::numericType}, - {DFUQResultField::size, "@DFUSFsize", DFUQResultFieldType::numericType}, - {DFUQResultField::origsize, "@size", DFUQResultFieldType::numericType}, - {DFUQResultField::workunit, "@workunit", DFUQResultFieldType::stringType}, - {DFUQResultField::nodegroup, "@DFUSFcluster", DFUQResultFieldType::stringType}, - {DFUQResultField::numsubfiles, "@numsubfiles", DFUQResultFieldType::numericType}, - {DFUQResultField::accessed, "@accessed", DFUQResultFieldType::stringType}, - {DFUQResultField::numparts, "@numparts", DFUQResultFieldType::numericType}, - {DFUQResultField::compressedsize, "@compressedSize", DFUQResultFieldType::numericType}, - {DFUQResultField::directory, "@directory", DFUQResultFieldType::stringType}, - {DFUQResultField::partmask, "@partmask", DFUQResultFieldType::stringType}, - {DFUQResultField::superowners, "@superowners", DFUQResultFieldType::stringType}, - {DFUQResultField::persistent, "@persistent", DFUQResultFieldType::boolType}, - {DFUQResultField::protect, "@protect", DFUQResultFieldType::stringType}, - {DFUQResultField::iscompressed, "@compressed", DFUQResultFieldType::boolType}, - {DFUQResultField::cost, "@cost", DFUQResultFieldType::floatType}, - {DFUQResultField::numDiskReads, "@numDiskReads", DFUQResultFieldType::numericType}, - {DFUQResultField::numDiskWrites, "@numDiskWrites", DFUQResultFieldType::numericType}, - {DFUQResultField::atRestCost, "@atRestCost", DFUQResultFieldType::floatType}, - {DFUQResultField::accessCost, "@accessCost", DFUQResultFieldType::floatType}, - {DFUQResultField::maxSkew, "@maxSkew", DFUQResultFieldType::numericType}, - {DFUQResultField::minSkew, "@minSkew", DFUQResultFieldType::numericType}, - {DFUQResultField::maxSkewPart, "@maxSkewPart", DFUQResultFieldType::numericType}, - {DFUQResultField::minSkewPart, "@minSkewPart", DFUQResultFieldType::numericType}, - {DFUQResultField::readCost, "@readCost", DFUQResultFieldType::floatType}, - {DFUQResultField::writeCost, "@writeCost", DFUQResultFieldType::floatType}, - {DFUQResultField::expireDays, "@expireDays", DFUQResultFieldType::numericType}, - {DFUQResultField::subfilenames, "@subfilenames", DFUQResultFieldType::stringType}, - {DFUQResultField::blockCompressed, "@blockCompressed", DFUQResultFieldType::boolType}, - {DFUQResultField::rowCompressed, "@rowCompressed", DFUQResultFieldType::boolType}, - {DFUQResultField::includeAll, "includeAll", DFUQResultFieldType::unknown} + {DFUQResultField::name, "@name", DFUQResultFieldType::stringType, "Attr/@name"}, + {DFUQResultField::description, "@description", DFUQResultFieldType::stringType, "Attr/@description"}, + {DFUQResultField::nodegroups, "@group", DFUQResultFieldType::stringType, "Attr/@group"}, + {DFUQResultField::kind, "@kind", DFUQResultFieldType::stringType, "Attr/@kind"}, + {DFUQResultField::timemodified, "@modified", DFUQResultFieldType::stringType, "@modified"}, + {DFUQResultField::job, "@job", DFUQResultFieldType::stringType, "Attr/@job"}, + {DFUQResultField::owner, "@owner", DFUQResultFieldType::stringType, "Attr/@owner"}, + {DFUQResultField::recordcount, "@DFUSFrecordCount", DFUQResultFieldType::numericType, ""}, + {DFUQResultField::origrecordcount, "@recordCount", DFUQResultFieldType::numericType, "Attr/@recordCount"}, + {DFUQResultField::recordsize, "@recordSize", DFUQResultFieldType::numericType, "Attr/@recordSize"}, + {DFUQResultField::size, "@DFUSFsize", DFUQResultFieldType::numericType, ""}, + {DFUQResultField::origsize, "@size", DFUQResultFieldType::numericType, "Attr/@size"}, + {DFUQResultField::workunit, "@workunit", DFUQResultFieldType::stringType, "Attr/@workunit"}, + {DFUQResultField::nodegroup, "@DFUSFcluster", DFUQResultFieldType::stringType, ""}, + {DFUQResultField::numsubfiles, "@numsubfiles", DFUQResultFieldType::numericType, "Attr/@numsubfiles"}, + {DFUQResultField::accessed, "@accessed", DFUQResultFieldType::stringType, "Attr/@accessed"}, + {DFUQResultField::numparts, "@numparts", DFUQResultFieldType::numericType, "@numparts"}, + {DFUQResultField::compressedsize, "@compressedSize", DFUQResultFieldType::numericType, "Attr/@compressedSize"}, + {DFUQResultField::directory, "@directory", DFUQResultFieldType::stringType, "@directory"}, + {DFUQResultField::partmask, "@partmask", DFUQResultFieldType::stringType, "@partmask"}, + {DFUQResultField::superowners, "@superowners", DFUQResultFieldType::stringType, "SuperOwner"}, + {DFUQResultField::persistent, "@persistent", DFUQResultFieldType::boolType, "Attr/@persistent"}, + {DFUQResultField::protect, "@protect", DFUQResultFieldType::stringType, "Attr/@protect"}, + {DFUQResultField::iscompressed, "@compressed", DFUQResultFieldType::boolType, "Attr/@compressed"}, + {DFUQResultField::cost, "@cost", DFUQResultFieldType::floatType, "Attr/@cost"}, + {DFUQResultField::numDiskReads, "@numDiskReads", DFUQResultFieldType::numericType, "Attr/@numDiskReads"}, + {DFUQResultField::numDiskWrites, "@numDiskWrites", DFUQResultFieldType::numericType, "Attr/@numDiskWrites"}, + {DFUQResultField::atRestCost, "@atRestCost", DFUQResultFieldType::floatType, "Attr/@atRestCost"}, + {DFUQResultField::accessCost, "@accessCost", DFUQResultFieldType::floatType, "Attr/@accessCost"}, + {DFUQResultField::maxSkew, "@maxSkew", DFUQResultFieldType::numericType, "Attr/@maxSkew"}, + {DFUQResultField::minSkew, "@minSkew", DFUQResultFieldType::numericType, "Attr/@minSkew"}, + {DFUQResultField::maxSkewPart, "@maxSkewPart", DFUQResultFieldType::numericType, "Attr/@maxSkewPart"}, + {DFUQResultField::minSkewPart, "@minSkewPart", DFUQResultFieldType::numericType, "Attr/@minSkewPart"}, + {DFUQResultField::readCost, "@readCost", DFUQResultFieldType::floatType, "Attr/@readCost"}, + {DFUQResultField::writeCost, "@writeCost", DFUQResultFieldType::floatType, "Attr/@writeCost"}, + {DFUQResultField::expireDays, "@expireDays", DFUQResultFieldType::numericType, "Attr/@expireDays"}, + {DFUQResultField::subfilenames, "@subfilenames", DFUQResultFieldType::stringType, "Attr/@subfilenames"}, + {DFUQResultField::blockCompressed, "@blockCompressed", DFUQResultFieldType::boolType, "Attr/@blockCompressed"}, + {DFUQResultField::rowCompressed, "@rowCompressed", DFUQResultFieldType::boolType, "Attr/@rowCompressed"}, + {DFUQResultField::includeAll, "includeAll", DFUQResultFieldType::unknown, ""} }; const size_t dfuqFieldInfosCount = sizeof(dfuqFieldInfos)/sizeof(dfuqFieldInfos[0]); static_assert(dfuqFieldInfosCount == static_cast(DFUQResultField::term), @@ -2263,6 +2264,50 @@ static const DFUQResultFieldMap dfuResultFieldStringMap = [] return map; }(); +/* Field name to attribute path map: maps simple field names to their full File attribute paths + key = simple field name (e.g., "name", "size", "recordcount") + value = { full-attribute-path, field-type } + + The full attribute path is: + - Direct attribute name for File-level attributes (e.g., "@directory", "@numparts") + - "Attr/" + attribute name for File/Attr properties (e.g., "Attr/@name", "Attr/@owner") + + This map converts user-specified field names into the actual paths needed to access + the attributes in the File property tree structure. + */ +typedef std::unordered_map, CaseInsensitiveHash, CaseInsensitiveEqual> DFUQFieldAttrPathMap; + +static const DFUQFieldAttrPathMap dfuFieldAttrPathMap = [] +{ + DFUQFieldAttrPathMap map; + + // Derive field names from dfuqFieldInfos using pre-defined attribute paths + // Skip fields not exposed for filtering + for (size_t i = 0; i < dfuqFieldInfosCount; ++i) + { + DFUQResultField field = dfuqFieldInfos[i].field; + assertex(field == (DFUQResultField)i); + + if (dfuqFieldInfos[i].attrPath.size() == 0) + continue; + + const char *attrName = dfuqFieldInfos[i].name.c_str(); + // Strip '@' prefix to get simple field name + const char* fieldName = (attrName[0] == '@') ? attrName + 1 : attrName; + + // Use the pre-defined attribute path from dfuqFieldInfos + map.emplace(fieldName, std::make_tuple(dfuqFieldInfos[i].attrPath, field, dfuqFieldInfos[i].type)); + } + + // Add field name aliases that override the defaults + // "recordcount" -> origrecordcount (the actual file's record count, not @DFUSFrecordCount) + // "size" -> origsize (the actual file's size, not @DFUSFsize) + map["recordcount"] = std::make_tuple("Attr/@recordCount", DFUQResultField::origrecordcount, DFUQResultFieldType::numericType); + map["size"] = std::make_tuple("Attr/@size", DFUQResultField::origsize, DFUQResultFieldType::numericType); + + return map; +}(); + const char* getDFUQResultFieldName(DFUQResultField field) { @@ -2305,6 +2350,24 @@ const char* getDFUQResultFieldTypeName(DFUQResultFieldType type) } } +bool getFileAttributePath(const char *fieldName, StringBuffer &attributePath, DFUQResultField &field, DFUQResultFieldType &type) +{ + if (isEmptyString(fieldName)) + return false; + + // Look up field name in the attribute path map + auto it = dfuFieldAttrPathMap.find(fieldName); + if (it != dfuFieldAttrPathMap.end()) + { + attributePath.set(std::get<0>(it->second).c_str()); + field = std::get<1>(it->second); + type = std::get<2>(it->second); + return true; + } + + return false; +} + static std::vector dfuQResultFieldsToVector(const DFUQResultField *fields, bool includeTerminator) { std::vector result; @@ -2396,6 +2459,15 @@ struct SerializeFileAttrOptions // special handling if DFUSFsize is included (which is calculated client side), to also include dependent origsize fieldsList[static_cast(DFUQResultField::origsize)] = additive; break; + case DFUQResultField::iscompressed: + fieldsList[static_cast(field)] = additive; + if (additive) + { + // The client uses these to decide if compresed or not. + fieldsList[static_cast(DFUQResultField::kind)] = true; + fieldsList[static_cast(DFUQResultField::blockCompressed)] = true; + } + break; default: { fieldsList[static_cast(field)] = additive; @@ -4354,7 +4426,7 @@ protected: friend class CDistributedFilePart; { // NB: this is non-standard, for situations where the cluster name is not known, // which happens where none has been provided/set to the file descriptor, - // and the file descriptor has been built up of parts with ips. + // and the file descriptor has been built up of parts with ips. // createClusterInfo will perform a reverse lookup to Dali to try to discover // a group name. cluster = createClusterInfo( @@ -11959,7 +12031,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I mb.read(lname); if (version >= 2) { - unsigned _opts; + unsigned _opts; mb.read(_opts); opts = static_cast(_opts); bool hasUser; @@ -14487,7 +14559,7 @@ IPropertyTreeIterator* CDistributedFileDirectory::getLogicalFiles( { fieldsWithSorted = dfuQResultFieldsToVector(fields, false); - // NB: could add sort fields here, that are already in fields, that's okay, they will take precedence by being last + // NB: could add sort fields here, that are already in fields, that's okay, they will take precedence by being last for (unsigned s=0; sortOrder[s] != DFUQResultField::term; s++) { DFUQResultField sortField = sortOrder[s] & DFUQResultField::fieldMask; diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index f05ffbad4f8..5b1cefabace 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -334,6 +334,7 @@ extern da_decl DFUQResultFieldType getDFUQResultFieldType(DFUQResultField field) extern da_decl DFUQResultField getDFUQResultField(const char *fieldName); extern da_decl DFUQResultField getDFUQResultFieldAndType(const char *fieldName); extern da_decl const char* getDFUQResultFieldTypeName(DFUQResultField field); +extern da_decl bool getFileAttributePath(const char *fieldName, StringBuffer &attributePath, DFUQResultField &field, DFUQResultFieldType &type); /** diff --git a/ecllibrary/std/File.ecl b/ecllibrary/std/File.ecl index 0caa13e9345..d402223bab0 100644 --- a/ecllibrary/std/File.ecl +++ b/ecllibrary/std/File.ecl @@ -112,7 +112,7 @@ EXPORT FsLandingZoneRecord := lib_fileservices.FsLandingZoneRecord; * @return Boolean value of 'noCommon' property. */ -EXPORT boolean GetNoCommonDefault() := +EXPORT boolean GetNoCommonDefault() := lib_fileservices.FileServices.GetNoCommonDefault(); /** @@ -225,7 +225,92 @@ EXPORT dataset(FsFilenameRecord) RemoteDirectory(varstring machineIP, varstring * locally. Defaults to blank. */ EXPORT dataset(FsLogicalFileInfoRecord) LogicalFileList(varstring namepattern='*', boolean includenormal=TRUE, boolean includesuper=FALSE, boolean unknownszero=FALSE, varstring foreigndali='') := - lib_fileservices.FileServices.LogicalFileList(namepattern, includenormal, includesuper, unknownszero, foreigndali); + lib_fileservices.FileServices.LogicalFileList(namepattern, includenormal, includesuper, unknownszero, foreigndali) + : DEPRECATED('Use Std.File.LogicalFileListFiltered for enhanced filtering and count information'); + +/** + * Returns information about logical files known to the system, including count and limit breach status. + * This function supports file limits, filtering, and field projection for efficient data retrieval. + * + * @param namepattern The mask of the files to list. Defaults to '*' (all files). + * @param filters Filter string using search-like syntax for complex queries. Defaults to blank. + * Multiple filters separated by spaces. Supports: + * + * Field matching: + * field:value - Exact wildcard match (owner:jsmith, name:*test*) + * !field:value - Negation (not matching) + * + * Numeric comparisons: + * field>value - Greater than (size>1000000, recordcount>0) + * field=value - Greater than or equal + * field<=value - Less than or equal + * field=value - Exact match (recordcount=0) + * + * String comparisons: + * modified>2024-01-01 - Date/string greater than + * modified<2024-12-31 - Date/string less than + * + * Property checks: + * has:propertyname - Property exists + * !has:propertyname - Property does not exist + * + * File type: + * is:superfile - Superfiles only + * is:normal - Normal files only + * is:any - All file types (default) + * + * Common fields: owner, group, name, size, recordcount, modified, recordsize, numparts + * + * Examples: + * 'owner:jsmith' - Files owned by jsmith + * 'owner:*doe' - Owner ends with 'doe' + * 'size>100000000' - Files larger than 100MB + * 'recordcount>0' - Files with records + * 'is:superfile' - Superfiles only + * 'is:normal !has:description' - Normal files without description + * 'owner:jsmith size>1000000' - Multiple filters (AND logic) + * 'modified>2024-01-01 is:normal' - Recent normal files + * @param fields Comma-separated list of field names to include in the result. Defaults to blank (all default fields). + * When blank or empty, returns default fields: name, modified, size, rowcount, cluster, superfile, owner. + * Available fields: name, modified, size, rowcount, cluster, superfile, owner, group, recordsize, numparts. + * Note: 'name' field is always included regardless of specification. + * Examples: + * '' - Default fields (backward compatible) + * 'name,size' - Only name and size + * 'name,superfile,owner' - Name, superfile flag, and owner + * @param unknownszero Whether to set file sizes that are unknown to zero(0) instead of minus-one (-1). Defaults to FALSE. + * @param foreigndali The IP address of the foreign dali used to resolve the file. If blank then the file is resolved + * locally. Defaults to blank. + * @param maxFileLimit Maximum number of files to return. Set to -1 to use server default limit (100,000). + * Maximum client-side limit is 1,000,000. Defaults to -1. + * @return A record containing: + * - count: Number of files returned + * - limitBreached: TRUE if more files match than the limit allows + * - files: Dataset of file information records + * + * Example usage: + * IMPORT Std; + * + * // List all files + * result1 := Std.File.LogicalFileListFiltered(); + * OUTPUT(result1.count); + * OUTPUT(result1.files); + * + * // List with limit + * result2 := Std.File.LogicalFileListFiltered(maxFileLimit := 100); + * + * // List files > 100MB + * result3 := Std.File.LogicalFileListFiltered(filters := 'size>100000000'); + * + * // List superfiles owned by jsmith + * result4 := Std.File.LogicalFileListFiltered(filters := 'owner:jsmith is:superfile'); + * + * // List normal files without description, modified recently + * result5 := Std.File.LogicalFileListFiltered(filters := 'is:normal !has:description modified>2024-01-01'); + */ +EXPORT lib_fileservices.FsLogicalFileListResult LogicalFileListFiltered(varstring namepattern='*', varstring filters='', varstring fields='', boolean unknownszero=FALSE, varstring foreigndali='', integer8 maxFileLimit=-1) := + lib_fileservices.FileServices.LogicalFileListFiltered(namepattern, filters, fields, unknownszero, foreigndali, maxFileLimit); /** * Compares two files, and returns a result indicating how well they match. @@ -485,8 +570,8 @@ EXPORT SprayVariable(varstring sourceIP='', varstring sourcePath, integer4 sourc * @param recordStructurePresent If TRUE derives the record structure from the header of the file. * @param quotedTerminator Can the terminator character be included in a quoted field. Defaults to TRUE. * If FALSE it allows quicker partitioning of the file (avoiding a complete file scan). - * @param encoding A null-terminated string containing the encoding. - * Can be set to one of the following: + * @param encoding A null-terminated string containing the encoding. + * Can be set to one of the following: * ascii, utf8, utf8n, utf16, utf16le, utf16be, utf32, utf32le,utf32be. If omitted, the default is ascii. * @param expireDays Number of days to auto-remove file. Default is -1, not expire. * @param dfuServerQueue Name of target DFU Server queue. Default is '' (empty) for the first DFU queue in the environment. @@ -1108,19 +1193,19 @@ EXPORT dataset(FsLandingZoneRecord) GetLandingZones() := * */ -EXPORT integer4 GetExpireDays(varstring lfn) := +EXPORT integer4 GetExpireDays(varstring lfn) := lib_fileservices.Fileservices.GetExpireDays(lfn); - + /** * Set the expire days property of the specified logical filename. * * @param lfn The name of the logical file. * @param expireDays Number of days before file expired. 0 means using system expire value. - * + * */ -EXPORT SetExpireDays(varstring lfn, integer4 expireDays) := +EXPORT SetExpireDays(varstring lfn, integer4 expireDays) := lib_fileservices.Fileservices.SetExpireDays(lfn, expireDays); /** @@ -1130,7 +1215,7 @@ EXPORT SetExpireDays(varstring lfn, integer4 expireDays) := * */ -EXPORT ClearExpireDays(varstring lfn) := +EXPORT ClearExpireDays(varstring lfn) := lib_fileservices.Fileservices.ClearExpireDays(lfn); END; \ No newline at end of file diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index 3e9b0f2bb6a..5f7be3cff9f 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -32,10 +32,14 @@ #include "rmtsmtp.hpp" #include "dfuplus.hpp" #include "daclient.hpp" +#include "dadfs.hpp" #include "dasds.hpp" #include "enginecontext.hpp" #include "environment.hpp" #include "ws_dfsclient.hpp" +#include "rtlfield.hpp" +#include "eclrtl.hpp" +#include "rtlds_imp.hpp" #define USE_DALIDFS #define SDS_LOCK_TIMEOUT 10000 @@ -2323,8 +2327,10 @@ FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx, siz throw e.getClear(); } MemoryBuffer mb; - if (!mask||!*mask) - mask ="*"; + if (isEmptyString(mask)) + mask = "*"; + else if (*mask == '~') + mask++; // Strip leading ~ if present, as internal APIs expect it without StringBuffer masklower(mask); masklower.toLowerCase(); @@ -2385,6 +2391,652 @@ FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx, siz } +/* + * Field source implementation for building file list result rows using IFieldSource pattern. + */ +class FileListResultFieldSource : public CInterfaceOf +{ +private: + unsigned count = 0; + bool limitBreached = false; + Owned iter; + IPropertyTree *currentFile; + bool unknownsZero = false; + StringBuffer tempStr; + +public: + FileListResultFieldSource(unsigned _count, bool _limitBreached, IPropertyTreeIterator *_iter, bool _unknownsZero) + : count(_count), limitBreached(_limitBreached), iter(_iter), unknownsZero(_unknownsZero), + currentFile(nullptr) + { + dbgassertex(iter); + } + virtual bool getBooleanResult(const RtlFieldInfo *field) override + { + // Parent level field: limitBreached + if (streq(field->name, "limitbreached")) + return limitBreached; + + // Child level field: superfile + if (streq(field->name, "superfile")) + { + int numsub = currentFile->getPropInt("@numsubfiles", -1); + return numsub >= 0; + } + + // Child level field: compressed + if (streq(field->name, "compressed")) + return currentFile->getPropBool("@compressed", false); + + // Child level field: persistent + if (streq(field->name, "persistent")) + return currentFile->getPropBool("@persistent", false); + + throw makeStringExceptionV(-1, "FileListResultFieldSource: Unexpected boolean field '%s'", field->name); + } + virtual void getDataResult(const RtlFieldInfo *field, size32_t &len, void * &result) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No data fields expected (field '%s')", field->name); + } + virtual double getRealResult(const RtlFieldInfo *field) override + { + // Child level field: readcost + if (streq(field->name, "readcost")) + return currentFile->getPropReal("@readCost", 0.0); + + // Child level field: writecost + if (streq(field->name, "writecost")) + return currentFile->getPropReal("@writeCost", 0.0); + + throw makeStringExceptionV(-1, "FileListResultFieldSource: Unexpected real field '%s'", field->name); + } + virtual __int64 getSignedResult(const RtlFieldInfo *field) override + { + // Child level field: size + if (streq(field->name, "size")) + { + __int64 fsz = currentFile->getPropInt64("@size", -1); + int numsub = currentFile->getPropInt("@numsubfiles", -1); + if ((fsz == -1) && (unknownsZero || (numsub == 0))) + fsz = 0; + return fsz; + } + + // Child level field: rowcount + if (streq(field->name, "rowcount")) + { + __int64 i64 = currentFile->getPropInt64("@recordCount", -1); + if (i64 == -1) + { + __int64 fsz = currentFile->getPropInt64("@size", -1); + if (fsz != -1) + { + int rsz = currentFile->getPropInt("@recordSize", 0); + if (rsz > 0) + i64 = fsz / rsz; + } + } + int numsub = currentFile->getPropInt("@numsubfiles", -1); + if ((i64 == -1) && (unknownsZero || (numsub == 0))) + i64 = 0; + return i64; + } + + // Child level field: recordsize + if (streq(field->name, "recordsize")) + return currentFile->getPropInt("@recordSize", 0); + + // Child level field: compressedsize + if (streq(field->name, "compressedsize")) + return currentFile->getPropInt64("@compressedSize", 0); + + // Child level field: expiredays + if (streq(field->name, "expiredays")) + return currentFile->getPropInt("@expireDays", 0); + + throw makeStringExceptionV(-1, "FileListResultFieldSource: Unexpected signed field '%s'", field->name); + } + virtual unsigned __int64 getUnsignedResult(const RtlFieldInfo *field) override + { + // Parent level field: count + if (streq(field->name, "count")) + return count; + + throw makeStringExceptionV(-1, "FileListResultFieldSource: Unexpected unsigned field '%s'", field->name); + } + virtual void getStringResult(const RtlFieldInfo *field, size32_t &len, char * &result) override + { + tempStr.clear(); + if (streq(field->name, "name")) // Child level field: name + { + const char *name = currentFile->queryProp("@name"); + if (!isEmptyString(name)) + tempStr.append(name); + } + else if (streq(field->name, "modified")) // Child level field: modified + { + currentFile->getProp("@modified", tempStr); + tempStr.padTo(19); + } + else if (streq(field->name, "owner")) + currentFile->getProp("@owner", tempStr); + else if (streq(field->name, "cluster")) // Child level field: cluster + currentFile->getProp("@group", tempStr); + else if (streq(field->name, "description")) + currentFile->getProp("@description", tempStr); + else if (streq(field->name, "workunit")) + currentFile->getProp("@workunit", tempStr); + else if (streq(field->name, "job")) + currentFile->getProp("@job", tempStr); + else if (streq(field->name, "directory")) + currentFile->getProp("@directory", tempStr); + else if (streq(field->name, "kind")) + currentFile->getProp("@kind", tempStr); + else if (streq(field->name, "protect")) + currentFile->getProp("@protect", tempStr); + else if (streq(field->name, "accessed")) + { + currentFile->getProp("@accessed", tempStr); + tempStr.padTo(19); + } + else + throw makeStringExceptionV(-1, "FileListResultFieldSource: Unexpected string field '%s'", field->name); + + len = tempStr.length(); + result = tempStr.detach(); + } + virtual void getUTF8Result(const RtlFieldInfo *field, size32_t &chars, char * &result) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No UTF8 fields expected (field '%s')", field->name); + } + virtual void getUnicodeResult(const RtlFieldInfo *field, size32_t &chars, UChar * &result) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No Unicode fields expected (field '%s')", field->name); + } + virtual void getDecimalResult(const RtlFieldInfo *field, Decimal &value) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No decimal fields expected (field '%s')", field->name); + } + virtual void processBeginSet(const RtlFieldInfo * field, bool &isAll) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No set fields expected (field '%s')", field->name); + } + virtual void processBeginDataset(const RtlFieldInfo * field) override + { + // Starting to iterate the 'files' dataset + currentFile = nullptr; + } + virtual void processBeginRow(const RtlFieldInfo * field) override + { + // Beginning a new row in the files dataset - currentFile is already set by processNextRow + } + virtual bool processNextSet(const RtlFieldInfo * field) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No set fields expected (field '%s')", field->name); + } + virtual bool processNextRow(const RtlFieldInfo * field) override + { + // Move to next file in the dataset + while (iter->next()) + { + currentFile = &iter->query(); + const char *name = currentFile->queryProp("@name"); + // Skip files without names + if (!isEmptyString(name)) // probably can never happen + return true; + } + return false; + } + virtual void processEndSet(const RtlFieldInfo * field) override + { + throw makeStringExceptionV(-1, "FileListResultFieldSource: No set fields expected (field '%s')", field->name); + } + virtual void processEndDataset(const RtlFieldInfo * field) override + { + // Finished iterating the 'files' dataset + currentFile = nullptr; + } + virtual void processEndRow(const RtlFieldInfo * field) override + { + // Finished processing current row - no cleanup needed + } +}; + +static bool validateFileField(const char *requestedField, StringBuffer &attrName, DFUQResultField &fieldEnum, DFUQResultFieldType &fieldType) +{ + // Map of ECL field name aliases to canonical internal field names + // This handles user-friendly names and their mappings + static const std::unordered_map fieldAliases = + { + {"superfile", "numsubfiles"}, // Derived from numsubfiles + {"rowcount", "recordcount"}, // ECL field name is "rowcount" but internal attribute is "recordCount" + {"cluster", "group"} // ECL field name is "cluster" but internal attribute is "group" + }; + + // Trim whitespace + StringBuffer fieldName(requestedField); + fieldName.trim(); + if (isEmptyString(fieldName)) + return false; + + auto it = fieldAliases.find(fieldName.str()); + if (it != fieldAliases.end()) + fieldName.set(it->second); + + return getFileAttributePath(fieldName, attrName, fieldEnum, fieldType); +} + +/* + * Parse user-friendly filter syntax and translate to internal DFUQFilter format + * + * User field names are simple enum-based names (e.g., "name", "size", "owner", "recordcount") + * that are validated against the DFUQResultField enum and automatically converted to the + * appropriate internal attribute names (e.g., "@name", "@size", "@owner", "@recordCount"). + * Invalid field names will cause an exception to be thrown. + * + * Valid user field names include: + * name, description, nodegroups, kind, timemodified, job, owner, recordcount, recordsize, + * size, workunit, nodegroup, numsubfiles, accessed, numparts, compressedsize, directory, + * partmask, superowners, persistent, protect, iscompressed, cost, numDiskReads, numDiskWrites, + * atRestCost, accessCost, maxSkew, minSkew, maxSkewPart, minSkewPart, readCost, writeCost, + * expireDays, subfilenames, blockCompressed, rowCompressed + * + * User syntax examples: + * owner:jsmith → WILDCARD|@owner|jsmith| + * owner:*smith → WILDCARD|@owner|*smith| + * size>1000000 → INTEGER64_RANGE|@size|1000000|9223372036854775807| + * size<1000 → INTEGER64_RANGE|@size|0|1000| + * size>=1000 → INTEGER64_RANGE|@size|1000|9223372036854775807| + * rowcount=0 → INTEGER64_RANGE|@recordCount|0|0| + * has:description → HAS_PROPERTY|@description|true| + * !has:description → HAS_PROPERTY|@description|false| + * is:superfile → SPECIAL|2|2| + * is:normal → SPECIAL|2|3| + * timemodified>2024-01-01 → STRING_RANGE|@modified|2024-01-01|9999-12-31| + * timemodified<2024-12-31 → STRING_RANGE|@modified|0000-00-00|2024-12-31| + * + * Multiple filters separated by commas + */ +static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internalFilter) +{ + if (isEmptyString(userFilter)) + return; + + StringArray terms; + terms.appendList(userFilter, ","); + + ForEachItemIn(i, terms) + { + const char *term = terms.item(i); + if (isEmptyString(term)) + continue; + + // Save original term for error messages + const char *originalTerm = term; + + // Check for negation prefix + bool negate = (term[0] == '!'); + if (negate) + { + term++; + if (isEmptyString(term)) + throw makeStringException(-1, "Invalid filter syntax: '!' must be followed by a filter term"); + } + + bool matched = false; + + // Parse has:property + if (strncmp(term, "has:", 4) == 0) + { + const char *prop = term + 4; + if (isEmptyString(prop)) + throw makeStringException(-1, "Invalid filter syntax: 'has:' requires a property name (e.g., 'has:description')"); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(prop, attrName, field, fieldType)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, prop); + + internalFilter.appendf("%u%c%s%c%s%c", + DFUQFThasProp, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + negate ? "false" : "true", DFUQFilterSeparator); + matched = true; + } + // Parse is:filetype + else if (strncmp(term, "is:", 3) == 0) + { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating 'is:' is not supported"); + + const char *fileType = term + 3; + if (isEmptyString(fileType)) + throw makeStringException(-1, "Invalid filter syntax: 'is:' requires a file type (superfile, normal, or any)"); + + if (stricmp(fileType, "superfile") == 0) + { + internalFilter.appendf("%u%c2%c2%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQFilterSeparator, DFUQFilterSeparator); + matched = true; + } + else if (stricmp(fileType, "normal") == 0) + { + internalFilter.appendf("%u%c2%c3%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQFilterSeparator, DFUQFilterSeparator); + matched = true; + } + else if (stricmp(fileType, "any") == 0) + { + internalFilter.appendf("%u%c2%c1%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQFilterSeparator, DFUQFilterSeparator); + matched = true; + } + else + throw makeStringExceptionV(-1, "Invalid filter syntax: 'is:%s' - must be superfile, normal, or any", fileType); + } + // Parse field:value (wildcard match) + else if (const char *colon = strchr(term, ':')) + { + StringBuffer fieldName; + fieldName.append(colon - term, term).trim(); + StringBuffer valueStr(colon + 1); + valueStr.trim(); + const char *value = valueStr.str(); + + if (fieldName.length() == 0) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - field name required before ':'", originalTerm); + if (isEmptyString(value)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value required after ':'", originalTerm); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName.str(), attrName, field, fieldType)) + throw makeStringException(-1, VStringBuffer("Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, fieldName.str()).str()); + + internalFilter.appendf("%u%c%s%c%s%c", + DFUQFTwildcardMatch, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator); + matched = true; + } + // Parse field>value, field=value, field<=value + else if (const char *op = strpbrk(term, "><")) + { + StringBuffer fieldName; + fieldName.append(op - term, term).trim(); + + if (fieldName.length() == 0) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - field name required before comparison operator", originalTerm); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName.str(), attrName, field, fieldType)) + throw makeStringException(-1, VStringBuffer("Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, fieldName.str()).str()); + + // Determine operator + bool hasEquals = (op[1] == '='); + StringBuffer valueStr(hasEquals ? (op + 2) : (op + 1)); + valueStr.trim(); + const char *value = valueStr.str(); + + if (isEmptyString(value)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value required after comparison operator", originalTerm); + + // Check if field is numeric/float type + bool isNumeric = (fieldType == DFUQResultFieldType::numericType); + bool isFloat = (fieldType == DFUQResultFieldType::floatType); + + if (isNumeric || isFloat) + { + // Parse numeric range + if (op[0] == '>') + { + // field > value or field >= value + __int64 minVal = _atoi64(value); + if (!hasEquals) + { + if (minVal == I64C(0x7FFFFFFFFFFFFFFF)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value too large for > comparison (would overflow)", originalTerm); + minVal++; + } + internalFilter.appendf("%u%c%s%c%lld%c%lld%c", + DFUQFTinteger64Range, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + minVal, DFUQFilterSeparator, I64C(0x7FFFFFFFFFFFFFFF), DFUQFilterSeparator); + } + else // op[0] == '<' + { + // field < value or field <= value + __int64 maxVal = _atoi64(value); + if (!hasEquals) + { + if (maxVal == (-I64C(0x7FFFFFFFFFFFFFFF) - 1)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value too small for < comparison (would underflow)", originalTerm); + maxVal--; + } + internalFilter.appendf("%u%c%s%c0%c%lld%c", + DFUQFTinteger64Range, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + DFUQFilterSeparator, maxVal, DFUQFilterSeparator); + } + } + else + { + // Parse string range (for dates, text, etc.) + if (op[0] == '>') + { + internalFilter.appendf("%u%c%s%c%s%c~~~~~~~~~~%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator, DFUQFilterSeparator); + } + else // op[0] == '<' + { + internalFilter.appendf("%u%c%s%c%c%s%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + DFUQFilterSeparator, value, DFUQFilterSeparator); + } + } + matched = true; + } + // Parse field=value as exact range match + else if (const char *eq = strchr(term, '=')) + { + StringBuffer fieldName; + fieldName.append(eq - term, term).trim(); + StringBuffer valueStr(eq + 1); + valueStr.trim(); + const char *value = valueStr.str(); + + if (fieldName.length() == 0) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - field name required before '='", originalTerm); + if (isEmptyString(value)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value required after '='", originalTerm); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName.str(), attrName, field, fieldType)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, fieldName.str()); + + // Check if numeric or float field + bool isNumeric = (fieldType == DFUQResultFieldType::numericType); + bool isFloat = (fieldType == DFUQResultFieldType::floatType); + + if (isNumeric || isFloat) + { + __int64 val = _atoi64(value); + internalFilter.appendf("%u%c%s%c%lld%c%lld%c", + DFUQFTinteger64Range, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + val, DFUQFilterSeparator, val, DFUQFilterSeparator); + } + else + { + // Exact string match via range + internalFilter.appendf("%u%c%s%c%s%c%s%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator, value, DFUQFilterSeparator); + } + matched = true; + } + + // If nothing matched, throw an error + if (!matched) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unrecognized filter format. Use field:value, field>value, field=value, has:property, or is:filetype", originalTerm); + } +} + +FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *foreigndali, __int64 maxFileLimit) +{ + IEngineContext *engineCtx = ctx->queryEngineContext(); + if (engineCtx && !engineCtx->allowDaliAccess()) + { + Owned e = makeStringException(-1, "FileServices.LogicalFileListFiltered cannot access Dali in this context - this normally means it is being called from a thor slave"); + EXCLOG(e, NULL); + throw e.getClear(); + } + + // Validate client-side max limit (1 million) + constexpr __int64 CLIENT_MAX_LIMIT = 1000000; + if (maxFileLimit > CLIENT_MAX_LIMIT) + { + Owned e = makeStringExceptionV(-1, "FileServices.LogicalFileListFiltered: maxFileLimit (%lld) exceeds client maximum of %lld", maxFileLimit, CLIENT_MAX_LIMIT); + EXCLOG(e, NULL); + throw e.getClear(); + } + + if (isEmptyString(mask)) + mask = "*"; + else if (*mask == '~') + mask++; // Strip leading ~ if present, as internal APIs expect it without + StringBuffer masklower(mask); + masklower.toLowerCase(); + + Owned foreignNode; + if (!isEmptyString(foreigndali)) + { + SocketEndpoint ep(foreigndali); + foreignNode.setown(createINode(ep)); + } + + // Build filter string - translate user-friendly syntax to internal format + StringBuffer filterBuf; + + // Parse user-provided filters (using friendly syntax like "owner:jsmith size>1000") + parseUserFilterSyntax(filters, filterBuf); + + // Append system filters: name pattern and max files limit + filterBuf.appendf("%u%c%u%c%s%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQSFFileNameWithPrefix, DFUQFilterSeparator, + masklower.str(), DFUQFilterSeparator); + + // Add max files limit if specified (and not -1 which means use server default) + if (maxFileLimit > 0) + { + filterBuf.appendf("%u%c%u%c%lld%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQSFMaxFiles, DFUQFilterSeparator, + maxFileLimit, DFUQFilterSeparator); + } + + // Parse and validate requested fields + std::vector fields; + StringArray requestedFieldNames; + + if (isEmptyString(requestedFields)) + { + requestedFieldNames.append("name"); + requestedFieldNames.append("superfile"); + requestedFieldNames.append("size"); + requestedFieldNames.append("rowcount"); + requestedFieldNames.append("modified"); + requestedFieldNames.append("owner"); + requestedFieldNames.append("cluster"); + } + else + { + // Parse comma-separated field list + requestedFieldNames.appendList(requestedFields, ","); + + // Ensure "name" is always included (required field) + bool hasName = false; + ForEachItemIn(idx, requestedFieldNames) + { + if (stricmp(requestedFieldNames.item(idx), "name") == 0) + { + hasName = true; + break; + } + } + if (!hasName) + requestedFieldNames.append("name"); + } + + // Validate field names and build field list for getDFAttributesFilteredIterator + ForEachItemIn(idx, requestedFieldNames) + { + // Trim whitespace + const char *fieldName = requestedFieldNames.item(idx); + if (isEmptyString(fieldName)) + continue; + + // Validate field name using existing validation function + StringBuffer attrPath; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName, attrPath, field, fieldType)) + throw makeStringExceptionV(-1, "FileServices.LogicalFileListFiltered: Invalid field name '%s'", fieldName); + + // Add field to list if not already present + if (std::find(fields.begin(), fields.end(), field) == fields.end()) + fields.push_back(field); + } + + // Always include numsubfiles for superfile detection + if (std::find(fields.begin(), fields.end(), DFUQResultField::numsubfiles) == fields.end()) + fields.push_back(DFUQResultField::numsubfiles); + + // Add terminator + fields.push_back(DFUQResultField::term); + + bool allMatchingFilesReceived = false; + unsigned count = 0; + Owned iter = queryDistributedFileDirectory().getDFAttributesFilteredIterator( + filterBuf.str(), + nullptr, // no local filters + fields.data(), // requested fields + ctx->queryUserDescriptor(), + true, // recursive + allMatchingFilesReceived, + &count, + foreignNode + ); + + // Build result row using IFieldSource pattern + RtlDynamicRowBuilder resultBuilder(*_rowAllocator); + Owned fieldSource = new FileListResultFieldSource(count, !allMatchingFilesReceived, iter.getClear(), unknownszero); + + const RtlTypeInfo *typeInfo = _rowAllocator->queryOutputMeta()->queryTypeInfo(); + RtlFieldStrInfo dummyField("", NULL, typeInfo); + size32_t len = typeInfo->build(resultBuilder, 0, &dummyField, *fieldSource); + + return (const byte *)resultBuilder.finalizeRowClear(len); +} + FILESERVICES_API void FILESERVICES_CALL fsSuperFileContents(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *lsuperfn, bool recurse) { MemoryBuffer mb; diff --git a/plugins/fileservices/fileservices.hpp b/plugins/fileservices/fileservices.hpp index 712bcbfff53..29b925be645 100644 --- a/plugins/fileservices/fileservices.hpp +++ b/plugins/fileservices/fileservices.hpp @@ -153,6 +153,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsGetFileDescription(ICodeContext *ct FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub); FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory_v2(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub, const char *planename); FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *mask, bool includenormal, bool includesuper, bool unknownszero,const char *foreigndali); +FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *foreigndali, __int64 maxFileLimit); FILESERVICES_API void FILESERVICES_CALL fsSuperFileContents(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lsuperlfn, bool recurse); FILESERVICES_API void FILESERVICES_CALL fsLogicalFileSuperOwners(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lfn); FILESERVICES_API char * FILESERVICES_CALL fsExternalLogicalFileName(const char *location, const char *path,bool abspath); diff --git a/plugins/proxies/lib_fileservices.ecllib b/plugins/proxies/lib_fileservices.ecllib index 8342c87f8fb..3bafad14665 100644 --- a/plugins/proxies/lib_fileservices.ecllib +++ b/plugins/proxies/lib_fileservices.ecllib @@ -21,6 +21,23 @@ export FsFilenameRecord := record string name{maxlength(1023)}; integer8 size; s export FsLogicalFileName := string{maxlength(255)}; export FsLogicalFileNameRecord := record FsLogicalFileName name; end; export FsLogicalFileInfoRecord := record(FsLogicalFileNameRecord) boolean superfile; integer8 size; integer8 rowcount; string19 modified; string owner{maxlength(255)}; string cluster{maxlength(255)}; end; +export FsLogicalFileInfoRecordEx := record(FsLogicalFileInfoRecord) + varstring description; + varstring workunit; + varstring job; + varstring directory; + boolean compressed; + integer4 recordsize; + varstring kind; + integer8 compressedsize; + boolean persistent; + varstring protect; + string19 accessed; + integer4 expiredays; + real8 readcost; + real8 writecost; +end; +export FsLogicalFileListResult := record unsigned4 count; boolean limitBreached; dataset(FsLogicalFileInfoRecordEx) files; end; export FsLogicalSuperSubRecord := record string supername{maxlength(255)}; string subname{maxlength(255)}; end; export FsFileRelationshipRecord := record string primaryfile {maxlength(1023)}; string secondaryfile {maxlength(1023)}; string primaryflds {maxlength(1023)}; string secondaryflds {maxlength(1023)}; string kind {maxlength(16)}; string cardinality {maxlength(16)}; boolean payload; string description {maxlength(1023)}; end; export integer4 RECFMV_RECSIZE := -2; // special value for SprayFixed record size @@ -79,6 +96,7 @@ export FileServices := SERVICE : plugin('fileservices'), time SetFileDescription(const varstring lfn,const varstring val) : c,action,context,entrypoint='fsSetFileDescription'; dataset(FsFilenameRecord) RemoteDirectory(const varstring machineIP,const varstring dir,const varstring mask='*',boolean sub=false,const varstring planeName='') : c,context,entrypoint='fsRemoteDirectory_v2'; dataset(FsLogicalFileInfoRecord) LogicalFileList(const varstring namepattern='*',boolean includenormal=true,boolean includesuper=false,boolean unknownszero=false,const varstring foreigndali='') : c,context,entrypoint='fsLogicalFileList'; + _linkcounted_ ROW(FsLogicalFileListResult) LogicalFileListFiltered(const varstring namepattern='*',const varstring filters='',const varstring fields='',boolean unknownszero=false,const varstring foreigndali='',integer8 maxFileLimit=-1) : c,context,entrypoint='fsLogicalFileListFiltered'; dataset(FsLogicalFileNameRecord) SuperFileContents(const varstring lsuperfn,boolean recurse=false) : c,context,entrypoint='fsSuperFileContents'; dataset(FsLogicalFileNameRecord) LogicalFileSuperOwners(const varstring lfn) : c,context,entrypoint='fsLogicalFileSuperOwners'; varstring ExternalLogicalFileName(const varstring location, const varstring path,boolean abspath=true) : c,entrypoint='fsExternalLogicalFileName'; diff --git a/testing/regress/ecl/key/logicalfilelist_filtered_test.xml b/testing/regress/ecl/key/logicalfilelist_filtered_test.xml new file mode 100644 index 00000000000..716b26cdbcd --- /dev/null +++ b/testing/regress/ecl/key/logicalfilelist_filtered_test.xml @@ -0,0 +1,186 @@ + + === Creating Test Files === + + + + + + + + + + + + + + + + + === Test Files Created === + + + Test 1: Basic usage (all test files) + + + 8 + + + false + + + 8 + + + Test 2: With file limit and breach detection + + + 5 + + + true + + + true + + + 5 + + + Test 3: Pattern matching (testfile*) + + + 2 + + + 2 + + + Test 4: SuperFiles only + + + 1 + + + 1 + + + Test 5: Normal files only (not superfiles) + + + 7 + + + 7 + + + Test 6: Record count range (100 <= rowcount < 1000) + + + 6 + + + false + + + 6 + + + Test 7: Combined filters (200 <= rowcount <= 500 AND normal files) + + + 4 + + + false + + + 4 + + + Test 8: Modified before 2026-02-12 (should exclude all test files) + + + 0 + + + true + + + 0 + + + Test 9: Files not in superfiles (!has:SuperOwners) + + + 6 + + + true + + + 6 + + + Test 10: Default fields (backward compatible) + + + 8 + + + true + + + file130400100false + file260800200false + file3152000500false + file43040001000false + file56080002000false + testfile191200300false + testfile2121600400false + super191200300true + + + Test 11: Name field auto-added (requested only size) + + + 8 + + + true + + + file130400 + file260800 + file3152000 + file4304000 + file5608000 + testfile191200 + testfile2121600 + super191200 + + + Test 12: Custom fields + filters combined + + + 6 + + + true + + + file2false60800 + file3false152000 + file4false304000 + file5false608000 + testfile1false91200 + testfile2false121600 + + + === Test Suite Complete === + + + === Cleaning Up Test Files === + + + === Cleanup Complete === + diff --git a/testing/regress/ecl/logicalfilelist_filtered_test.ecl b/testing/regress/ecl/logicalfilelist_filtered_test.ecl new file mode 100644 index 00000000000..15eae48cd62 --- /dev/null +++ b/testing/regress/ecl/logicalfilelist_filtered_test.ecl @@ -0,0 +1,285 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2026 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//nothor +//noroxie + +import $.setup; +prefix := setup.Files(false, false).QueryFilePrefix; + +IMPORT Std; + +// Test data record structure +TestRec := RECORD + UNSIGNED4 id; + STRING100 name; + STRING200 padding; +END; + +// Create test data with varying sizes +createTestData(UNSIGNED4 numRecords) := DATASET(numRecords, TRANSFORM(TestRec, + SELF.id := COUNTER, + SELF.name := 'Test record number ' + (STRING)COUNTER, + SELF.padding := 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' +)); + +// Setup: Create test files with various properties +setupFiles := SEQUENTIAL( + OUTPUT('=== Creating Test Files ==='), + + Std.File.DeleteSuperFile(prefix + 'super1'), + + // Small files with varying record counts + OUTPUT(createTestData(100), , prefix + 'file1', EXPIRE(1), OVERWRITE), + OUTPUT(createTestData(200), , prefix + 'file2', EXPIRE(1), OVERWRITE), + OUTPUT(createTestData(500), , prefix + 'file3', EXPIRE(1), OVERWRITE), + + // Medium record count files + OUTPUT(createTestData(1000), , prefix + 'file4', EXPIRE(1), OVERWRITE), + OUTPUT(createTestData(2000), , prefix + 'file5', EXPIRE(1), OVERWRITE), + + // Test-prefixed files for pattern matching + OUTPUT(createTestData(300), , prefix + 'testfile1', EXPIRE(1), OVERWRITE), + OUTPUT(createTestData(400), , prefix + 'testfile2', EXPIRE(1), OVERWRITE), + + // Create a superfile for testing + Std.File.CreateSuperFile(prefix + 'super1'), + Std.File.AddSuperFile(prefix + 'super1', prefix + 'file1'), + Std.File.AddSuperFile(prefix + 'super1', prefix + 'file2'), + + OUTPUT('=== Test Files Created ===') +); + +// Cleanup: Delete test files +cleanupFiles := SEQUENTIAL( + OUTPUT('=== Cleaning Up Test Files ==='), + + Std.File.DeleteSuperFile(prefix + 'super1'), + Std.File.DeleteLogicalFile(prefix + 'file1', true), + Std.File.DeleteLogicalFile(prefix + 'file2', true), + Std.File.DeleteLogicalFile(prefix + 'file3', true), + Std.File.DeleteLogicalFile(prefix + 'file4', true), + Std.File.DeleteLogicalFile(prefix + 'file5', true), + Std.File.DeleteLogicalFile(prefix + 'testfile1', true), + Std.File.DeleteLogicalFile(prefix + 'testfile2', true), + + Std.File.DeleteSuperFile(prefix + 'super1'), + + OUTPUT('=== Cleanup Complete ===') +); + +// Test 1: Basic usage - list files in our test scope +test1 := MODULE + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*'); + EXPORT name := 'Test 1: Basic usage (all test files)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test1_Count')), + OUTPUT(result.limitBreached, NAMED('Test1_LimitBreached')), + OUTPUT(COUNT(result.files), NAMED('Test1_FileCount')) + ); +END; + +// Test 2: Usage with explicit limit +test2 := MODULE + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', maxFileLimit := 5); + EXPORT name := 'Test 2: With file limit and breach detection'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test2_Count')), + OUTPUT(result.limitBreached, NAMED('Test2_LimitBreached')), + OUTPUT(COUNT(result.files) <= 5, NAMED('Test2_LimitRespected')), + OUTPUT(COUNT(result.files), NAMED('Test2_ActualFileCount')) + ); +END; + +// Test 3: Pattern matching +test3 := MODULE + EXPORT result := Std.File.LogicalFileListFiltered(prefix + 'testfile*'); + EXPORT name := 'Test 3: Pattern matching (testfile*)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test3_Count')), + OUTPUT(COUNT(result.files), NAMED('Test3_FileCount')) + ); +END; + +// Test 4: SuperFiles only +test4 := MODULE + EXPORT filters := 'is:superfile'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 4: SuperFiles only'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test4_Count')), + OUTPUT(COUNT(result.files), NAMED('Test4_FileCount')) + ); +END; + +// Test 5: Normal files only (not superfiles) +test5 := MODULE + EXPORT filters := 'is:normal'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 5: Normal files only (not superfiles)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test5_Count')), + OUTPUT(COUNT(result.files), NAMED('Test5_FileCount')) + ); +END; + +// Test 6: Filter - record count range (should exclude file4=1000 and file5=2000) +test6 := MODULE + EXPORT filters := 'rowcount>=100,rowcount<1000'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 6: Record count range (100 <= rowcount < 1000)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test6_Count')), + OUTPUT(result.count = 5, NAMED('Test6_ExpectedCount')), // Should match: file1(100), file2(200), file3(500), testfile1(300), testfile2(400) + OUTPUT(COUNT(result.files), NAMED('Test6_FileCount')) + ); +END; + +// Test 7: Combined filters - record count with normal files (excludes super1 and large files) +test7 := MODULE + EXPORT filters := 'rowcount>=200,rowcount<=500,is:normal'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 7: Combined filters (200 <= rowcount <= 500 AND normal files)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test7_Count')), + OUTPUT(result.count = 3, NAMED('Test7_ExpectedCount')), // Should match: file2(200), file3(500), testfile1(300), testfile2(400) = 4, but super1 excluded + OUTPUT(COUNT(result.files), NAMED('Test7_FileCount')) + ); +END; + +// Test 8: Filter - date range excludes all recent files (proves date filtering works) +test8 := MODULE + EXPORT filters := 'modified<2026-02-12'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 8: Modified before 2026-02-12 (should exclude all test files)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test8_Count')), + OUTPUT(result.count = 0, NAMED('Test8_ExpectedZero')), // All test files created in 2026, so none match + OUTPUT(COUNT(result.files), NAMED('Test8_FileCount')) + ); +END; + +// Test 9: Filter - files NOT in superfiles (should exclude file1 and file2 which are in super1) +test9 := MODULE + EXPORT filters := '!has:SuperOwners'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); + EXPORT name := 'Test 9: Files not in superfiles (!has:SuperOwners)'; + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test9_Count')), + OUTPUT(result.count = 6, NAMED('Test9_ExpectedCount')), // Should be 6: file3,file4,file5,testfile1,testfile2,super1 (excludes file1,file2) + OUTPUT(COUNT(result.files), NAMED('Test9_FileCount')) + ); +END; + +// Test 10: Empty fields parameter returns default 7 fields +test10 := MODULE + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', fields := ''); + EXPORT name := 'Test 10: Default fields (backward compatible)'; + // Strip prefix from names and remove modified/cluster fields to avoid variability + EXPORT filesNormalized := TABLE(result.files, { + STRING name := name[LENGTH(prefix)..]; + UNSIGNED8 size := size; + UNSIGNED8 rowcount := rowcount; + BOOLEAN superfile := superfile; + }); + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test10_Count')), + OUTPUT(result.count = 8, NAMED('Test10_AllFilesReturned')), + OUTPUT(filesNormalized, NAMED('Test10_Files')) + ); +END; + +// Test 11: Minimal fields - name is mandatory even if not specified +test11 := MODULE + EXPORT fields := 'size'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', fields := fields); + EXPORT name := 'Test 11: Name field auto-added (requested only size)'; + // Strip prefix from names and remove modified/cluster fields to avoid variability + EXPORT filesNormalized := TABLE(result.files, { + STRING name := name[LENGTH(prefix)..]; + UNSIGNED8 size := size; + }); + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test11_Count')), + OUTPUT(result.count = 8, NAMED('Test11_AllFilesReturned')), + OUTPUT(filesNormalized, NAMED('Test11_Files')) + ); +END; + +// Test 12: Combine custom fields with filters (tests both features together) +test12 := MODULE + EXPORT fields := 'name,superfile,size'; + EXPORT filters := 'is:normal,rowcount>=200'; + EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters, fields := fields); + EXPORT name := 'Test 12: Custom fields + filters combined'; + // Strip prefix from names and remove modified/cluster fields to avoid variability + EXPORT filesNormalized := TABLE(result.files, { + STRING name := name[LENGTH(prefix)..]; + BOOLEAN superfile := superfile; + UNSIGNED8 size := size; + }); + EXPORT verify := SEQUENTIAL( + OUTPUT(result.count, NAMED('Test12_Count')), + OUTPUT(result.count = 6, NAMED('Test12_ExpectedCount')), // file2(200), file3(500), file4(1000), file5(2000), testfile1(300), testfile2(400) - all normal files >= 200 + OUTPUT(filesNormalized, NAMED('Test12_Files')) + ); +END; + +// Run all tests with setup and cleanup +SEQUENTIAL( + setupFiles, + + OUTPUT(test1.name), + test1.verify, + + OUTPUT(test2.name), + test2.verify, + + OUTPUT(test3.name), + test3.verify, + + OUTPUT(test4.name), + test4.verify, + + OUTPUT(test5.name), + test5.verify, + + OUTPUT(test6.name), + test6.verify, + + OUTPUT(test7.name), + test7.verify, + + OUTPUT(test8.name), + test8.verify, + + OUTPUT(test9.name), + test9.verify, + + OUTPUT(test10.name), + test10.verify, + + OUTPUT(test11.name), + test11.verify, + + OUTPUT(test12.name), + test12.verify, + + OUTPUT('=== Test Suite Complete ==='), + + cleanupFiles +); From 90b89c57e88c1ca9539e11a2e079a1569caf3346 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 18 Feb 2026 19:07:33 +0000 Subject: [PATCH 2/4] Review changes. Tighten LogicalFileListFiltered parsing; rename remoteDfs; update regress tests Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 2 +- ecllibrary/std/File.ecl | 18 +-- plugins/fileservices/fileservices.cpp | 106 +++++++++--------- plugins/fileservices/fileservices.hpp | 2 +- plugins/proxies/lib_fileservices.ecllib | 2 +- testing/regress/ecl/issue10022.ecl | 4 +- .../ecl/key/logicalfilelist_filtered_test.xml | 6 +- .../ecl/logicalfilelist_filtered_test.ecl | 10 +- testing/regress/ecl/multilfn.ecl | 4 +- 9 files changed, 76 insertions(+), 78 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index a906ab73c53..4f9282034da 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -2463,7 +2463,7 @@ struct SerializeFileAttrOptions fieldsList[static_cast(field)] = additive; if (additive) { - // The client uses these to decide if compresed or not. + // The client uses these to decide if compressed or not. fieldsList[static_cast(DFUQResultField::kind)] = true; fieldsList[static_cast(DFUQResultField::blockCompressed)] = true; } diff --git a/ecllibrary/std/File.ecl b/ecllibrary/std/File.ecl index d402223bab0..ef07eac06d5 100644 --- a/ecllibrary/std/File.ecl +++ b/ecllibrary/std/File.ecl @@ -234,7 +234,7 @@ EXPORT dataset(FsLogicalFileInfoRecord) LogicalFileList(varstring namepattern='* * * @param namepattern The mask of the files to list. Defaults to '*' (all files). * @param filters Filter string using search-like syntax for complex queries. Defaults to blank. - * Multiple filters separated by spaces. Supports: + * Multiple filters separated by commas. Supports: * * Field matching: * field:value - Exact wildcard match (owner:jsmith, name:*test*) @@ -260,7 +260,7 @@ EXPORT dataset(FsLogicalFileInfoRecord) LogicalFileList(varstring namepattern='* * is:normal - Normal files only * is:any - All file types (default) * - * Common fields: owner, group, name, size, recordcount, modified, recordsize, numparts + * Common fields: owner, cluster, name, size, recordcount, modified, recordsize, numparts * * Examples: * 'owner:jsmith' - Files owned by jsmith @@ -273,15 +273,15 @@ EXPORT dataset(FsLogicalFileInfoRecord) LogicalFileList(varstring namepattern='* * 'modified>2024-01-01 is:normal' - Recent normal files * @param fields Comma-separated list of field names to include in the result. Defaults to blank (all default fields). * When blank or empty, returns default fields: name, modified, size, rowcount, cluster, superfile, owner. - * Available fields: name, modified, size, rowcount, cluster, superfile, owner, group, recordsize, numparts. + * Available fields: name, modified, size, rowcount, cluster, superfile, owner, recordsize, numparts. * Note: 'name' field is always included regardless of specification. * Examples: * '' - Default fields (backward compatible) * 'name,size' - Only name and size * 'name,superfile,owner' - Name, superfile flag, and owner * @param unknownszero Whether to set file sizes that are unknown to zero(0) instead of minus-one (-1). Defaults to FALSE. - * @param foreigndali The IP address of the foreign dali used to resolve the file. If blank then the file is resolved - * locally. Defaults to blank. + * @param remoteDfs The name of the remote DFS service to perform the lookup on. If blank then the list is + * resolved locally. Defaults to blank. (Not yet supported; specifying this will raise an error.) * @param maxFileLimit Maximum number of files to return. Set to -1 to use server default limit (100,000). * Maximum client-side limit is 1,000,000. Defaults to -1. * @return A record containing: @@ -304,13 +304,13 @@ EXPORT dataset(FsLogicalFileInfoRecord) LogicalFileList(varstring namepattern='* * result3 := Std.File.LogicalFileListFiltered(filters := 'size>100000000'); * * // List superfiles owned by jsmith - * result4 := Std.File.LogicalFileListFiltered(filters := 'owner:jsmith is:superfile'); + * result4 := Std.File.LogicalFileListFiltered(filters := 'owner:jsmith,is:superfile'); * * // List normal files without description, modified recently - * result5 := Std.File.LogicalFileListFiltered(filters := 'is:normal !has:description modified>2024-01-01'); + * result5 := Std.File.LogicalFileListFiltered(filters := 'is:normal,!has:description,modified>2024-01-01'); */ -EXPORT lib_fileservices.FsLogicalFileListResult LogicalFileListFiltered(varstring namepattern='*', varstring filters='', varstring fields='', boolean unknownszero=FALSE, varstring foreigndali='', integer8 maxFileLimit=-1) := - lib_fileservices.FileServices.LogicalFileListFiltered(namepattern, filters, fields, unknownszero, foreigndali, maxFileLimit); +EXPORT lib_fileservices.FsLogicalFileListResult LogicalFileListFiltered(varstring namepattern='*', varstring filters='', varstring fields='', boolean unknownszero=FALSE, varstring remoteDfs='', integer8 maxFileLimit=-1) := + lib_fileservices.FileServices.LogicalFileListFiltered(namepattern, filters, fields, unknownszero, remoteDfs, maxFileLimit); /** * Compares two files, and returns a result indicating how well they match. diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index 5f7be3cff9f..1a30e2a9e07 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2406,8 +2406,7 @@ class FileListResultFieldSource : public CInterfaceOf public: FileListResultFieldSource(unsigned _count, bool _limitBreached, IPropertyTreeIterator *_iter, bool _unknownsZero) - : count(_count), limitBreached(_limitBreached), iter(_iter), unknownsZero(_unknownsZero), - currentFile(nullptr) + : count(_count), limitBreached(_limitBreached), iter(_iter), currentFile(nullptr), unknownsZero(_unknownsZero) { dbgassertex(iter); } @@ -2683,8 +2682,6 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal throw makeStringException(-1, "Invalid filter syntax: '!' must be followed by a filter term"); } - bool matched = false; - // Parse has:property if (strncmp(term, "has:", 4) == 0) { @@ -2703,7 +2700,6 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal DFUQFThasProp, DFUQFilterSeparator, attrName.str(), DFUQFilterSeparator, negate ? "false" : "true", DFUQFilterSeparator); - matched = true; } // Parse is:filetype else if (strncmp(term, "is:", 3) == 0) @@ -2715,33 +2711,25 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal if (isEmptyString(fileType)) throw makeStringException(-1, "Invalid filter syntax: 'is:' requires a file type (superfile, normal, or any)"); - if (stricmp(fileType, "superfile") == 0) - { - internalFilter.appendf("%u%c2%c2%c", - DFUQFTspecial, DFUQFilterSeparator, - DFUQFilterSeparator, DFUQFilterSeparator); - matched = true; - } - else if (stricmp(fileType, "normal") == 0) - { - internalFilter.appendf("%u%c2%c3%c", - DFUQFTspecial, DFUQFilterSeparator, - DFUQFilterSeparator, DFUQFilterSeparator); - matched = true; - } - else if (stricmp(fileType, "any") == 0) - { - internalFilter.appendf("%u%c2%c1%c", - DFUQFTspecial, DFUQFilterSeparator, - DFUQFilterSeparator, DFUQFilterSeparator); - matched = true; - } + DFUQFileTypeFilter fileTypeFilter = DFUQFFTall; + if (strieq(fileType, "any")) + fileTypeFilter = DFUQFFTall; + else if (strieq(fileType, "superfile")) + fileTypeFilter = DFUQFFTsuperfileonly; + else if (strieq(fileType, "normal")) + fileTypeFilter = DFUQFFTnonsuperfileonly; else throw makeStringExceptionV(-1, "Invalid filter syntax: 'is:%s' - must be superfile, normal, or any", fileType); + + internalFilter.appendf("%u%c%u%c%u%c", + DFUQFTspecial, DFUQFilterSeparator, (char)DFUQSFFileType, + DFUQFilterSeparator, (char)fileTypeFilter, DFUQFilterSeparator); } // Parse field:value (wildcard match) else if (const char *colon = strchr(term, ':')) { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating field:value filters is not supported"); StringBuffer fieldName; fieldName.append(colon - term, term).trim(); StringBuffer valueStr(colon + 1); @@ -2764,11 +2752,12 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal DFUQFTwildcardMatch, DFUQFilterSeparator, attrName.str(), DFUQFilterSeparator, value, DFUQFilterSeparator); - matched = true; } // Parse field>value, field=value, field<=value else if (const char *op = strpbrk(term, "><")) { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating comparison filters is not supported"); StringBuffer fieldName; fieldName.append(op - term, term).trim(); @@ -2801,7 +2790,11 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal if (op[0] == '>') { // field > value or field >= value - __int64 minVal = _atoi64(value); + char *endptr; + __int64 minVal = (__int64) strtoll(value, &endptr, 10); + if (!isEmptyString(endptr)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value '%s' must be an integer", originalTerm, value); + if (!hasEquals) { if (minVal == I64C(0x7FFFFFFFFFFFFFFF)) @@ -2816,7 +2809,10 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal else // op[0] == '<' { // field < value or field <= value - __int64 maxVal = _atoi64(value); + char *endptr; + __int64 maxVal = (__int64) strtoll(value, &endptr, 10); + if (!isEmptyString(endptr)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value '%s' must be an integer", originalTerm, value); if (!hasEquals) { if (maxVal == (-I64C(0x7FFFFFFFFFFFFFFF) - 1)) @@ -2832,8 +2828,12 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal else { // Parse string range (for dates, text, etc.) + // String range filter only supports inclusive bounds (>=, <=) + // since the filter uses standard string comparison if (op[0] == '>') { + if (!hasEquals) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - exclusive comparison (>) is not supported for string fields; use >= instead", originalTerm); internalFilter.appendf("%u%c%s%c%s%c~~~~~~~~~~%c", DFUQFTstringRange, DFUQFilterSeparator, attrName.str(), DFUQFilterSeparator, @@ -2841,17 +2841,20 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal } else // op[0] == '<' { + if (!hasEquals) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - exclusive comparison (<) is not supported for string fields; use <= instead", originalTerm); internalFilter.appendf("%u%c%s%c%c%s%c", DFUQFTstringRange, DFUQFilterSeparator, attrName.str(), DFUQFilterSeparator, DFUQFilterSeparator, value, DFUQFilterSeparator); } } - matched = true; } // Parse field=value as exact range match else if (const char *eq = strchr(term, '=')) { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating equality filters is not supported"); StringBuffer fieldName; fieldName.append(eq - term, term).trim(); StringBuffer valueStr(eq + 1); @@ -2874,7 +2877,7 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal bool isNumeric = (fieldType == DFUQResultFieldType::numericType); bool isFloat = (fieldType == DFUQResultFieldType::floatType); - if (isNumeric || isFloat) + if (isNumeric) { __int64 val = _atoi64(value); internalFilter.appendf("%u%c%s%c%lld%c%lld%c", @@ -2882,6 +2885,14 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal attrName.str(), DFUQFilterSeparator, val, DFUQFilterSeparator, val, DFUQFilterSeparator); } + else if (isFloat) + { + // Exact string match via range for floating values + internalFilter.appendf("%u%c%s%c%s%c%s%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator, value, DFUQFilterSeparator); + } else { // Exact string match via range @@ -2890,16 +2901,16 @@ static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internal attrName.str(), DFUQFilterSeparator, value, DFUQFilterSeparator, value, DFUQFilterSeparator); } - matched = true; } - - // If nothing matched, throw an error - if (!matched) + else + { + // If nothing matched, throw an error throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unrecognized filter format. Use field:value, field>value, field=value, has:property, or is:filetype", originalTerm); + } } } -FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *foreigndali, __int64 maxFileLimit) +FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *remoteDfs, __int64 maxFileLimit) { IEngineContext *engineCtx = ctx->queryEngineContext(); if (engineCtx && !engineCtx->allowDaliAccess()) @@ -2925,12 +2936,9 @@ FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeC StringBuffer masklower(mask); masklower.toLowerCase(); - Owned foreignNode; - if (!isEmptyString(foreigndali)) - { - SocketEndpoint ep(foreigndali); - foreignNode.setown(createINode(ep)); - } + if (!isEmptyString(remoteDfs)) + throw makeStringException(-1, "FileServices.LogicalFileListFiltered: remoteDfs is not supported yet"); + // Build filter string - translate user-friendly syntax to internal format StringBuffer filterBuf; @@ -2973,16 +2981,7 @@ FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeC requestedFieldNames.appendList(requestedFields, ","); // Ensure "name" is always included (required field) - bool hasName = false; - ForEachItemIn(idx, requestedFieldNames) - { - if (stricmp(requestedFieldNames.item(idx), "name") == 0) - { - hasName = true; - break; - } - } - if (!hasName) + if (!requestedFieldNames.contains("name", true)) requestedFieldNames.append("name"); } @@ -3022,8 +3021,7 @@ FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeC ctx->queryUserDescriptor(), true, // recursive allMatchingFilesReceived, - &count, - foreignNode + &count ); // Build result row using IFieldSource pattern diff --git a/plugins/fileservices/fileservices.hpp b/plugins/fileservices/fileservices.hpp index 29b925be645..ff0135449ac 100644 --- a/plugins/fileservices/fileservices.hpp +++ b/plugins/fileservices/fileservices.hpp @@ -153,7 +153,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsGetFileDescription(ICodeContext *ct FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub); FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory_v2(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub, const char *planename); FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *mask, bool includenormal, bool includesuper, bool unknownszero,const char *foreigndali); -FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *foreigndali, __int64 maxFileLimit); +FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeContext *ctx, IEngineRowAllocator *_rowAllocator, const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *remoteDfs, __int64 maxFileLimit); FILESERVICES_API void FILESERVICES_CALL fsSuperFileContents(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lsuperlfn, bool recurse); FILESERVICES_API void FILESERVICES_CALL fsLogicalFileSuperOwners(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lfn); FILESERVICES_API char * FILESERVICES_CALL fsExternalLogicalFileName(const char *location, const char *path,bool abspath); diff --git a/plugins/proxies/lib_fileservices.ecllib b/plugins/proxies/lib_fileservices.ecllib index 3bafad14665..f3a582abdd5 100644 --- a/plugins/proxies/lib_fileservices.ecllib +++ b/plugins/proxies/lib_fileservices.ecllib @@ -96,7 +96,7 @@ export FileServices := SERVICE : plugin('fileservices'), time SetFileDescription(const varstring lfn,const varstring val) : c,action,context,entrypoint='fsSetFileDescription'; dataset(FsFilenameRecord) RemoteDirectory(const varstring machineIP,const varstring dir,const varstring mask='*',boolean sub=false,const varstring planeName='') : c,context,entrypoint='fsRemoteDirectory_v2'; dataset(FsLogicalFileInfoRecord) LogicalFileList(const varstring namepattern='*',boolean includenormal=true,boolean includesuper=false,boolean unknownszero=false,const varstring foreigndali='') : c,context,entrypoint='fsLogicalFileList'; - _linkcounted_ ROW(FsLogicalFileListResult) LogicalFileListFiltered(const varstring namepattern='*',const varstring filters='',const varstring fields='',boolean unknownszero=false,const varstring foreigndali='',integer8 maxFileLimit=-1) : c,context,entrypoint='fsLogicalFileListFiltered'; + _linkcounted_ ROW(FsLogicalFileListResult) LogicalFileListFiltered(const varstring namepattern='*',const varstring filters='',const varstring fields='',boolean unknownszero=false,const varstring remoteDfs='',integer8 maxFileLimit=-1) : c,context,entrypoint='fsLogicalFileListFiltered'; dataset(FsLogicalFileNameRecord) SuperFileContents(const varstring lsuperfn,boolean recurse=false) : c,context,entrypoint='fsSuperFileContents'; dataset(FsLogicalFileNameRecord) LogicalFileSuperOwners(const varstring lfn) : c,context,entrypoint='fsLogicalFileSuperOwners'; varstring ExternalLogicalFileName(const varstring location, const varstring path,boolean abspath=true) : c,entrypoint='fsExternalLogicalFileName'; diff --git a/testing/regress/ecl/issue10022.ecl b/testing/regress/ecl/issue10022.ecl index 74791cbbb9a..7f8def3c13d 100644 --- a/testing/regress/ecl/issue10022.ecl +++ b/testing/regress/ecl/issue10022.ecl @@ -14,8 +14,8 @@ p3 := ds(line[1]='!') : persist(prefix + 'persist_gh3',multiple(10)); p4 := ds(line[1] = ' ') : persist(prefix + 'persist_gh4'); -lfl1 := File.LogicalFileList(prefix + 'persist_gh*'); -delall := NOTHOR(APPLY(lfl1,File.DeleteLogicalFile(name, true))); +lfl1 := File.LogicalFileListFiltered(prefix + 'persist_gh*'); +delall := NOTHOR(APPLY(lfl1.files, File.DeleteLogicalFile(name, true))); sequential( delall, diff --git a/testing/regress/ecl/key/logicalfilelist_filtered_test.xml b/testing/regress/ecl/key/logicalfilelist_filtered_test.xml index 716b26cdbcd..f08254c060f 100644 --- a/testing/regress/ecl/key/logicalfilelist_filtered_test.xml +++ b/testing/regress/ecl/key/logicalfilelist_filtered_test.xml @@ -79,7 +79,7 @@ 6 - false + true 6 @@ -91,13 +91,13 @@ 4 - false + true 4 - Test 8: Modified before 2026-02-12 (should exclude all test files) + Test 8: Modified <= <YESTERDAY> (should exclude all test files) 0 diff --git a/testing/regress/ecl/logicalfilelist_filtered_test.ecl b/testing/regress/ecl/logicalfilelist_filtered_test.ecl index 15eae48cd62..853c758b616 100644 --- a/testing/regress/ecl/logicalfilelist_filtered_test.ecl +++ b/testing/regress/ecl/logicalfilelist_filtered_test.ecl @@ -144,7 +144,7 @@ test6 := MODULE EXPORT name := 'Test 6: Record count range (100 <= rowcount < 1000)'; EXPORT verify := SEQUENTIAL( OUTPUT(result.count, NAMED('Test6_Count')), - OUTPUT(result.count = 5, NAMED('Test6_ExpectedCount')), // Should match: file1(100), file2(200), file3(500), testfile1(300), testfile2(400) + OUTPUT(result.count = 6, NAMED('Test6_ExpectedCount')), // Should match: file1(100), file2(200), file3(500), testfile1(300), testfile2(400), super1(300) OUTPUT(COUNT(result.files), NAMED('Test6_FileCount')) ); END; @@ -156,19 +156,19 @@ test7 := MODULE EXPORT name := 'Test 7: Combined filters (200 <= rowcount <= 500 AND normal files)'; EXPORT verify := SEQUENTIAL( OUTPUT(result.count, NAMED('Test7_Count')), - OUTPUT(result.count = 3, NAMED('Test7_ExpectedCount')), // Should match: file2(200), file3(500), testfile1(300), testfile2(400) = 4, but super1 excluded + OUTPUT(result.count = 4, NAMED('Test7_ExpectedCount')), // Should match: file2(200), file3(500), testfile1(300), testfile2(400) = 4, but super1 excluded OUTPUT(COUNT(result.files), NAMED('Test7_FileCount')) ); END; // Test 8: Filter - date range excludes all recent files (proves date filtering works) test8 := MODULE - EXPORT filters := 'modified<2026-02-12'; + EXPORT filters := 'modified<=' + Std.Date.DateToString(Std.Date.Today() - 1, '%Y-%m-%d'); EXPORT result := Std.File.LogicalFileListFiltered(prefix + '*', filters := filters); - EXPORT name := 'Test 8: Modified before 2026-02-12 (should exclude all test files)'; + EXPORT name := 'Test 8: Modified <= (should exclude all test files)'; EXPORT verify := SEQUENTIAL( OUTPUT(result.count, NAMED('Test8_Count')), - OUTPUT(result.count = 0, NAMED('Test8_ExpectedZero')), // All test files created in 2026, so none match + OUTPUT(result.count = 0, NAMED('Test8_ExpectedZero')), // All test files created before yesterday, so none match OUTPUT(COUNT(result.files), NAMED('Test8_FileCount')) ); END; diff --git a/testing/regress/ecl/multilfn.ecl b/testing/regress/ecl/multilfn.ecl index f1ece2bce57..09011327992 100644 --- a/testing/regress/ecl/multilfn.ecl +++ b/testing/regress/ecl/multilfn.ecl @@ -640,9 +640,9 @@ o15 := OUTPUT(COUNT(i2)-COUNT(ds)); i3 := DATASET('~regress::' + WORKUNIT + '::' + '{BB,dd ddd}',rec,flat); o16 := OUTPUT(COUNT(JOIN(i3,sample(sds,5,2)+sample(sds,5,4),left.song=right.song,FULL ONLY))); -lfl1 := FileServices.LogicalFileList(str.ToLowerCase(thorlib.getExpandLogicalName('nhtest_'+WORKUNIT+'::*'))); +lfl1 := FileServices.LogicalFileListFiltered(str.ToLowerCase(thorlib.getExpandLogicalName('nhtest_'+WORKUNIT+'::*'))); -a1 := NOTHOR(APPLY(lfl1,FileServices.DeleteLogicalFile('~'+name))); +a1 := NOTHOR(APPLY(lfl1.files,FileServices.DeleteLogicalFile('~'+name))); SEQUENTIAL(o1,o2,o3,o4,o5,o6,o7,o8,o9,o10,o11,o12,o13,o14,o15,o16,a1); From ff8437dbd7d53d9ea67b7c542c22fcb5de572663 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:24:28 +0000 Subject: [PATCH 3/4] Initial plan From f3567d6b18b8c98b1d8fa50c5c9dc56de39997f7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Feb 2026 11:34:10 +0000 Subject: [PATCH 4/4] Add remoteDfs support to LogicalFileListFiltered - implementation phase 1 Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com> --- esp/clients/ws_dfsclient/ws_dfsclient.cpp | 93 +++++ esp/clients/ws_dfsclient/ws_dfsclient.hpp | 1 + esp/scm/ws_dfs.ecm | 21 +- esp/services/ws_dfsservice/ws_dfsservice.cpp | 404 +++++++++++++++++++ esp/services/ws_dfsservice/ws_dfsservice.hpp | 1 + plugins/fileservices/fileservices.cpp | 37 +- 6 files changed, 554 insertions(+), 3 deletions(-) diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp index 7eb850e778f..f47e8705c3b 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp @@ -841,6 +841,99 @@ IDFSFile *lookupDFSFile(const char *logicalName, AccessMode accessMode, unsigned throw makeStringExceptionV(0, "DFSFileLookup timed out: file=%s, timeoutSecs=%u", logicalName, timeoutSecs); } +IPropertyTree *listFilteredDFSFiles(const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *remoteDfs, __int64 maxFileLimit, unsigned timeoutSecs, unsigned keepAliveExpiryFrequency, IUserDescriptor *userDesc) +{ + if (isEmptyString(remoteDfs)) + throw makeStringException(-1, "ws_dfsclient::listFilteredDFSFiles: remoteDfs parameter is required"); + + StringBuffer serviceUrl; + StringBuffer serviceSecret; + bool secretProvided = false; + bool useDafilesrv = false; + + // Get remote storage configuration + Owned remoteStorage = getRemoteStorage(remoteDfs); + if (!remoteStorage) + throw makeStringExceptionV(0, "Remote storage '%s' not found", remoteDfs); + + serviceUrl.set(remoteStorage->queryProp("@service")); + + if (startsWith(serviceUrl, "https")) + { + // NB: standard configuration should not supply a secret, the secret name will be auto-generated based on the URL + // If a manual secret name is defined, it will be used to connect to the DFS service + // A blank secret name can be defined to support connecting to bare-metal DFS services that do not support client certificates. + if (remoteStorage->hasProp("@secret")) + { + secretProvided = true; + serviceSecret.set(remoteStorage->queryProp("@secret")); + } + } + + bool useSSL = startsWith(serviceUrl, "https"); + if (useSSL && !secretProvided) + generateDynamicUrlSecretName(serviceSecret, serviceUrl, nullptr); + + DBGLOG("Listing filtered files on '%s'", serviceUrl.str()); + Owned dfsClient = getDfsClient(serviceUrl, userDesc); + + unsigned __int64 clientLeaseId = ensureClientLease(dfsClient, serviceUrl, serviceSecret, userDesc); + + Owned dfsResp; + Owned dfsReq = dfsClient->createDFSListFilteredRequest(); + if (useSSL && serviceSecret.length()) + configureClientSSL(dfsReq->rpc(), serviceSecret.str()); + + // Set request parameters - send human-readable forms + dfsReq->setMask(mask); + dfsReq->setFilters(filters); + dfsReq->setRequestedFields(requestedFields); + dfsReq->setUnknownSizeZero(unknownszero); + dfsReq->setMaxFileLimit(maxFileLimit); + dfsReq->setLeaseId(clientLeaseId); + dfsReq->setRequestTimeout(timeoutSecs); + + CTimeMon tm(timeoutSecs*1000); // NB: this timeout loop is to cater for *a* esp disappearing (e.g. if behind load balancer) + while (true) + { + try + { + unsigned remaining; + if (tm.timedout(&remaining)) + break; + dfsReq->setRequestTimeout(remaining/1000); + dfsResp.setown(dfsClient->DFSListFiltered(dfsReq)); + + const IMultiException *excep = &dfsResp->getExceptions(); // NB: warning despite getXX name, this does not Link + if (excep->ordinality() > 0) + throw LINK((IMultiException *)excep); // NB - const IException.. not caught in general.. + + const char *base64Resp = dfsResp->getResult(); + MemoryBuffer compressedRespMb; + JBASE64_Decode(base64Resp, compressedRespMb); + MemoryBuffer decompressedRespMb; + fastLZDecompressToBuffer(decompressedRespMb, compressedRespMb); + Owned resultTree = createPTree(decompressedRespMb); + + return resultTree.getClear(); + } + catch (IException *e) + { + /* NB: there should really be a different IException class and a specific error code + * The server knows it's an unsupported method. + */ + if (SOAP_SERVER_ERROR != e->errorCode()) + throw; + e->Release(); + } + + if (tm.timedout()) + break; + Sleep(5000); // sanity sleep + } + throw makeStringExceptionV(0, "DFSListFiltered timed out: timeoutSecs=%u", timeoutSecs); +} + IDistributedFile *createLegacyDFSFile(IDFSFile *dfsFile) { if (dfsFile->queryFileMeta()->getPropBool("@isSuper")) diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.hpp b/esp/clients/ws_dfsclient/ws_dfsclient.hpp index 777787c0d65..3fcc6c9c194 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.hpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.hpp @@ -50,6 +50,7 @@ interface IDFSFile : extends IInterface WS_DFSCLIENT_API IDFSFile *lookupDFSFile(const char *logicalName, AccessMode accessMode, unsigned timeoutSecs, unsigned keepAliveExpiryFrequency, IUserDescriptor *userDesc); WS_DFSCLIENT_API IDistributedFile *createLegacyDFSFile(IDFSFile *dfsFile); WS_DFSCLIENT_API IDistributedFile *lookupLegacyDFSFile(const char *logicalName, AccessMode accessMode, unsigned timeoutSecs, unsigned keepAliveExpiryFrequency, IUserDescriptor *userDesc); +WS_DFSCLIENT_API IPropertyTree *listFilteredDFSFiles(const char *mask, const char *filters, const char *requestedFields, bool unknownszero, const char *remoteDfs, __int64 maxFileLimit, unsigned timeoutSecs, unsigned keepAliveExpiryFrequency, IUserDescriptor *userDesc); WS_DFSCLIENT_API IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout); WS_DFSCLIENT_API IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout); diff --git a/esp/scm/ws_dfs.ecm b/esp/scm/ws_dfs.ecm index 6b97353b445..451110f0bb7 100644 --- a/esp/scm/ws_dfs.ecm +++ b/esp/scm/ws_dfs.ecm @@ -48,17 +48,34 @@ ESPresponse [exceptions_inline] KeepAliveResponse { }; +ESPrequest [version("1.02")] DFSListFilteredRequest +{ + string Mask; // file name pattern (e.g., "*" or "myfiles*") + string Filters; // user-friendly filter syntax (comma-separated) + string RequestedFields; // comma-separated fields to return + bool UnknownSizeZero(false); // treat unknown sizes as zero + int64 MaxFileLimit(-1); // max files to return (-1 = server default) + int64 LeaseId; // lease to associate any locks to + int RequestTimeout(300); // Max seconds to block waiting (default = 5 mins) +}; + +ESPresponse [exceptions_inline, version("1.02")] DFSListFilteredResponse +{ + string Result; // base64-encoded, compressed result data +}; + // =========================================================================== ESPservice [ auth_feature("DEFERRED"), - version("1.01"), - default_client_version("1.01"), + version("1.02"), + default_client_version("1.02"), noforms, exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfs { ESPmethod [auth_feature("DfsAccess:READ"), min_ver("1.01")] GetLease(LeaseRequest, LeaseResponse); ESPmethod [auth_feature("DfsAccess:READ"), min_ver("1.01")] KeepAlive(KeepAliveRequest, KeepAliveResponse); ESPmethod [auth_feature("DfsAccess:READ"), min_ver("1.01")] DFSFileLookup(DFSFileLookupRequest, DFSFileLookupResponse); + ESPmethod [auth_feature("DfsAccess:READ"), min_ver("1.02")] DFSListFiltered(DFSListFilteredRequest, DFSListFilteredResponse); }; SCMexportdef(WsDfs); diff --git a/esp/services/ws_dfsservice/ws_dfsservice.cpp b/esp/services/ws_dfsservice/ws_dfsservice.cpp index d74cf9bb702..5fdc20de9e2 100644 --- a/esp/services/ws_dfsservice/ws_dfsservice.cpp +++ b/esp/services/ws_dfsservice/ws_dfsservice.cpp @@ -31,6 +31,10 @@ #include "ws_dfsclient.hpp" #include "ws_dfsservice.hpp" +#include +#include +#include + using namespace wsdfs; // all fake for now @@ -237,3 +241,403 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r return true; } +// Helper functions for DFSListFiltered + +// Case-insensitive string hash and comparison for field aliases +struct CaseInsensitiveHash +{ + size_t operator()(const std::string &s) const + { + std::string lower = s; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + return std::hash()(lower); + } +}; + +struct CaseInsensitiveEqual +{ + bool operator()(const std::string &a, const std::string &b) const + { + return std::equal(a.begin(), a.end(), b.begin(), b.end(), + [](char ca, char cb) { return tolower(ca) == tolower(cb); }); + } +}; + +static bool validateFileField(const char *requestedField, StringBuffer &attrName, DFUQResultField &fieldEnum, DFUQResultFieldType &fieldType) +{ + // Map of ECL field name aliases to canonical internal field names + // This handles user-friendly names and their mappings + static const std::unordered_map fieldAliases = + { + {"superfile", "numsubfiles"}, // Derived from numsubfiles + {"rowcount", "recordcount"}, // ECL field name is "rowcount" but internal attribute is "recordCount" + {"cluster", "group"} // ECL field name is "cluster" but internal attribute is "group" + }; + + // Trim whitespace + StringBuffer fieldName(requestedField); + fieldName.trim(); + if (isEmptyString(fieldName)) + return false; + + auto it = fieldAliases.find(fieldName.str()); + if (it != fieldAliases.end()) + fieldName.set(it->second); + + return getFileAttributePath(fieldName, attrName, fieldEnum, fieldType); +} + +static void parseUserFilterSyntax(const char *userFilter, StringBuffer &internalFilter) +{ + if (isEmptyString(userFilter)) + return; + + StringArray terms; + terms.appendList(userFilter, ","); + + ForEachItemIn(i, terms) + { + const char *term = terms.item(i); + if (isEmptyString(term)) + continue; + + // Save original term for error messages + const char *originalTerm = term; + + // Check for negation prefix + bool negate = (term[0] == '!'); + if (negate) + { + term++; + if (isEmptyString(term)) + throw makeStringException(-1, "Invalid filter syntax: '!' must be followed by a filter term"); + } + + // Parse has:property + if (strncmp(term, "has:", 4) == 0) + { + const char *prop = term + 4; + if (isEmptyString(prop)) + throw makeStringException(-1, "Invalid filter syntax: 'has:' requires a property name (e.g., 'has:description')"); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(prop, attrName, field, fieldType)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, prop); + + internalFilter.appendf("%u%c%s%c%s%c", + DFUQFThasProp, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + negate ? "false" : "true", DFUQFilterSeparator); + } + // Parse is:filetype + else if (strncmp(term, "is:", 3) == 0) + { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating 'is:' is not supported"); + + const char *fileType = term + 3; + if (isEmptyString(fileType)) + throw makeStringException(-1, "Invalid filter syntax: 'is:' requires a file type (superfile, normal, or any)"); + + DFUQFileTypeFilter fileTypeFilter = DFUQFFTall; + if (strieq(fileType, "any")) + fileTypeFilter = DFUQFFTall; + else if (strieq(fileType, "superfile")) + fileTypeFilter = DFUQFFTsuperfileonly; + else if (strieq(fileType, "normal")) + fileTypeFilter = DFUQFFTnonsuperfileonly; + else + throw makeStringExceptionV(-1, "Invalid filter syntax: 'is:%s' - must be superfile, normal, or any", fileType); + + internalFilter.appendf("%u%c%u%c%u%c", + DFUQFTspecial, DFUQFilterSeparator, (char)DFUQSFFileType, + DFUQFilterSeparator, (char)fileTypeFilter, DFUQFilterSeparator); + } + // Parse field:value (wildcard match) + else if (const char *colon = strchr(term, ':')) + { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating field:value filters is not supported"); + StringBuffer fieldName; + fieldName.append(colon - term, term).trim(); + StringBuffer valueStr(colon + 1); + valueStr.trim(); + const char *value = valueStr.str(); + + if (fieldName.length() == 0) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - field name required before ':'", originalTerm); + if (isEmptyString(value)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value required after ':'", originalTerm); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName.str(), attrName, field, fieldType)) + throw makeStringException(-1, VStringBuffer("Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, fieldName.str()).str()); + + internalFilter.appendf("%u%c%s%c%s%c", + DFUQFTwildcardMatch, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator); + } + // Parse field>value, field=value, field<=value + else if (const char *op = strpbrk(term, "><")) + { + if (negate) + throw makeStringExceptionV(-1, "Invalid filter syntax: negating comparison filters is not supported"); + StringBuffer fieldName; + fieldName.append(op - term, term).trim(); + + if (fieldName.length() == 0) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - field name required before comparison operator", originalTerm); + + // Validate and convert field name + StringBuffer attrName; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName.str(), attrName, field, fieldType)) + throw makeStringException(-1, VStringBuffer("Invalid filter syntax: '%s' - unknown field name '%s'", originalTerm, fieldName.str()).str()); + + // Determine operator + bool hasEquals = (op[1] == '='); + StringBuffer valueStr(hasEquals ? (op + 2) : (op + 1)); + valueStr.trim(); + const char *value = valueStr.str(); + + if (isEmptyString(value)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value required after comparison operator", originalTerm); + + // Check if field is numeric/float type + bool isNumeric = (fieldType == DFUQResultFieldType::numericType); + bool isFloat = (fieldType == DFUQResultFieldType::floatType); + + if (isNumeric || isFloat) + { + // Parse numeric range + if (op[0] == '>') + { + // field > value or field >= value + char *endptr; + __int64 minVal = (__int64) strtoll(value, &endptr, 10); + if (!isEmptyString(endptr)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value '%s' must be an integer", originalTerm, value); + + if (!hasEquals) + { + if (minVal == I64C(0x7FFFFFFFFFFFFFFF)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value too large for > comparison (would overflow)", originalTerm); + minVal++; + } + internalFilter.appendf("%u%c%s%c%lld%c%lld%c", + DFUQFTinteger64Range, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + minVal, DFUQFilterSeparator, I64C(0x7FFFFFFFFFFFFFFF), DFUQFilterSeparator); + } + else // op[0] == '<' + { + // field < value or field <= value + char *endptr; + __int64 maxVal = (__int64) strtoll(value, &endptr, 10); + if (!isEmptyString(endptr)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value '%s' must be an integer", originalTerm, value); + if (!hasEquals) + { + if (maxVal == (-I64C(0x7FFFFFFFFFFFFFFF) - 1)) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - value too small for < comparison (would underflow)", originalTerm); + maxVal--; + } + internalFilter.appendf("%u%c%s%c0%c%lld%c", + DFUQFTinteger64Range, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + DFUQFilterSeparator, maxVal, DFUQFilterSeparator); + } + } + else + { + // Parse string range (for dates, text, etc.) + // String range filter only supports inclusive bounds (>=, <=) + // since the filter uses standard string comparison + if (op[0] == '>') + { + if (!hasEquals) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - exclusive comparison (>) is not supported for string fields; use >= instead", originalTerm); + internalFilter.appendf("%u%c%s%c%s%c~~~~~~~~~~%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + value, DFUQFilterSeparator, DFUQFilterSeparator); + } + else // op[0] == '<' + { + if (!hasEquals) + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - exclusive comparison (<) is not supported for string fields; use <= instead", originalTerm); + internalFilter.appendf("%u%c%s%c%c%s%c", + DFUQFTstringRange, DFUQFilterSeparator, + attrName.str(), DFUQFilterSeparator, + DFUQFilterSeparator, value, DFUQFilterSeparator); + } + } + } + else + { + // Unknown filter format + throw makeStringExceptionV(-1, "Invalid filter syntax: '%s' - unrecognized format", originalTerm); + } + } +} + +bool CWsDfsEx::onDFSListFiltered(IEspContext &context, IEspDFSListFilteredRequest &req, IEspDFSListFilteredResponse &resp) +{ + try + { + const char *mask = req.getMask(); + const char *filters = req.getFilters(); + const char *requestedFields = req.getRequestedFields(); + bool unknownszero = req.getUnknownSizeZero(); + __int64 maxFileLimit = req.getMaxFileLimit(); + + StringBuffer userID; + context.getUserID(userID); + Owned userDesc; + if (!userID.isEmpty()) + { + userDesc.setown(createUserDescriptor()); + userDesc->set(userID.str(), context.queryPassword(), context.querySignature()); + } + + // Validate server-side max limit (10 million) + constexpr __int64 SERVER_MAX_LIMIT = 10000000; + if (maxFileLimit > SERVER_MAX_LIMIT) + { + throw makeStringExceptionV(-1, "WsDfs.DFSListFiltered: maxFileLimit (%lld) exceeds server maximum of %lld", maxFileLimit, SERVER_MAX_LIMIT); + } + + if (isEmptyString(mask)) + mask = "*"; + else if (*mask == '~') + mask++; // Strip leading ~ if present, as internal APIs expect it without + StringBuffer masklower(mask); + masklower.toLowerCase(); + + // Build filter string - translate user-friendly syntax to internal format + StringBuffer filterBuf; + + // Parse user-provided filters (using friendly syntax like "owner:jsmith size>1000") + parseUserFilterSyntax(filters, filterBuf); + + // Append system filters: name pattern and max files limit + filterBuf.appendf("%u%c%u%c%s%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQSFFileNameWithPrefix, DFUQFilterSeparator, + masklower.str(), DFUQFilterSeparator); + + // Add max files limit if specified (and not -1 which means use server default) + if (maxFileLimit > 0) + { + filterBuf.appendf("%u%c%u%c%lld%c", + DFUQFTspecial, DFUQFilterSeparator, + DFUQSFMaxFiles, DFUQFilterSeparator, + maxFileLimit, DFUQFilterSeparator); + } + + // Parse and validate requested fields + std::vector fields; + StringArray requestedFieldNames; + + if (isEmptyString(requestedFields)) + { + requestedFieldNames.append("name"); + requestedFieldNames.append("superfile"); + requestedFieldNames.append("size"); + requestedFieldNames.append("rowcount"); + requestedFieldNames.append("modified"); + requestedFieldNames.append("owner"); + requestedFieldNames.append("cluster"); + } + else + { + // Parse comma-separated field list + requestedFieldNames.appendList(requestedFields, ","); + + // Ensure "name" is always included (required field) + if (!requestedFieldNames.contains("name", true)) + requestedFieldNames.append("name"); + } + + // Validate field names and build field list for getDFAttributesFilteredIterator + ForEachItemIn(idx, requestedFieldNames) + { + // Trim whitespace + const char *fieldName = requestedFieldNames.item(idx); + if (isEmptyString(fieldName)) + continue; + + // Validate field name using existing validation function + StringBuffer attrPath; + DFUQResultField field; + DFUQResultFieldType fieldType; + if (!validateFileField(fieldName, attrPath, field, fieldType)) + throw makeStringExceptionV(-1, "WsDfs.DFSListFiltered: Invalid field name '%s'", fieldName); + + // Add field to list if not already present + if (std::find(fields.begin(), fields.end(), field) == fields.end()) + fields.push_back(field); + } + + // Always include numsubfiles for superfile detection + if (std::find(fields.begin(), fields.end(), DFUQResultField::numsubfiles) == fields.end()) + fields.push_back(DFUQResultField::numsubfiles); + + // Add terminator + fields.push_back(DFUQResultField::term); + + bool allMatchingFilesReceived = false; + unsigned count = 0; + Owned iter = queryDistributedFileDirectory().getDFAttributesFilteredIterator( + filterBuf.str(), + nullptr, // no local filters + fields.data(), // requested fields + userDesc, + true, // recursive + allMatchingFilesReceived, + &count + ); + + // Build result tree + Owned resultTree = createPTree(); + resultTree->setPropInt("@count", count); + resultTree->setPropBool("@allMatchingFilesReceived", allMatchingFilesReceived); + + // Add files to result + IPropertyTree *filesTree = resultTree->addPropTree("Files"); + ForEach(*iter) + { + IPropertyTree &file = iter->query(); + IPropertyTree *fileTree = filesTree->addPropTree("File", &file); + + // Handle unknownszero flag for size field + if (unknownszero && !fileTree->hasProp("@size")) + fileTree->setPropInt64("@size", 0); + } + + // Serialize response + MemoryBuffer respMb, compressedRespMb; + resultTree->serialize(respMb); + fastLZCompressToBuffer(compressedRespMb, respMb.length(), respMb.bytes()); + StringBuffer respStr; + JBASE64_Encode(compressedRespMb.bytes(), compressedRespMb.length(), respStr, false); + resp.setResult(respStr.str()); + + LOG(MCauditInfo,",FileList,EspProcess,READ,%s,%s,%u,%s", mask, filters?filters:"", count, userID.str()); + } + catch (IException *e) + { + FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR); + } + return true; +} + + diff --git a/esp/services/ws_dfsservice/ws_dfsservice.hpp b/esp/services/ws_dfsservice/ws_dfsservice.hpp index a1b8f3d2894..c87c83555b8 100644 --- a/esp/services/ws_dfsservice/ws_dfsservice.hpp +++ b/esp/services/ws_dfsservice/ws_dfsservice.hpp @@ -33,6 +33,7 @@ class CWsDfsEx : public CWsDfs virtual bool onGetLease(IEspContext &context, IEspLeaseRequest &req, IEspLeaseResponse &resp); virtual bool onKeepAlive(IEspContext &context, IEspKeepAliveRequest &req, IEspKeepAliveResponse &resp); virtual bool onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &req, IEspDFSFileLookupResponse &resp); + virtual bool onDFSListFiltered(IEspContext &context, IEspDFSListFilteredRequest &req, IEspDFSListFilteredResponse &resp); }; diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index 1a30e2a9e07..28a7abc3957 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2936,8 +2936,43 @@ FILESERVICES_API const byte * FILESERVICES_CALL fsLogicalFileListFiltered(ICodeC StringBuffer masklower(mask); masklower.toLowerCase(); + // Handle remote DFS if (!isEmptyString(remoteDfs)) - throw makeStringException(-1, "FileServices.LogicalFileListFiltered: remoteDfs is not supported yet"); + { + // Call remote DFS service to get file list + Owned resultTree = wsdfs::listFilteredDFSFiles( + mask, + filters, + requestedFields, + unknownszero, + remoteDfs, + maxFileLimit, + 300, // timeout secs + wsdfs::keepAliveExpiryFrequency, + ctx->queryUserDescriptor() + ); + + // Extract results from remote response + unsigned count = resultTree->getPropInt("@count", 0); + bool allMatchingFilesReceived = resultTree->getPropBool("@allMatchingFilesReceived", true); + + // Get iterator over Files/File elements + IPropertyTree *filesTree = resultTree->queryPropTree("Files"); + if (!filesTree) + filesTree = resultTree; // fallback if no Files wrapper + + Owned iter = filesTree->getElements("File"); + + // Build result row using IFieldSource pattern + RtlDynamicRowBuilder resultBuilder(*_rowAllocator); + Owned fieldSource = new FileListResultFieldSource(count, !allMatchingFilesReceived, iter.getClear(), unknownszero); + + const RtlTypeInfo *typeInfo = _rowAllocator->queryOutputMeta()->queryTypeInfo(); + RtlFieldStrInfo dummyField("", NULL, typeInfo); + size32_t len = typeInfo->build(resultBuilder, 0, &dummyField, *fieldSource); + + return (const byte *)resultBuilder.finalizeRowClear(len); + } // Build filter string - translate user-friendly syntax to internal format