diff --git a/.gitignore b/.gitignore index 40fa2ba..6d62090 100644 --- a/.gitignore +++ b/.gitignore @@ -21,9 +21,16 @@ coverage # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release -lib +/lib # Dependency directory # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git node_modules test/dump + +# So o files +*.so +*.o +*.lo +.vscode +/csrc/lib/jansson/* diff --git a/.npmignore b/.npmignore index 30d74d2..0fc3898 100644 --- a/.npmignore +++ b/.npmignore @@ -1 +1,3 @@ -test \ No newline at end of file +test +.vscode +csrc/lib/jansson diff --git a/Makefile b/Makefile index c5a2dd7..406246d 100644 --- a/Makefile +++ b/Makefile @@ -3,17 +3,18 @@ BABEL := ./node_modules/.bin/babel THIS_FILE := $(lastword $(MAKEFILE_LIST)) BUILD_DIR := ./lib -SOURCES := index.js filtered-list-bust.lua sorted-filtered-list.lua groupped-list.lua +SOURCES := index.js $(BUILD_DIR)/%.js: %.js $(BABEL) $*.js -d $@ -$(BUILD_DIR)/%.lua: %.lua - cp $*.lua $@ - clean: rm -rf $(BUILD_DIR) build: $(foreach src, $(SOURCES), $(BUILD_DIR)/$(src)) all: clean build + +redis-module: + make -C csrc/ build + diff --git a/README.md b/README.md index 8d7555b..845c883 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,16 @@ [![Build Status](https://travis-ci.org/makeomatic/redis-filtered-sort.svg)](https://travis-ci.org/makeomatic/redis-filtered-sort) -Exports LUA script, which is able to perform multi filter operations, as well as sorts +Wraps Redis `FilterSortModule` api, which is able to perform multi filter operations, as well as sorts This basically replicates `http://redis.io/commands/sort` but with extra features and ability to run it in clustered mode with hashed keys, which resolve to the same slot +## Dependencies +Redis must have `FilterSortModule` enabled. +Please see [HOWTO](./doc/build.md) build module. +API provided by module in [API](./doc/api.md). + ## Installation `npm i redis-filtered-sort -S` diff --git a/csrc/.gitignore b/csrc/.gitignore new file mode 100644 index 0000000..1be334a --- /dev/null +++ b/csrc/.gitignore @@ -0,0 +1,7 @@ +*.o +*.a +*.so +*.db +.vscode +lib/rmutil/test_vector + diff --git a/csrc/Makefile b/csrc/Makefile new file mode 100644 index 0000000..2745bea --- /dev/null +++ b/csrc/Makefile @@ -0,0 +1,41 @@ +include variables.include + +ifdef IMAGE_NAME + DOCKER_IMAGE = $(IMAGE_NAME) +endif + +all: clean build + +docker-push: + docker push $(DOCKER_IMAGE) + +docker-build: + docker build . -f ./docker/Dockerfile -t $(DOCKER_IMAGE) +docker-rebuild: + docker build . --no-cache -f ./docker/Dockerfile -t $(DOCKER_IMAGE) + +build: + cd redis-filtered-sort && $(MAKE) all + +clean: + cd redis-filtered-sort && $(MAKE) $@ + +deps: jansson + + +clean-deps: + rm -rf $(JANSSON_LIBDIR)/* + +jansson: jansson_src jansson_configure + (cd $(JANSSON_LIBDIR) && make CFLAGS='$(CFLAGS)') + +jansson_configure: + (cd $(JANSSON_LIBDIR) && ./configure) + +jansson_clean: + (cd $(JANSSON_LIBDIR) && make clean) + +jansson_src: $(JANSSON_LIBDIR)/src/jansson.h + +$(JANSSON_LIBDIR)/src/jansson.h: + mkdir -p $(JANSSON_LIBDIR) && wget -O- $(JANSSON_LINK) | tar xvz --directory=$(JANSSON_LIBDIR) --strip-components=1 jansson-$(JANSSON_VERSION) \ No newline at end of file diff --git a/csrc/docker/Dockerfile b/csrc/docker/Dockerfile new file mode 100644 index 0000000..9785733 --- /dev/null +++ b/csrc/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine AS build-mod +RUN apk add --no-cache --virtual .build-deps \ + coreutils \ + gcc \ + linux-headers \ + make \ + musl-dev \ + && apk add \ + bash +RUN mkdir /src +WORKDIR /src + +ADD . /src/ +RUN make deps && make + +FROM redis:5.0.5-alpine +COPY --from=build-mod /src/redis-filtered-sort/filter_module.so /usr/local/lib/redis_filtered_sort.so +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["redis-server", "--loadmodule", "/usr/local/lib/redis_filtered_sort.so"] +EXPOSE 6379 diff --git a/csrc/lib/jansson/.gitkeep b/csrc/lib/jansson/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/csrc/lib/redismodule.h b/csrc/lib/redismodule.h new file mode 100644 index 0000000..c334d2c --- /dev/null +++ b/csrc/lib/redismodule.h @@ -0,0 +1,509 @@ +#ifndef REDISMODULE_H +#define REDISMODULE_H + +#include +#include +#include + +/* ---------------- Defines common between core and modules --------------- */ + +/* Error status return values. */ +#define REDISMODULE_OK 0 +#define REDISMODULE_ERR 1 + +/* API versions. */ +#define REDISMODULE_APIVER_1 1 + +/* API flags and constants */ +#define REDISMODULE_READ (1<<0) +#define REDISMODULE_WRITE (1<<1) + +#define REDISMODULE_LIST_HEAD 0 +#define REDISMODULE_LIST_TAIL 1 + +/* Key types. */ +#define REDISMODULE_KEYTYPE_EMPTY 0 +#define REDISMODULE_KEYTYPE_STRING 1 +#define REDISMODULE_KEYTYPE_LIST 2 +#define REDISMODULE_KEYTYPE_HASH 3 +#define REDISMODULE_KEYTYPE_SET 4 +#define REDISMODULE_KEYTYPE_ZSET 5 +#define REDISMODULE_KEYTYPE_MODULE 6 + +/* Reply types. */ +#define REDISMODULE_REPLY_UNKNOWN -1 +#define REDISMODULE_REPLY_STRING 0 +#define REDISMODULE_REPLY_ERROR 1 +#define REDISMODULE_REPLY_INTEGER 2 +#define REDISMODULE_REPLY_ARRAY 3 +#define REDISMODULE_REPLY_NULL 4 + +/* Postponed array length. */ +#define REDISMODULE_POSTPONED_ARRAY_LEN -1 + +/* Expire */ +#define REDISMODULE_NO_EXPIRE -1 + +/* Sorted set API flags. */ +#define REDISMODULE_ZADD_XX (1<<0) +#define REDISMODULE_ZADD_NX (1<<1) +#define REDISMODULE_ZADD_ADDED (1<<2) +#define REDISMODULE_ZADD_UPDATED (1<<3) +#define REDISMODULE_ZADD_NOP (1<<4) + +/* Hash API flags. */ +#define REDISMODULE_HASH_NONE 0 +#define REDISMODULE_HASH_NX (1<<0) +#define REDISMODULE_HASH_XX (1<<1) +#define REDISMODULE_HASH_CFIELDS (1<<2) +#define REDISMODULE_HASH_EXISTS (1<<3) + +/* Context Flags: Info about the current context returned by + * RM_GetContextFlags(). */ + +/* The command is running in the context of a Lua script */ +#define REDISMODULE_CTX_FLAGS_LUA (1<<0) +/* The command is running inside a Redis transaction */ +#define REDISMODULE_CTX_FLAGS_MULTI (1<<1) +/* The instance is a master */ +#define REDISMODULE_CTX_FLAGS_MASTER (1<<2) +/* The instance is a slave */ +#define REDISMODULE_CTX_FLAGS_SLAVE (1<<3) +/* The instance is read-only (usually meaning it's a slave as well) */ +#define REDISMODULE_CTX_FLAGS_READONLY (1<<4) +/* The instance is running in cluster mode */ +#define REDISMODULE_CTX_FLAGS_CLUSTER (1<<5) +/* The instance has AOF enabled */ +#define REDISMODULE_CTX_FLAGS_AOF (1<<6) +/* The instance has RDB enabled */ +#define REDISMODULE_CTX_FLAGS_RDB (1<<7) +/* The instance has Maxmemory set */ +#define REDISMODULE_CTX_FLAGS_MAXMEMORY (1<<8) +/* Maxmemory is set and has an eviction policy that may delete keys */ +#define REDISMODULE_CTX_FLAGS_EVICT (1<<9) +/* Redis is out of memory according to the maxmemory flag. */ +#define REDISMODULE_CTX_FLAGS_OOM (1<<10) +/* Less than 25% of memory available according to maxmemory. */ +#define REDISMODULE_CTX_FLAGS_OOM_WARNING (1<<11) + +#define REDISMODULE_NOTIFY_GENERIC (1<<2) /* g */ +#define REDISMODULE_NOTIFY_STRING (1<<3) /* $ */ +#define REDISMODULE_NOTIFY_LIST (1<<4) /* l */ +#define REDISMODULE_NOTIFY_SET (1<<5) /* s */ +#define REDISMODULE_NOTIFY_HASH (1<<6) /* h */ +#define REDISMODULE_NOTIFY_ZSET (1<<7) /* z */ +#define REDISMODULE_NOTIFY_EXPIRED (1<<8) /* x */ +#define REDISMODULE_NOTIFY_EVICTED (1<<9) /* e */ +#define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ + + +/* A special pointer that we can use between the core and the module to signal + * field deletion, and that is impossible to be a valid pointer. */ +#define REDISMODULE_HASH_DELETE ((RedisModuleString*)(long)1) + +/* Error messages. */ +#define REDISMODULE_ERRORMSG_WRONGTYPE "WRONGTYPE Operation against a key holding the wrong kind of value" + +#define REDISMODULE_POSITIVE_INFINITE (1.0/0.0) +#define REDISMODULE_NEGATIVE_INFINITE (-1.0/0.0) + +/* Cluster API defines. */ +#define REDISMODULE_NODE_ID_LEN 40 +#define REDISMODULE_NODE_MYSELF (1<<0) +#define REDISMODULE_NODE_MASTER (1<<1) +#define REDISMODULE_NODE_SLAVE (1<<2) +#define REDISMODULE_NODE_PFAIL (1<<3) +#define REDISMODULE_NODE_FAIL (1<<4) +#define REDISMODULE_NODE_NOFAILOVER (1<<5) + +#define REDISMODULE_CLUSTER_FLAG_NONE 0 +#define REDISMODULE_CLUSTER_FLAG_NO_FAILOVER (1<<1) +#define REDISMODULE_CLUSTER_FLAG_NO_REDIRECTION (1<<2) + +#define REDISMODULE_NOT_USED(V) ((void) V) + +/* This type represents a timer handle, and is returned when a timer is + * registered and used in order to invalidate a timer. It's just a 64 bit + * number, because this is how each timer is represented inside the radix tree + * of timers that are going to expire, sorted by expire time. */ +typedef uint64_t RedisModuleTimerID; + +/* ------------------------- End of common defines ------------------------ */ + +#ifndef REDISMODULE_CORE + +typedef long long mstime_t; + +/* Incomplete structures for compiler checks but opaque access. */ +typedef struct RedisModuleCtx RedisModuleCtx; +typedef struct RedisModuleKey RedisModuleKey; +typedef struct RedisModuleString RedisModuleString; +typedef struct RedisModuleCallReply RedisModuleCallReply; +typedef struct RedisModuleIO RedisModuleIO; +typedef struct RedisModuleType RedisModuleType; +typedef struct RedisModuleDigest RedisModuleDigest; +typedef struct RedisModuleBlockedClient RedisModuleBlockedClient; +typedef struct RedisModuleClusterInfo RedisModuleClusterInfo; +typedef struct RedisModuleDict RedisModuleDict; +typedef struct RedisModuleDictIter RedisModuleDictIter; + +typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); +typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); +typedef int (*RedisModuleNotificationFunc)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); +typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver); +typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value); +typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value); +typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value); +typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value); +typedef void (*RedisModuleTypeFreeFunc)(void *value); +typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); +typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); + +#define REDISMODULE_TYPE_METHOD_VERSION 1 +typedef struct RedisModuleTypeMethods { + uint64_t version; + RedisModuleTypeLoadFunc rdb_load; + RedisModuleTypeSaveFunc rdb_save; + RedisModuleTypeRewriteFunc aof_rewrite; + RedisModuleTypeMemUsageFunc mem_usage; + RedisModuleTypeDigestFunc digest; + RedisModuleTypeFreeFunc free; +} RedisModuleTypeMethods; + +#define REDISMODULE_GET_API(name) \ + RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name)) + +#define REDISMODULE_API_FUNC(x) (*x) + + +void *REDISMODULE_API_FUNC(RedisModule_Alloc)(size_t bytes); +void *REDISMODULE_API_FUNC(RedisModule_Realloc)(void *ptr, size_t bytes); +void REDISMODULE_API_FUNC(RedisModule_Free)(void *ptr); +void *REDISMODULE_API_FUNC(RedisModule_Calloc)(size_t nmemb, size_t size); +char *REDISMODULE_API_FUNC(RedisModule_Strdup)(const char *str); +int REDISMODULE_API_FUNC(RedisModule_GetApi)(const char *, void *); +int REDISMODULE_API_FUNC(RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep); +void REDISMODULE_API_FUNC(RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver); +int REDISMODULE_API_FUNC(RedisModule_IsModuleNameBusy)(const char *name); +int REDISMODULE_API_FUNC(RedisModule_WrongArity)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongLong)(RedisModuleCtx *ctx, long long ll); +int REDISMODULE_API_FUNC(RedisModule_GetSelectedDb)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_SelectDb)(RedisModuleCtx *ctx, int newid); +void *REDISMODULE_API_FUNC(RedisModule_OpenKey)(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode); +void REDISMODULE_API_FUNC(RedisModule_CloseKey)(RedisModuleKey *kp); +int REDISMODULE_API_FUNC(RedisModule_KeyType)(RedisModuleKey *kp); +size_t REDISMODULE_API_FUNC(RedisModule_ValueLength)(RedisModuleKey *kp); +int REDISMODULE_API_FUNC(RedisModule_ListPush)(RedisModuleKey *kp, int where, RedisModuleString *ele); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ListPop)(RedisModuleKey *key, int where); +RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_Call)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...); +const char *REDISMODULE_API_FUNC(RedisModule_CallReplyProto)(RedisModuleCallReply *reply, size_t *len); +void REDISMODULE_API_FUNC(RedisModule_FreeCallReply)(RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_CallReplyType)(RedisModuleCallReply *reply); +long long REDISMODULE_API_FUNC(RedisModule_CallReplyInteger)(RedisModuleCallReply *reply); +size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *reply); +RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str); +const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len); +void REDISMODULE_API_FUNC(RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll); +int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(const RedisModuleString *str, double *d); +void REDISMODULE_API_FUNC(RedisModule_AutoMemory)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...); +int REDISMODULE_API_FUNC(RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx); +const char *REDISMODULE_API_FUNC(RedisModule_CallReplyStringPtr)(RedisModuleCallReply *reply, size_t *len); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromCallReply)(RedisModuleCallReply *reply); +int REDISMODULE_API_FUNC(RedisModule_DeleteKey)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_UnlinkKey)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_StringSet)(RedisModuleKey *key, RedisModuleString *str); +char *REDISMODULE_API_FUNC(RedisModule_StringDMA)(RedisModuleKey *key, size_t *len, int mode); +int REDISMODULE_API_FUNC(RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen); +mstime_t REDISMODULE_API_FUNC(RedisModule_GetExpire)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire); +int REDISMODULE_API_FUNC(RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr); +int REDISMODULE_API_FUNC(RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore); +int REDISMODULE_API_FUNC(RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score); +int REDISMODULE_API_FUNC(RedisModule_ZsetRem)(RedisModuleKey *key, RedisModuleString *ele, int *deleted); +void REDISMODULE_API_FUNC(RedisModule_ZsetRangeStop)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex); +int REDISMODULE_API_FUNC(RedisModule_ZsetLastInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex); +int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max); +int REDISMODULE_API_FUNC(RedisModule_ZsetLastInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ZsetRangeCurrentElement)(RedisModuleKey *key, double *score); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangeNext)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangePrev)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_ZsetRangeEndReached)(RedisModuleKey *key); +int REDISMODULE_API_FUNC(RedisModule_HashSet)(RedisModuleKey *key, int flags, ...); +int REDISMODULE_API_FUNC(RedisModule_HashGet)(RedisModuleKey *key, int flags, ...); +int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos); +unsigned long long REDISMODULE_API_FUNC(RedisModule_GetClientId)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_GetContextFlags)(RedisModuleCtx *ctx); +void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes); +RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods); +int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); +RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); +void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); +void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); +uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); +int64_t REDISMODULE_API_FUNC(RedisModule_LoadSigned)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_EmitAOF)(RedisModuleIO *io, const char *cmdname, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule_SaveString)(RedisModuleIO *io, RedisModuleString *s); +void REDISMODULE_API_FUNC(RedisModule_SaveStringBuffer)(RedisModuleIO *io, const char *str, size_t len); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_LoadString)(RedisModuleIO *io); +char *REDISMODULE_API_FUNC(RedisModule_LoadStringBuffer)(RedisModuleIO *io, size_t *lenptr); +void REDISMODULE_API_FUNC(RedisModule_SaveDouble)(RedisModuleIO *io, double value); +double REDISMODULE_API_FUNC(RedisModule_LoadDouble)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value); +float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); +void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str); +int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); +RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io); +long long REDISMODULE_API_FUNC(RedisModule_Milliseconds)(void); +void REDISMODULE_API_FUNC(RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len); +void REDISMODULE_API_FUNC(RedisModule_DigestAddLongLong)(RedisModuleDigest *md, long long ele); +void REDISMODULE_API_FUNC(RedisModule_DigestEndSequence)(RedisModuleDigest *md); +RedisModuleDict *REDISMODULE_API_FUNC(RedisModule_CreateDict)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_FreeDict)(RedisModuleCtx *ctx, RedisModuleDict *d); +uint64_t REDISMODULE_API_FUNC(RedisModule_DictSize)(RedisModuleDict *d); +int REDISMODULE_API_FUNC(RedisModule_DictSetC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr); +int REDISMODULE_API_FUNC(RedisModule_DictReplaceC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr); +int REDISMODULE_API_FUNC(RedisModule_DictSet)(RedisModuleDict *d, RedisModuleString *key, void *ptr); +int REDISMODULE_API_FUNC(RedisModule_DictReplace)(RedisModuleDict *d, RedisModuleString *key, void *ptr); +void *REDISMODULE_API_FUNC(RedisModule_DictGetC)(RedisModuleDict *d, void *key, size_t keylen, int *nokey); +void *REDISMODULE_API_FUNC(RedisModule_DictGet)(RedisModuleDict *d, RedisModuleString *key, int *nokey); +int REDISMODULE_API_FUNC(RedisModule_DictDelC)(RedisModuleDict *d, void *key, size_t keylen, void *oldval); +int REDISMODULE_API_FUNC(RedisModule_DictDel)(RedisModuleDict *d, RedisModuleString *key, void *oldval); +RedisModuleDictIter *REDISMODULE_API_FUNC(RedisModule_DictIteratorStartC)(RedisModuleDict *d, const char *op, void *key, size_t keylen); +RedisModuleDictIter *REDISMODULE_API_FUNC(RedisModule_DictIteratorStart)(RedisModuleDict *d, const char *op, RedisModuleString *key); +void REDISMODULE_API_FUNC(RedisModule_DictIteratorStop)(RedisModuleDictIter *di); +int REDISMODULE_API_FUNC(RedisModule_DictIteratorReseekC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); +int REDISMODULE_API_FUNC(RedisModule_DictIteratorReseek)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); +void *REDISMODULE_API_FUNC(RedisModule_DictNextC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr); +void *REDISMODULE_API_FUNC(RedisModule_DictPrevC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictNext)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr); +int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); +int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); + +/* Experimental APIs */ +#ifdef REDISMODULE_EXPERIMENTAL_API +#define REDISMODULE_EXPERIMENTAL_API_VERSION 3 +RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms); +int REDISMODULE_API_FUNC(RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata); +int REDISMODULE_API_FUNC(RedisModule_IsBlockedReplyRequest)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_IsBlockedTimeoutRequest)(RedisModuleCtx *ctx); +void *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx *ctx); +RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientHandle)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_AbortBlock)(RedisModuleBlockedClient *bc); +RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc); +void REDISMODULE_API_FUNC(RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb); +int REDISMODULE_API_FUNC(RedisModule_BlockedClientDisconnected)(RedisModuleCtx *ctx); +void REDISMODULE_API_FUNC(RedisModule_RegisterClusterMessageReceiver)(RedisModuleCtx *ctx, uint8_t type, RedisModuleClusterMessageReceiver callback); +int REDISMODULE_API_FUNC(RedisModule_SendClusterMessage)(RedisModuleCtx *ctx, char *target_id, uint8_t type, unsigned char *msg, uint32_t len); +int REDISMODULE_API_FUNC(RedisModule_GetClusterNodeInfo)(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags); +char **REDISMODULE_API_FUNC(RedisModule_GetClusterNodesList)(RedisModuleCtx *ctx, size_t *numnodes); +void REDISMODULE_API_FUNC(RedisModule_FreeClusterNodesList)(char **ids); +RedisModuleTimerID REDISMODULE_API_FUNC(RedisModule_CreateTimer)(RedisModuleCtx *ctx, mstime_t period, RedisModuleTimerProc callback, void *data); +int REDISMODULE_API_FUNC(RedisModule_StopTimer)(RedisModuleCtx *ctx, RedisModuleTimerID id, void **data); +int REDISMODULE_API_FUNC(RedisModule_GetTimerInfo)(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remaining, void **data); +const char *REDISMODULE_API_FUNC(RedisModule_GetMyClusterID)(void); +size_t REDISMODULE_API_FUNC(RedisModule_GetClusterSize)(void); +void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t len); +void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len); +void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback); +void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags); +#endif + +/* This is included inline inside each Redis module. */ +static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused)); +static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { + void *getapifuncptr = ((void**)ctx)[0]; + RedisModule_GetApi = (int (*)(const char *, void *)) (unsigned long)getapifuncptr; + REDISMODULE_GET_API(Alloc); + REDISMODULE_GET_API(Calloc); + REDISMODULE_GET_API(Free); + REDISMODULE_GET_API(Realloc); + REDISMODULE_GET_API(Strdup); + REDISMODULE_GET_API(CreateCommand); + REDISMODULE_GET_API(SetModuleAttribs); + REDISMODULE_GET_API(IsModuleNameBusy); + REDISMODULE_GET_API(WrongArity); + REDISMODULE_GET_API(ReplyWithLongLong); + REDISMODULE_GET_API(ReplyWithError); + REDISMODULE_GET_API(ReplyWithSimpleString); + REDISMODULE_GET_API(ReplyWithArray); + REDISMODULE_GET_API(ReplySetArrayLength); + REDISMODULE_GET_API(ReplyWithStringBuffer); + REDISMODULE_GET_API(ReplyWithString); + REDISMODULE_GET_API(ReplyWithNull); + REDISMODULE_GET_API(ReplyWithCallReply); + REDISMODULE_GET_API(ReplyWithDouble); + REDISMODULE_GET_API(ReplySetArrayLength); + REDISMODULE_GET_API(GetSelectedDb); + REDISMODULE_GET_API(SelectDb); + REDISMODULE_GET_API(OpenKey); + REDISMODULE_GET_API(CloseKey); + REDISMODULE_GET_API(KeyType); + REDISMODULE_GET_API(ValueLength); + REDISMODULE_GET_API(ListPush); + REDISMODULE_GET_API(ListPop); + REDISMODULE_GET_API(StringToLongLong); + REDISMODULE_GET_API(StringToDouble); + REDISMODULE_GET_API(Call); + REDISMODULE_GET_API(CallReplyProto); + REDISMODULE_GET_API(FreeCallReply); + REDISMODULE_GET_API(CallReplyInteger); + REDISMODULE_GET_API(CallReplyType); + REDISMODULE_GET_API(CallReplyLength); + REDISMODULE_GET_API(CallReplyArrayElement); + REDISMODULE_GET_API(CallReplyStringPtr); + REDISMODULE_GET_API(CreateStringFromCallReply); + REDISMODULE_GET_API(CreateString); + REDISMODULE_GET_API(CreateStringFromLongLong); + REDISMODULE_GET_API(CreateStringFromString); + REDISMODULE_GET_API(CreateStringPrintf); + REDISMODULE_GET_API(FreeString); + REDISMODULE_GET_API(StringPtrLen); + REDISMODULE_GET_API(AutoMemory); + REDISMODULE_GET_API(Replicate); + REDISMODULE_GET_API(ReplicateVerbatim); + REDISMODULE_GET_API(DeleteKey); + REDISMODULE_GET_API(UnlinkKey); + REDISMODULE_GET_API(StringSet); + REDISMODULE_GET_API(StringDMA); + REDISMODULE_GET_API(StringTruncate); + REDISMODULE_GET_API(GetExpire); + REDISMODULE_GET_API(SetExpire); + REDISMODULE_GET_API(ZsetAdd); + REDISMODULE_GET_API(ZsetIncrby); + REDISMODULE_GET_API(ZsetScore); + REDISMODULE_GET_API(ZsetRem); + REDISMODULE_GET_API(ZsetRangeStop); + REDISMODULE_GET_API(ZsetFirstInScoreRange); + REDISMODULE_GET_API(ZsetLastInScoreRange); + REDISMODULE_GET_API(ZsetFirstInLexRange); + REDISMODULE_GET_API(ZsetLastInLexRange); + REDISMODULE_GET_API(ZsetRangeCurrentElement); + REDISMODULE_GET_API(ZsetRangeNext); + REDISMODULE_GET_API(ZsetRangePrev); + REDISMODULE_GET_API(ZsetRangeEndReached); + REDISMODULE_GET_API(HashSet); + REDISMODULE_GET_API(HashGet); + REDISMODULE_GET_API(IsKeysPositionRequest); + REDISMODULE_GET_API(KeyAtPos); + REDISMODULE_GET_API(GetClientId); + REDISMODULE_GET_API(GetContextFlags); + REDISMODULE_GET_API(PoolAlloc); + REDISMODULE_GET_API(CreateDataType); + REDISMODULE_GET_API(ModuleTypeSetValue); + REDISMODULE_GET_API(ModuleTypeGetType); + REDISMODULE_GET_API(ModuleTypeGetValue); + REDISMODULE_GET_API(SaveUnsigned); + REDISMODULE_GET_API(LoadUnsigned); + REDISMODULE_GET_API(SaveSigned); + REDISMODULE_GET_API(LoadSigned); + REDISMODULE_GET_API(SaveString); + REDISMODULE_GET_API(SaveStringBuffer); + REDISMODULE_GET_API(LoadString); + REDISMODULE_GET_API(LoadStringBuffer); + REDISMODULE_GET_API(SaveDouble); + REDISMODULE_GET_API(LoadDouble); + REDISMODULE_GET_API(SaveFloat); + REDISMODULE_GET_API(LoadFloat); + REDISMODULE_GET_API(EmitAOF); + REDISMODULE_GET_API(Log); + REDISMODULE_GET_API(LogIOError); + REDISMODULE_GET_API(StringAppendBuffer); + REDISMODULE_GET_API(RetainString); + REDISMODULE_GET_API(StringCompare); + REDISMODULE_GET_API(GetContextFromIO); + REDISMODULE_GET_API(Milliseconds); + REDISMODULE_GET_API(DigestAddStringBuffer); + REDISMODULE_GET_API(DigestAddLongLong); + REDISMODULE_GET_API(DigestEndSequence); + REDISMODULE_GET_API(CreateDict); + REDISMODULE_GET_API(FreeDict); + REDISMODULE_GET_API(DictSize); + REDISMODULE_GET_API(DictSetC); + REDISMODULE_GET_API(DictReplaceC); + REDISMODULE_GET_API(DictSet); + REDISMODULE_GET_API(DictReplace); + REDISMODULE_GET_API(DictGetC); + REDISMODULE_GET_API(DictGet); + REDISMODULE_GET_API(DictDelC); + REDISMODULE_GET_API(DictDel); + REDISMODULE_GET_API(DictIteratorStartC); + REDISMODULE_GET_API(DictIteratorStart); + REDISMODULE_GET_API(DictIteratorStop); + REDISMODULE_GET_API(DictIteratorReseekC); + REDISMODULE_GET_API(DictIteratorReseek); + REDISMODULE_GET_API(DictNextC); + REDISMODULE_GET_API(DictPrevC); + REDISMODULE_GET_API(DictNext); + REDISMODULE_GET_API(DictPrev); + REDISMODULE_GET_API(DictCompare); + REDISMODULE_GET_API(DictCompareC); + +#ifdef REDISMODULE_EXPERIMENTAL_API + REDISMODULE_GET_API(GetThreadSafeContext); + REDISMODULE_GET_API(FreeThreadSafeContext); + REDISMODULE_GET_API(ThreadSafeContextLock); + REDISMODULE_GET_API(ThreadSafeContextUnlock); + REDISMODULE_GET_API(BlockClient); + REDISMODULE_GET_API(UnblockClient); + REDISMODULE_GET_API(IsBlockedReplyRequest); + REDISMODULE_GET_API(IsBlockedTimeoutRequest); + REDISMODULE_GET_API(GetBlockedClientPrivateData); + REDISMODULE_GET_API(GetBlockedClientHandle); + REDISMODULE_GET_API(AbortBlock); + REDISMODULE_GET_API(SetDisconnectCallback); + REDISMODULE_GET_API(SubscribeToKeyspaceEvents); + REDISMODULE_GET_API(BlockedClientDisconnected); + REDISMODULE_GET_API(RegisterClusterMessageReceiver); + REDISMODULE_GET_API(SendClusterMessage); + REDISMODULE_GET_API(GetClusterNodeInfo); + REDISMODULE_GET_API(GetClusterNodesList); + REDISMODULE_GET_API(FreeClusterNodesList); + REDISMODULE_GET_API(CreateTimer); + REDISMODULE_GET_API(StopTimer); + REDISMODULE_GET_API(GetTimerInfo); + REDISMODULE_GET_API(GetMyClusterID); + REDISMODULE_GET_API(GetClusterSize); + REDISMODULE_GET_API(GetRandomBytes); + REDISMODULE_GET_API(GetRandomHexChars); + REDISMODULE_GET_API(SetClusterFlags); +#endif + + if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; + RedisModule_SetModuleAttribs(ctx,name,ver,apiver); + return REDISMODULE_OK; +} + +#else + +/* Things only defined for the modules core, not exported to modules + * including this file. */ +#define RedisModuleString robj + +#endif /* REDISMODULE_CORE */ +#endif /* REDISMOUDLE_H */ \ No newline at end of file diff --git a/csrc/lib/rmutil/Makefile b/csrc/lib/rmutil/Makefile new file mode 100644 index 0000000..09e023b --- /dev/null +++ b/csrc/lib/rmutil/Makefile @@ -0,0 +1,31 @@ +# set environment variable RM_INCLUDE_DIR to the location of redismodule.h +ifndef RM_INCLUDE_DIR + RM_INCLUDE_DIR=../ +endif + +CFLAGS ?= -g -fPIC -O3 -std=gnu99 -Wall -Wno-unused-function +CFLAGS += -I$(RM_INCLUDE_DIR) +CC=gcc + +OBJS=util.o strings.o sds.o vector.o alloc.o periodic.o + +all: librmutil.a + +clean: + rm -rf *.o *.a + +librmutil.a: $(OBJS) + ar rcs $@ $^ + +test_vector: test_vector.o vector.o + $(CC) -Wall -o $@ $^ -lc -lpthread -O0 + @(sh -c ./$@) +.PHONY: test_vector + +test_periodic: test_periodic.o periodic.o + $(CC) -Wall -o $@ $^ -lc -lpthread -O0 + @(sh -c ./$@) +.PHONY: test_periodic + +test: test_periodic test_vector +.PHONY: test diff --git a/csrc/lib/rmutil/alloc.c b/csrc/lib/rmutil/alloc.c new file mode 100644 index 0000000..6eee805 --- /dev/null +++ b/csrc/lib/rmutil/alloc.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include "alloc.h" + +/* A patched implementation of strdup that will use our patched calloc */ +char *rmalloc_strndup(const char *s, size_t n) { + char *ret = calloc(n + 1, sizeof(char)); + if (ret) + memcpy(ret, s, n); + return ret; +} + +/* + * Re-patching RedisModule_Alloc and friends to the original malloc functions + * + * This function should be called if you are working with malloc-patched code + * outside of redis, usually for unit tests. Call it once when entering your unit + * tests' main(). + * + * Since including "alloc.h" while defining REDIS_MODULE_TARGET + * replaces all malloc functions in redis with the RM_Alloc family of functions, + * when running that code outside of redis, your app will crash. This function + * patches the RM_Alloc functions back to the original mallocs. */ +void RMUTil_InitAlloc() { + + RedisModule_Alloc = malloc; + RedisModule_Realloc = realloc; + RedisModule_Calloc = calloc; + RedisModule_Free = free; + RedisModule_Strdup = strdup; +} diff --git a/csrc/lib/rmutil/alloc.h b/csrc/lib/rmutil/alloc.h new file mode 100644 index 0000000..050ff72 --- /dev/null +++ b/csrc/lib/rmutil/alloc.h @@ -0,0 +1,51 @@ +#ifndef __RMUTIL_ALLOC__ +#define __RMUTIL_ALLOC__ + +/* Automatic Redis Module Allocation functions monkey-patching. + * + * Including this file while REDIS_MODULE_TARGET is defined, will explicitly + * override malloc, calloc, realloc & free with RedisModule_Alloc, + * RedisModule_Callc, etc implementations, that allow Redis better control and + * reporting over allocations per module. + * + * You should include this file in all c files AS THE LAST INCLUDED FILE + * + * This only has effect when when compiling with the macro REDIS_MODULE_TARGET + * defined. The idea is that for unit tests it will not be defined, but for the + * module build target it will be. + * + */ + +#include +#include + +char *rmalloc_strndup(const char *s, size_t n); + +#ifdef REDIS_MODULE_TARGET /* Set this when compiling your code as a module */ + +#define malloc(size) RedisModule_Alloc(size) +#define calloc(count, size) RedisModule_Calloc(count, size) +#define realloc(ptr, size) RedisModule_Realloc(ptr, size) +#define free(ptr) RedisModule_Free(ptr) + +#ifdef strdup +#undef strdup +#endif +#define strdup(ptr) RedisModule_Strdup(ptr) + +/* More overriding */ +// needed to avoid calling strndup->malloc +#ifdef strndup +#undef strndup +#endif +#define strndup(s, n) rmalloc_strndup(s, n) + +#else + +#endif /* REDIS_MODULE_TARGET */ +/* This function should be called if you are working with malloc-patched code + * outside of redis, usually for unit tests. Call it once when entering your unit + * tests' main() */ +void RMUTil_InitAlloc(); + +#endif /* __RMUTIL_ALLOC__ */ diff --git a/csrc/lib/rmutil/heap.c b/csrc/lib/rmutil/heap.c new file mode 100644 index 0000000..fd1fffb --- /dev/null +++ b/csrc/lib/rmutil/heap.c @@ -0,0 +1,107 @@ +#include "heap.h" + +/* Byte-wise swap two items of size SIZE. */ +#define SWAP(a, b, size) \ + do \ + { \ + register size_t __size = (size); \ + register char *__a = (a), *__b = (b); \ + do \ + { \ + char __tmp = *__a; \ + *__a++ = *__b; \ + *__b++ = __tmp; \ + } while (--__size > 0); \ + } while (0) + +inline char *__vector_GetPtr(Vector *v, size_t pos) { + return v->data + (pos * v->elemSize); +} + +void __sift_up(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)) { + size_t len = last - first; + if (len > 1) { + len = (len - 2) / 2; + size_t ptr = first + len; + if (cmp(__vector_GetPtr(v, ptr), __vector_GetPtr(v, --last)) < 0) { + char t[v->elemSize]; + memcpy(t, __vector_GetPtr(v, last), v->elemSize); + do { + memcpy(__vector_GetPtr(v, last), __vector_GetPtr(v, ptr), v->elemSize); + last = ptr; + if (len == 0) + break; + len = (len - 1) / 2; + ptr = first + len; + } while (cmp(__vector_GetPtr(v, ptr), t) < 0); + memcpy(__vector_GetPtr(v, last), t, v->elemSize); + } + } +} + +void __sift_down(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *), size_t start) { + // left-child of __start is at 2 * __start + 1 + // right-child of __start is at 2 * __start + 2 + size_t len = last - first; + size_t child = start - first; + + if (len < 2 || (len - 2) / 2 < child) + return; + + child = 2 * child + 1; + + if ((child + 1) < len && cmp(__vector_GetPtr(v, first + child), __vector_GetPtr(v, first + child + 1)) < 0) { + // right-child exists and is greater than left-child + ++child; + } + + // check if we are in heap-order + if (cmp(__vector_GetPtr(v, first + child), __vector_GetPtr(v, start)) < 0) + // we are, __start is larger than it's largest child + return; + + char top[v->elemSize]; + memcpy(top, __vector_GetPtr(v, start), v->elemSize); + do { + // we are not in heap-order, swap the parent with it's largest child + memcpy(__vector_GetPtr(v, start), __vector_GetPtr(v, first + child), v->elemSize); + start = first + child; + + if ((len - 2) / 2 < child) + break; + + // recompute the child based off of the updated parent + child = 2 * child + 1; + + if ((child + 1) < len && cmp(__vector_GetPtr(v, first + child), __vector_GetPtr(v, first + child + 1)) < 0) { + // right-child exists and is greater than left-child + ++child; + } + + // check if we are in heap-order + } while (cmp(__vector_GetPtr(v, first + child), top) >= 0); + memcpy(__vector_GetPtr(v, start), top, v->elemSize); +} + + +void Make_Heap(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)) { + if (last - first > 1) { + // start from the first parent, there is no need to consider children + for (int start = (last - first - 2) / 2; start >= 0; --start) { + __sift_down(v, first, last, cmp, first + start); + } + } +} + + +inline void Heap_Push(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)) { + __sift_up(v, first, last, cmp); +} + + +inline void Heap_Pop(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)) { + if (last - first > 1) { + SWAP(__vector_GetPtr(v, first), __vector_GetPtr(v, --last), v->elemSize); + __sift_down(v, first, last, cmp, first); + } +} diff --git a/csrc/lib/rmutil/heap.h b/csrc/lib/rmutil/heap.h new file mode 100644 index 0000000..b49efce --- /dev/null +++ b/csrc/lib/rmutil/heap.h @@ -0,0 +1,38 @@ +#ifndef __HEAP_H__ +#define __HEAP_H__ + +#include "vector.h" + + +/* Make heap from range + * Rearranges the elements in the range [first,last) in such a way that they form a heap. + * A heap is a way to organize the elements of a range that allows for fast retrieval of the element with the highest + * value at any moment (with pop_heap), even repeatedly, while allowing for fast insertion of new elements (with + * push_heap). + * The element with the highest value is always pointed by first. The order of the other elements depends on the + * particular implementation, but it is consistent throughout all heap-related functions of this header. + * The elements are compared using cmp. + */ +void Make_Heap(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)); + + +/* Push element into heap range + * Given a heap in the range [first,last-1), this function extends the range considered a heap to [first,last) by + * placing the value in (last-1) into its corresponding location within it. + * A range can be organized into a heap by calling make_heap. After that, its heap properties are preserved if elements + * are added and removed from it using push_heap and pop_heap, respectively. + */ +void Heap_Push(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)); + + +/* Pop element from heap range + * Rearranges the elements in the heap range [first,last) in such a way that the part considered a heap is shortened + * by one: The element with the highest value is moved to (last-1). + * While the element with the highest value is moved from first to (last-1) (which now is out of the heap), the other + * elements are reorganized in such a way that the range [first,last-1) preserves the properties of a heap. + * A range can be organized into a heap by calling make_heap. After that, its heap properties are preserved if elements + * are added and removed from it using push_heap and pop_heap, respectively. + */ +void Heap_Pop(Vector *v, size_t first, size_t last, int (*cmp)(void *, void *)); + +#endif //__HEAP_H__ diff --git a/csrc/lib/rmutil/logging.h b/csrc/lib/rmutil/logging.h new file mode 100644 index 0000000..ed92cb3 --- /dev/null +++ b/csrc/lib/rmutil/logging.h @@ -0,0 +1,11 @@ +#ifndef __RMUTIL_LOGGING_H__ +#define __RMUTIL_LOGGING_H__ + +/* Convenience macros for redis logging */ + +#define RM_LOG_DEBUG(ctx, ...) RedisModule_Log(ctx, "debug", __VA_ARGS__) +#define RM_LOG_VERBOSE(ctx, ...) RedisModule_Log(ctx, "verbose", __VA_ARGS__) +#define RM_LOG_NOTICE(ctx, ...) RedisModule_Log(ctx, "notice", __VA_ARGS__) +#define RM_LOG_WARNING(ctx, ...) RedisModule_Log(ctx, "warning", __VA_ARGS__) + +#endif \ No newline at end of file diff --git a/csrc/lib/rmutil/periodic.c b/csrc/lib/rmutil/periodic.c new file mode 100644 index 0000000..dbddceb --- /dev/null +++ b/csrc/lib/rmutil/periodic.c @@ -0,0 +1,88 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "periodic.h" +#include +#include +#include + +typedef struct RMUtilTimer { + RMutilTimerFunc cb; + RMUtilTimerTerminationFunc onTerm; + void *privdata; + struct timespec interval; + pthread_t thread; + pthread_mutex_t lock; + pthread_cond_t cond; +} RMUtilTimer; + +static struct timespec timespecAdd(struct timespec *a, struct timespec *b) { + struct timespec ret; + ret.tv_sec = a->tv_sec + b->tv_sec; + + long long ns = a->tv_nsec + b->tv_nsec; + ret.tv_sec += ns / 1000000000; + ret.tv_nsec = ns % 1000000000; + return ret; +} + +static void *rmutilTimer_Loop(void *ctx) { + RMUtilTimer *tm = ctx; + + int rc = ETIMEDOUT; + struct timespec ts; + + pthread_mutex_lock(&tm->lock); + while (rc != 0) { + clock_gettime(CLOCK_REALTIME, &ts); + struct timespec timeout = timespecAdd(&ts, &tm->interval); + if ((rc = pthread_cond_timedwait(&tm->cond, &tm->lock, &timeout)) == ETIMEDOUT) { + + // Create a thread safe context if we're running inside redis + RedisModuleCtx *rctx = NULL; + if (RedisModule_GetThreadSafeContext) rctx = RedisModule_GetThreadSafeContext(NULL); + + // call our callback... + tm->cb(rctx, tm->privdata); + + // If needed - free the thread safe context. + // It's up to the user to decide whether automemory is active there + if (rctx) RedisModule_FreeThreadSafeContext(rctx); + } + if (rc == EINVAL) { + perror("Error waiting for condition"); + break; + } + } + + // call the termination callback if needed + if (tm->onTerm != NULL) { + tm->onTerm(tm->privdata); + } + + // free resources associated with the timer + pthread_cond_destroy(&tm->cond); + free(tm); + + return NULL; +} + +/* set a new frequency for the timer. This will take effect AFTER the next trigger */ +void RMUtilTimer_SetInterval(struct RMUtilTimer *t, struct timespec newInterval) { + t->interval = newInterval; +} + +RMUtilTimer *RMUtil_NewPeriodicTimer(RMutilTimerFunc cb, RMUtilTimerTerminationFunc onTerm, + void *privdata, struct timespec interval) { + RMUtilTimer *ret = malloc(sizeof(*ret)); + *ret = (RMUtilTimer){ + .privdata = privdata, .interval = interval, .cb = cb, .onTerm = onTerm, + }; + pthread_cond_init(&ret->cond, NULL); + pthread_mutex_init(&ret->lock, NULL); + + pthread_create(&ret->thread, NULL, rmutilTimer_Loop, ret); + return ret; +} + +int RMUtilTimer_Terminate(struct RMUtilTimer *t) { + return pthread_cond_signal(&t->cond); +} diff --git a/csrc/lib/rmutil/periodic.h b/csrc/lib/rmutil/periodic.h new file mode 100644 index 0000000..6740072 --- /dev/null +++ b/csrc/lib/rmutil/periodic.h @@ -0,0 +1,46 @@ +#ifndef RMUTIL_PERIODIC_H_ +#define RMUTIL_PERIODIC_H_ +#include +#include + +/** periodic.h - Utility periodic timer running a task repeatedly every given time interval */ + +/* RMUtilTimer - opaque context for the timer */ +struct RMUtilTimer; + +/* RMutilTimerFunc - callback type for timer tasks. The ctx is a thread-safe redis module context + * that should be locked/unlocked by the callback when running stuff against redis. privdata is + * pre-existing private data */ +typedef void (*RMutilTimerFunc)(RedisModuleCtx *ctx, void *privdata); + +typedef void (*RMUtilTimerTerminationFunc)(void *privdata); + +/* Create and start a new periodic timer. Each timer has its own thread and can only be run and + * stopped once. The timer runs `cb` every `interval` with `privdata` passed to the callback. */ +struct RMUtilTimer *RMUtil_NewPeriodicTimer(RMutilTimerFunc cb, RMUtilTimerTerminationFunc onTerm, + void *privdata, struct timespec interval); + +/* set a new frequency for the timer. This will take effect AFTER the next trigger */ +void RMUtilTimer_SetInterval(struct RMUtilTimer *t, struct timespec newInterval); + +/* Stop the timer loop, call the termination callbck to free up any resources linked to the timer, + * and free the timer after stopping. + * + * This function doesn't wait for the thread to terminate, as it may cause a race condition if the + * timer's callback is waiting for the redis global lock. + * Instead you should make sure any resources are freed by the callback after the thread loop is + * finished. + * + * The timer is freed automatically, so the callback doesn't need to do anything about it. + * The callback gets the timer's associated privdata as its argument. + * + * If no callback is specified we do not free up privdata. If privdata is NULL we still call the + * callback, as it may log stuff or free global resources. + */ +int RMUtilTimer_Terminate(struct RMUtilTimer *t); + +/* DEPRECATED - do not use this function (well now you can't), use terminate instead + Free the timer context. The caller should be responsible for freeing the private data at this + * point */ +// void RMUtilTimer_Free(struct RMUtilTimer *t); +#endif \ No newline at end of file diff --git a/csrc/lib/rmutil/priority_queue.c b/csrc/lib/rmutil/priority_queue.c new file mode 100644 index 0000000..5c16c08 --- /dev/null +++ b/csrc/lib/rmutil/priority_queue.c @@ -0,0 +1,36 @@ +#include "priority_queue.h" +#include "heap.h" + +PriorityQueue *__newPriorityQueueSize(size_t elemSize, size_t cap, int (*cmp)(void *, void *)) { + PriorityQueue *pq = malloc(sizeof(PriorityQueue)); + pq->v = __newVectorSize(elemSize, cap); + pq->cmp = cmp; + return pq; +} + +inline size_t Priority_Queue_Size(PriorityQueue *pq) { + return Vector_Size(pq->v); +} + +inline int Priority_Queue_Top(PriorityQueue *pq, void *ptr) { + return Vector_Get(pq->v, 0, ptr); +} + +inline size_t __priority_Queue_PushPtr(PriorityQueue *pq, void *elem) { + size_t top = __vector_PushPtr(pq->v, elem); + Heap_Push(pq->v, 0, top, pq->cmp); + return top; +} + +inline void Priority_Queue_Pop(PriorityQueue *pq) { + if (pq->v->top == 0) { + return; + } + Heap_Pop(pq->v, 0, pq->v->top, pq->cmp); + pq->v->top--; +} + +void Priority_Queue_Free(PriorityQueue *pq) { + Vector_Free(pq->v); + free(pq); +} diff --git a/csrc/lib/rmutil/priority_queue.h b/csrc/lib/rmutil/priority_queue.h new file mode 100644 index 0000000..89f8c73 --- /dev/null +++ b/csrc/lib/rmutil/priority_queue.h @@ -0,0 +1,55 @@ +#ifndef __PRIORITY_QUEUE_H__ +#define __PRIORITY_QUEUE_H__ + +#include "vector.h" + +/* Priority queue + * Priority queues are designed such that its first element is always the greatest of the elements it contains. + * This context is similar to a heap, where elements can be inserted at any moment, and only the max heap element can be + * retrieved (the one at the top in the priority queue). + * Priority queues are implemented as Vectors. Elements are popped from the "back" of Vector, which is known as the top + * of the priority queue. + */ +typedef struct { + Vector *v; + + int (*cmp)(void *, void *); +} PriorityQueue; + +/* Construct priority queue + * Constructs a priority_queue container adaptor object. + */ +PriorityQueue *__newPriorityQueueSize(size_t elemSize, size_t cap, int (*cmp)(void *, void *)); + +#define NewPriorityQueue(type, cap, cmp) __newPriorityQueueSize(sizeof(type), cap, cmp) + +/* Return size + * Returns the number of elements in the priority_queue. + */ +size_t Priority_Queue_Size(PriorityQueue *pq); + +/* Access top element + * Copy the top element in the priority_queue to ptr. + * The top element is the element that compares higher in the priority_queue. + */ +int Priority_Queue_Top(PriorityQueue *pq, void *ptr); + +/* Insert element + * Inserts a new element in the priority_queue. + */ +size_t __priority_Queue_PushPtr(PriorityQueue *pq, void *elem); + +#define Priority_Queue_Push(pq, elem) __priority_Queue_PushPtr(pq, &(typeof(elem)){elem}) + +/* Remove top element + * Removes the element on top of the priority_queue, effectively reducing its size by one. The element removed is the + * one with the highest value. + * The value of this element can be retrieved before being popped by calling Priority_Queue_Top. + */ +void Priority_Queue_Pop(PriorityQueue *pq); + +/* free the priority queue and the underlying data. Does not release its elements if + * they are pointers */ +void Priority_Queue_Free(PriorityQueue *pq); + +#endif //__PRIORITY_QUEUE_H__ diff --git a/csrc/lib/rmutil/sds.c b/csrc/lib/rmutil/sds.c new file mode 100644 index 0000000..e3dd673 --- /dev/null +++ b/csrc/lib/rmutil/sds.c @@ -0,0 +1,1274 @@ +/* SDSLib 2.0 -- A C dynamic strings library + * + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Oran Agra + * Copyright (c) 2015, Redis Labs, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include "sds.h" +#include "sdsalloc.h" + +static inline int sdsHdrSize(char type) { + switch(type&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return sizeof(struct sdshdr5); + case SDS_TYPE_8: + return sizeof(struct sdshdr8); + case SDS_TYPE_16: + return sizeof(struct sdshdr16); + case SDS_TYPE_32: + return sizeof(struct sdshdr32); + case SDS_TYPE_64: + return sizeof(struct sdshdr64); + } + return 0; +} + +static inline char sdsReqType(size_t string_size) { + if (string_size < 32) + return SDS_TYPE_5; + if (string_size < 0xff) + return SDS_TYPE_8; + if (string_size < 0xffff) + return SDS_TYPE_16; + if (string_size < 0xffffffff) + return SDS_TYPE_32; + return SDS_TYPE_64; +} + +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ +sds sdsnewlen(const void *init, size_t initlen) { + void *sh; + sds s; + char type = sdsReqType(initlen); + /* Empty strings are usually created in order to append. Use type 8 + * since type 5 is not good at this. */ + if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; + int hdrlen = sdsHdrSize(type); + unsigned char *fp; /* flags pointer. */ + + sh = s_malloc(hdrlen+initlen+1); + if (!init) + memset(sh, 0, hdrlen+initlen+1); + if (sh == NULL) return NULL; + s = (char*)sh+hdrlen; + fp = ((unsigned char*)s)-1; + switch(type) { + case SDS_TYPE_5: { + *fp = type | (initlen << SDS_TYPE_BITS); + break; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + sh->len = initlen; + sh->alloc = initlen; + *fp = type; + break; + } + } + if (initlen && init) + memcpy(s, init, initlen); + s[initlen] = '\0'; + return s; +} + +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ +sds sdsempty(void) { + return sdsnewlen("",0); +} + +/* Create a new sds string starting from a null terminated C string. */ +sds sdsnew(const char *init) { + size_t initlen = (init == NULL) ? 0 : strlen(init); + return sdsnewlen(init, initlen); +} + +/* Duplicate an sds string. */ +sds sdsdup(const sds s) { + return sdsnewlen(s, sdslen(s)); +} + +/* Free an sds string. No operation is performed if 's' is NULL. */ +void sdsfree(sds s) { + if (s == NULL) return; + s_free((char*)s-sdsHdrSize(s[-1])); +} + +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ +void sdsupdatelen(sds s) { + int reallen = strlen(s); + sdssetlen(s, reallen); +} + +/* Modify an sds string in-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ +void sdsclear(sds s) { + sdssetlen(s, 0); + s[0] = '\0'; +} + +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *length* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { + void *sh, *newsh; + size_t avail = sdsavail(s); + size_t len, newlen; + char type, oldtype = s[-1] & SDS_TYPE_MASK; + int hdrlen; + + /* Return ASAP if there is enough space left. */ + if (avail >= addlen) return s; + + len = sdslen(s); + sh = (char*)s-sdsHdrSize(oldtype); + newlen = (len+addlen); + if (newlen < SDS_MAX_PREALLOC) + newlen *= 2; + else + newlen += SDS_MAX_PREALLOC; + + type = sdsReqType(newlen); + + /* Don't use type 5: the user is appending to the string and type 5 is + * not able to remember empty space, so sdsMakeRoomFor() must be called + * at every appending operation. */ + if (type == SDS_TYPE_5) type = SDS_TYPE_8; + + hdrlen = sdsHdrSize(type); + if (oldtype==type) { + newsh = s_realloc(sh, hdrlen+newlen+1); + if (newsh == NULL) return NULL; + s = (char*)newsh+hdrlen; + } else { + /* Since the header size changes, need to move the string forward, + * and can't use realloc */ + newsh = s_malloc(hdrlen+newlen+1); + if (newsh == NULL) return NULL; + memcpy((char*)newsh+hdrlen, s, len+1); + s_free(sh); + s = (char*)newsh+hdrlen; + s[-1] = type; + sdssetlen(s, len); + } + sdssetalloc(s, newlen); + return s; +} + +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdsRemoveFreeSpace(sds s) { + void *sh, *newsh; + char type, oldtype = s[-1] & SDS_TYPE_MASK; + int hdrlen; + size_t len = sdslen(s); + sh = (char*)s-sdsHdrSize(oldtype); + + type = sdsReqType(len); + hdrlen = sdsHdrSize(type); + if (oldtype==type) { + newsh = s_realloc(sh, hdrlen+len+1); + if (newsh == NULL) return NULL; + s = (char*)newsh+hdrlen; + } else { + newsh = s_malloc(hdrlen+len+1); + if (newsh == NULL) return NULL; + memcpy((char*)newsh+hdrlen, s, len+1); + s_free(sh); + s = (char*)newsh+hdrlen; + s[-1] = type; + sdssetlen(s, len); + } + sdssetalloc(s, len); + return s; +} + +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ +size_t sdsAllocSize(sds s) { + size_t alloc = sdsalloc(s); + return sdsHdrSize(s[-1])+alloc+1; +} + +/* Return the pointer of the actual SDS allocation (normally SDS strings + * are referenced by the start of the string buffer). */ +void *sdsAllocPtr(sds s) { + return (void*) (s-sdsHdrSize(s[-1])); +} + +/* Increment the sds length and decrements the left free space at the + * end of the string according to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Usage example: + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nread); + */ +void sdsIncrLen(sds s, int incr) { + unsigned char flags = s[-1]; + size_t len; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: { + unsigned char *fp = ((unsigned char*)s)-1; + unsigned char oldlen = SDS_TYPE_5_LEN(flags); + assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr))); + *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS); + len = oldlen+incr; + break; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); + len = (sh->len += incr); + break; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr))); + len = (sh->len += incr); + break; + } + default: len = 0; /* Just to avoid compilation warnings. */ + } + s[len] = '\0'; +} + +/* Grow the sds to have the specified length. Bytes that were not part of + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ +sds sdsgrowzero(sds s, size_t len) { + size_t curlen = sdslen(s); + + if (len <= curlen) return s; + s = sdsMakeRoomFor(s,len-curlen); + if (s == NULL) return NULL; + + /* Make sure added region doesn't contain garbage */ + memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ + sdssetlen(s, len); + return s; +} + +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatlen(sds s, const void *t, size_t len) { + size_t curlen = sdslen(s); + + s = sdsMakeRoomFor(s,len); + if (s == NULL) return NULL; + memcpy(s+curlen, t, len); + sdssetlen(s, curlen+len); + s[curlen+len] = '\0'; + return s; +} + +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscat(sds s, const char *t) { + return sdscatlen(s, t, strlen(t)); +} + +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatsds(sds s, const sds t) { + return sdscatlen(s, t, sdslen(t)); +} + +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ +sds sdscpylen(sds s, const char *t, size_t len) { + if (sdsalloc(s) < len) { + s = sdsMakeRoomFor(s,len-sdslen(s)); + if (s == NULL) return NULL; + } + memcpy(s, t, len); + s[len] = '\0'; + sdssetlen(s, len); + return s; +} + +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ +sds sdscpy(sds s, const char *t) { + return sdscpylen(s, t, strlen(t)); +} + +/* Helper for sdscatlonglong() doing the actual number -> string + * conversion. 's' must point to a string with room for at least + * SDS_LLSTR_SIZE bytes. + * + * The function returns the length of the null-terminated string + * representation stored at 's'. */ +#define SDS_LLSTR_SIZE 21 +int sdsll2str(char *s, long long value) { + char *p, aux; + unsigned long long v; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + v = (value < 0) ? -value : value; + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p++ = '-'; + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Identical sdsll2str(), but for unsigned long long type. */ +int sdsull2str(char *s, unsigned long long v) { + char *p, aux; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ +sds sdsfromlonglong(long long value) { + char buf[SDS_LLSTR_SIZE]; + int len = sdsll2str(buf,value); + + return sdsnewlen(buf,len); +} + +/* Like sdscatprintf() but gets va_list instead of being variadic. */ +sds sdscatvprintf(sds s, const char *fmt, va_list ap) { + va_list cpy; + char staticbuf[1024], *buf = staticbuf, *t; + size_t buflen = strlen(fmt)*2; + + /* We try to start using a static buffer for speed. + * If not possible we revert to heap allocation. */ + if (buflen > sizeof(staticbuf)) { + buf = s_malloc(buflen); + if (buf == NULL) return NULL; + } else { + buflen = sizeof(staticbuf); + } + + /* Try with buffers two times bigger every time we fail to + * fit the string in the current buffer size. */ + while(1) { + buf[buflen-2] = '\0'; + va_copy(cpy,ap); + vsnprintf(buf, buflen, fmt, cpy); + va_end(cpy); + if (buf[buflen-2] != '\0') { + if (buf != staticbuf) s_free(buf); + buflen *= 2; + buf = s_malloc(buflen); + if (buf == NULL) return NULL; + continue; + } + break; + } + + /* Finally concat the obtained string to the SDS string and return it. */ + t = sdscat(s, buf); + if (buf != staticbuf) s_free(buf); + return t; +} + +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ +sds sdscatprintf(sds s, const char *fmt, ...) { + va_list ap; + char *t; + va_start(ap, fmt); + t = sdscatvprintf(s,fmt,ap); + va_end(ap); + return t; +} + +/* This function is similar to sdscatprintf, but much faster as it does + * not rely on sprintf() family functions implemented by the libc that + * are often very slow. Moreover directly handling the sds string as + * new data is concatenated provides a performance improvement. + * + * However this function only handles an incompatible subset of printf-alike + * format specifiers: + * + * %s - C String + * %S - SDS string + * %i - signed int + * %I - 64 bit signed integer (long long, int64_t) + * %u - unsigned int + * %U - 64 bit unsigned integer (unsigned long long, uint64_t) + * %% - Verbatim "%" character. + */ +sds sdscatfmt(sds s, char const *fmt, ...) { + size_t initlen = sdslen(s); + const char *f = fmt; + int i; + va_list ap; + + va_start(ap,fmt); + f = fmt; /* Next format specifier byte to process. */ + i = initlen; /* Position of the next byte to write to dest str. */ + while(*f) { + char next, *str; + size_t l; + long long num; + unsigned long long unum; + + /* Make sure there is always space for at least 1 char. */ + if (sdsavail(s)==0) { + s = sdsMakeRoomFor(s,1); + } + + switch(*f) { + case '%': + next = *(f+1); + f++; + switch(next) { + case 's': + case 'S': + str = va_arg(ap,char*); + l = (next == 's') ? strlen(str) : sdslen(str); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,str,l); + sdsinclen(s,l); + i += l; + break; + case 'i': + case 'I': + if (next == 'i') + num = va_arg(ap,int); + else + num = va_arg(ap,long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsll2str(buf,num); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,buf,l); + sdsinclen(s,l); + i += l; + } + break; + case 'u': + case 'U': + if (next == 'u') + unum = va_arg(ap,unsigned int); + else + unum = va_arg(ap,unsigned long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsull2str(buf,unum); + if (sdsavail(s) < l) { + s = sdsMakeRoomFor(s,l); + } + memcpy(s+i,buf,l); + sdsinclen(s,l); + i += l; + } + break; + default: /* Handle %% and generally %. */ + s[i++] = next; + sdsinclen(s,1); + break; + } + break; + default: + s[i++] = *f; + sdsinclen(s,1); + break; + } + f++; + } + va_end(ap); + + /* Add null-term */ + s[i] = '\0'; + return s; +} + +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"Aa. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ +sds sdstrim(sds s, const char *cset) { + char *start, *end, *sp, *ep; + size_t len; + + sp = start = s; + ep = end = s+sdslen(s)-1; + while(sp <= end && strchr(cset, *sp)) sp++; + while(ep > sp && strchr(cset, *ep)) ep--; + len = (sp > ep) ? 0 : ((ep-sp)+1); + if (s != sp) memmove(s, sp, len); + s[len] = '\0'; + sdssetlen(s,len); + return s; +} + +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdsrange(s,1,-1); => "ello World" + */ +void sdsrange(sds s, int start, int end) { + size_t newlen, len = sdslen(s); + + if (len == 0) return; + if (start < 0) { + start = len+start; + if (start < 0) start = 0; + } + if (end < 0) { + end = len+end; + if (end < 0) end = 0; + } + newlen = (start > end) ? 0 : (end-start)+1; + if (newlen != 0) { + if (start >= (signed)len) { + newlen = 0; + } else if (end >= (signed)len) { + end = len-1; + newlen = (start > end) ? 0 : (end-start)+1; + } + } else { + start = 0; + } + if (start && newlen) memmove(s, s+start, newlen); + s[newlen] = 0; + sdssetlen(s,newlen); +} + +/* Apply tolower() to every character of the sds string 's'. */ +void sdstolower(sds s) { + int len = sdslen(s), j; + + for (j = 0; j < len; j++) s[j] = tolower(s[j]); +} + +/* Apply toupper() to every character of the sds string 's'. */ +void sdstoupper(sds s) { + int len = sdslen(s), j; + + for (j = 0; j < len; j++) s[j] = toupper(s[j]); +} + +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * positive if s1 > s2. + * negative if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ +int sdscmp(const sds s1, const sds s2) { + size_t l1, l2, minlen; + int cmp; + + l1 = sdslen(s1); + l2 = sdslen(s2); + minlen = (l1 < l2) ? l1 : l2; + cmp = memcmp(s1,s2,minlen); + if (cmp == 0) return l1-l2; + return cmp; +} + +/* Split 's' with separator in 'sep'. An array + * of sds strings is returned. *count will be set + * by reference to the number of tokens returned. + * + * On out of memory, zero length string, zero length + * separator, NULL is returned. + * + * Note that 'sep' is able to split a string using + * a multi-character separator. For example + * sdssplit("foo_-_bar","_-_"); will return two + * elements "foo" and "bar". + * + * This version of the function is binary-safe but + * requires length arguments. sdssplit() is just the + * same function but for zero-terminated strings. + */ +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { + int elements = 0, slots = 5, start = 0, j; + sds *tokens; + + if (seplen < 1 || len < 0) return NULL; + + tokens = s_malloc(sizeof(sds)*slots); + if (tokens == NULL) return NULL; + + if (len == 0) { + *count = 0; + return tokens; + } + for (j = 0; j < (len-(seplen-1)); j++) { + /* make sure there is room for the next element and the final one */ + if (slots < elements+2) { + sds *newtokens; + + slots *= 2; + newtokens = s_realloc(tokens,sizeof(sds)*slots); + if (newtokens == NULL) goto cleanup; + tokens = newtokens; + } + /* search the separator */ + if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { + tokens[elements] = sdsnewlen(s+start,j-start); + if (tokens[elements] == NULL) goto cleanup; + elements++; + start = j+seplen; + j = j+seplen-1; /* skip the separator */ + } + } + /* Add the final element. We are sure there is room in the tokens array. */ + tokens[elements] = sdsnewlen(s+start,len-start); + if (tokens[elements] == NULL) goto cleanup; + elements++; + *count = elements; + return tokens; + +cleanup: + { + int i; + for (i = 0; i < elements; i++) sdsfree(tokens[i]); + s_free(tokens); + *count = 0; + return NULL; + } +} + +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ +void sdsfreesplitres(sds *tokens, int count) { + if (!tokens) return; + while(count--) + sdsfree(tokens[count]); + s_free(tokens); +} + +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatrepr(sds s, const char *p, size_t len) { + s = sdscatlen(s,"\"",1); + while(len--) { + switch(*p) { + case '\\': + case '"': + s = sdscatprintf(s,"\\%c",*p); + break; + case '\n': s = sdscatlen(s,"\\n",2); break; + case '\r': s = sdscatlen(s,"\\r",2); break; + case '\t': s = sdscatlen(s,"\\t",2); break; + case '\a': s = sdscatlen(s,"\\a",2); break; + case '\b': s = sdscatlen(s,"\\b",2); break; + default: + if (isprint(*p)) + s = sdscatprintf(s,"%c",*p); + else + s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); + break; + } + p++; + } + return sdscatlen(s,"\"",1); +} + +/* Helper function for sdssplitargs() that returns non zero if 'c' + * is a valid hex digit. */ +int is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +/* Helper function for sdssplitargs() that converts a hex digit into an + * integer from 0 to 15 */ +int hex_digit_to_int(char c) { + switch(c) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 10; + case 'b': case 'B': return 11; + case 'c': case 'C': return 12; + case 'd': case 'D': return 13; + case 'e': case 'E': return 14; + case 'f': case 'F': return 15; + default: return 0; + } +} + +/* Split a line into arguments, where every argument can be in the + * following programming-language REPL-alike form: + * + * foo bar "newline are supported\n" and "\xff\x00otherstuff" + * + * The number of arguments is stored into *argc, and an array + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). + * + * Note that sdscatrepr() is able to convert back a string into + * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' + */ +sds *sdssplitargs(const char *line, int *argc) { + const char *p = line; + char *current = NULL; + char **vector = NULL; + + *argc = 0; + while(1) { + /* skip blanks */ + while(*p && isspace(*p)) p++; + if (*p) { + /* get a token */ + int inq=0; /* set to 1 if we are in "quotes" */ + int insq=0; /* set to 1 if we are in 'single quotes' */ + int done=0; + + if (current == NULL) current = sdsempty(); + while(!done) { + if (inq) { + if (*p == '\\' && *(p+1) == 'x' && + is_hex_digit(*(p+2)) && + is_hex_digit(*(p+3))) + { + unsigned char byte; + + byte = (hex_digit_to_int(*(p+2))*16)+ + hex_digit_to_int(*(p+3)); + current = sdscatlen(current,(char*)&byte,1); + p += 3; + } else if (*p == '\\' && *(p+1)) { + char c; + + p++; + switch(*p) { + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'b': c = '\b'; break; + case 'a': c = '\a'; break; + default: c = *p; break; + } + current = sdscatlen(current,&c,1); + } else if (*p == '"') { + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else if (insq) { + if (*p == '\\' && *(p+1) == '\'') { + p++; + current = sdscatlen(current,"'",1); + } else if (*p == '\'') { + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else { + switch(*p) { + case ' ': + case '\n': + case '\r': + case '\t': + case '\0': + done=1; + break; + case '"': + inq=1; + break; + case '\'': + insq=1; + break; + default: + current = sdscatlen(current,p,1); + break; + } + } + if (*p) p++; + } + /* add the token to the vector */ + vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); + vector[*argc] = current; + (*argc)++; + current = NULL; + } else { + /* Even on empty input string return something not NULL. */ + if (vector == NULL) vector = s_malloc(sizeof(void*)); + return vector; + } + } + +err: + while((*argc)--) + sdsfree(vector[*argc]); + s_free(vector); + if (current) sdsfree(current); + *argc = 0; + return NULL; +} + +/* Modify the string substituting all the occurrences of the set of + * characters specified in the 'from' string to the corresponding character + * in the 'to' array. + * + * For instance: sdsmapchars(mystring, "ho", "01", 2) + * will have the effect of turning the string "hello" into "0ell1". + * + * The function returns the sds string pointer, that is always the same + * as the input pointer since no resize is needed. */ +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = sdslen(s); + + for (j = 0; j < l; j++) { + for (i = 0; i < setlen; i++) { + if (s[j] == from[i]) { + s[j] = to[i]; + break; + } + } + } + return s; +} + +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscat(join,sep); + } + return join; +} + +/* Like sdsjoin, but joins an array of SDS strings. */ +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscatsds(join, argv[j]); + if (j != argc-1) join = sdscatlen(join,sep,seplen); + } + return join; +} + +/* Wrappers to the allocators used by SDS. Note that SDS will actually + * just use the macros defined into sdsalloc.h in order to avoid to pay + * the overhead of function calls. Here we define these wrappers only for + * the programs SDS is linked to, if they want to touch the SDS internals + * even if they use a different allocator. */ +void *sds_malloc(size_t size) { return s_malloc(size); } +void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } +void sds_free(void *ptr) { s_free(ptr); } + +#if defined(SDS_TEST_MAIN) +#include +#include "testhelp.h" +#include "limits.h" + +#define UNUSED(x) (void)(x) +int sdsTest(void) { + { + sds x = sdsnew("foo"), y; + + test_cond("Create a string and obtain the length", + sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) + + sdsfree(x); + x = sdsnewlen("foo",2); + test_cond("Create a string with specified length", + sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) + + x = sdscat(x,"bar"); + test_cond("Strings concatenation", + sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); + + x = sdscpy(x,"a"); + test_cond("sdscpy() against an originally longer string", + sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) + + x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); + test_cond("sdscpy() against an originally shorter string", + sdslen(x) == 33 && + memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) + + sdsfree(x); + x = sdscatprintf(sdsempty(),"%d",123); + test_cond("sdscatprintf() seems working in the base case", + sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) + + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("sdscatfmt() seems working in the base case", + sdslen(x) == 60 && + memcmp(x,"--Hello Hi! World -9223372036854775808," + "9223372036854775807--",60) == 0) + printf("[%s]\n",x); + + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("sdscatfmt() seems working with unsigned numbers", + sdslen(x) == 35 && + memcmp(x,"--4294967295,18446744073709551615--",35) == 0) + + sdsfree(x); + x = sdsnew(" x "); + sdstrim(x," x"); + test_cond("sdstrim() works when all chars match", + sdslen(x) == 0) + + sdsfree(x); + x = sdsnew(" x "); + sdstrim(x," "); + test_cond("sdstrim() works when a single char remains", + sdslen(x) == 1 && x[0] == 'x') + + sdsfree(x); + x = sdsnew("xxciaoyyy"); + sdstrim(x,"xy"); + test_cond("sdstrim() correctly trims characters", + sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) + + y = sdsdup(x); + sdsrange(y,1,1); + test_cond("sdsrange(...,1,1)", + sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,1,-1); + test_cond("sdsrange(...,1,-1)", + sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,-2,-1); + test_cond("sdsrange(...,-2,-1)", + sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,2,1); + test_cond("sdsrange(...,2,1)", + sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,1,100); + test_cond("sdsrange(...,1,100)", + sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,100,100); + test_cond("sdsrange(...,100,100)", + sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("foo"); + y = sdsnew("foa"); + test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("bar"); + y = sdsnew("bar"); + test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) + + sdsfree(y); + sdsfree(x); + x = sdsnew("aar"); + y = sdsnew("bar"); + test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + sdsfree(y); + sdsfree(x); + x = sdsnewlen("\a\n\0foo\r",7); + y = sdscatrepr(sdsempty(),x,sdslen(x)); + test_cond("sdscatrepr(...data...)", + memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) + + { + unsigned int oldfree; + char *p; + int step = 10, j, i; + + sdsfree(x); + sdsfree(y); + x = sdsnew("0"); + test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0); + + /* Run the test a few times in order to hit the first two + * SDS header types. */ + for (i = 0; i < 10; i++) { + int oldlen = sdslen(x); + x = sdsMakeRoomFor(x,step); + int type = x[-1]&SDS_TYPE_MASK; + + test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen); + if (type != SDS_TYPE_5) { + test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step); + oldfree = sdsavail(x); + } + p = x+oldlen; + for (j = 0; j < step; j++) { + p[j] = 'A'+j; + } + sdsIncrLen(x,step); + } + test_cond("sdsMakeRoomFor() content", + memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0); + test_cond("sdsMakeRoomFor() final length",sdslen(x)==101); + + sdsfree(x); + } + } + test_report() + return 0; +} +#endif + +#ifdef SDS_TEST_MAIN +int main(void) { + return sdsTest(); +} +#endif diff --git a/csrc/lib/rmutil/sds.h b/csrc/lib/rmutil/sds.h new file mode 100644 index 0000000..394f8b5 --- /dev/null +++ b/csrc/lib/rmutil/sds.h @@ -0,0 +1,273 @@ +/* SDSLib 2.0 -- A C dynamic strings library + * + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Oran Agra + * Copyright (c) 2015, Redis Labs, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SDS_H +#define __SDS_H + +#define SDS_MAX_PREALLOC (1024*1024) + +#include +#include +#include + +typedef char *sds; + +/* Note: sdshdr5 is never used, we just access the flags byte directly. + * However is here to document the layout of type 5 SDS strings. */ +struct __attribute__ ((__packed__)) sdshdr5 { + unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr8 { + uint8_t len; /* used */ + uint8_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr16 { + uint16_t len; /* used */ + uint16_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr32 { + uint32_t len; /* used */ + uint32_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; +struct __attribute__ ((__packed__)) sdshdr64 { + uint64_t len; /* used */ + uint64_t alloc; /* excluding the header and null terminator */ + unsigned char flags; /* 3 lsb of type, 5 unused bits */ + char buf[]; +}; + +#define SDS_TYPE_5 0 +#define SDS_TYPE_8 1 +#define SDS_TYPE_16 2 +#define SDS_TYPE_32 3 +#define SDS_TYPE_64 4 +#define SDS_TYPE_MASK 7 +#define SDS_TYPE_BITS 3 +#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T))); +#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) +#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) + +static inline size_t sdslen(const sds s) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return SDS_TYPE_5_LEN(flags); + case SDS_TYPE_8: + return SDS_HDR(8,s)->len; + case SDS_TYPE_16: + return SDS_HDR(16,s)->len; + case SDS_TYPE_32: + return SDS_HDR(32,s)->len; + case SDS_TYPE_64: + return SDS_HDR(64,s)->len; + } + return 0; +} + +static inline size_t sdsavail(const sds s) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: { + return 0; + } + case SDS_TYPE_8: { + SDS_HDR_VAR(8,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_16: { + SDS_HDR_VAR(16,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_32: { + SDS_HDR_VAR(32,s); + return sh->alloc - sh->len; + } + case SDS_TYPE_64: { + SDS_HDR_VAR(64,s); + return sh->alloc - sh->len; + } + } + return 0; +} + +static inline void sdssetlen(sds s, size_t newlen) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + { + unsigned char *fp = ((unsigned char*)s)-1; + *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + } + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->len = newlen; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->len = newlen; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->len = newlen; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->len = newlen; + break; + } +} + +static inline void sdsinclen(sds s, size_t inc) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + { + unsigned char *fp = ((unsigned char*)s)-1; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + } + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->len += inc; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->len += inc; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->len += inc; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->len += inc; + break; + } +} + +/* sdsalloc() = sdsavail() + sdslen() */ +static inline size_t sdsalloc(const sds s) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + return SDS_TYPE_5_LEN(flags); + case SDS_TYPE_8: + return SDS_HDR(8,s)->alloc; + case SDS_TYPE_16: + return SDS_HDR(16,s)->alloc; + case SDS_TYPE_32: + return SDS_HDR(32,s)->alloc; + case SDS_TYPE_64: + return SDS_HDR(64,s)->alloc; + } + return 0; +} + +static inline void sdssetalloc(sds s, size_t newlen) { + unsigned char flags = s[-1]; + switch(flags&SDS_TYPE_MASK) { + case SDS_TYPE_5: + /* Nothing to do, this type has no total allocation info. */ + break; + case SDS_TYPE_8: + SDS_HDR(8,s)->alloc = newlen; + break; + case SDS_TYPE_16: + SDS_HDR(16,s)->alloc = newlen; + break; + case SDS_TYPE_32: + SDS_HDR(32,s)->alloc = newlen; + break; + case SDS_TYPE_64: + SDS_HDR(64,s)->alloc = newlen; + break; + } +} + +sds sdsnewlen(const void *init, size_t initlen); +sds sdsnew(const char *init); +sds sdsempty(void); +sds sdsdup(const sds s); +void sdsfree(sds s); +sds sdsgrowzero(sds s, size_t len); +sds sdscatlen(sds s, const void *t, size_t len); +sds sdscat(sds s, const char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); + +sds sdscatvprintf(sds s, const char *fmt, va_list ap); +#ifdef __GNUC__ +sds sdscatprintf(sds s, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); +#else +sds sdscatprintf(sds s, const char *fmt, ...); +#endif + +sds sdscatfmt(sds s, char const *fmt, ...); +sds sdstrim(sds s, const char *cset); +void sdsrange(sds s, int start, int end); +void sdsupdatelen(sds s); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void sdsfreesplitres(sds *tokens, int count); +void sdstolower(sds s); +void sdstoupper(sds s); +sds sdsfromlonglong(long long value); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep); +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); +void *sdsAllocPtr(sds s); + +/* Export the allocator used by SDS to the program using SDS. + * Sometimes the program SDS is linked to, may use a different set of + * allocators, but may want to allocate or free things that SDS will + * respectively free or allocate. */ +void *sds_malloc(size_t size); +void *sds_realloc(void *ptr, size_t size); +void sds_free(void *ptr); + +#ifdef REDIS_TEST +int sdsTest(int argc, char *argv[]); +#endif + +#endif diff --git a/csrc/lib/rmutil/sdsalloc.h b/csrc/lib/rmutil/sdsalloc.h new file mode 100644 index 0000000..1538fdf --- /dev/null +++ b/csrc/lib/rmutil/sdsalloc.h @@ -0,0 +1,47 @@ +/* SDSLib 2.0 -- A C dynamic strings library + * + * Copyright (c) 2006-2015, Salvatore Sanfilippo + * Copyright (c) 2015, Redis Labs, Inc + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* SDS allocator selection. + * + * This file is used in order to change the SDS allocator at compile time. + * Just define the following defines to what you want to use. Also add + * the include of your alternate allocator if needed (not needed in order + * to use the default libc allocator). */ + +#if defined(__MACH__) +#include +#else +#include +#endif +//#include "zmalloc.h" +#define s_malloc malloc +#define s_realloc realloc +#define s_free free diff --git a/csrc/lib/rmutil/strings.c b/csrc/lib/rmutil/strings.c new file mode 100644 index 0000000..3933013 --- /dev/null +++ b/csrc/lib/rmutil/strings.c @@ -0,0 +1,81 @@ +#include +#include +#include +#include "strings.h" +#include "alloc.h" + +#include "sds.h" + +// RedisModuleString *RMUtil_CreateFormattedString(RedisModuleCtx *ctx, const char *fmt, ...) { +// sds s = sdsempty(); + +// va_list ap; +// va_start(ap, fmt); +// s = sdscatvprintf(s, fmt, ap); +// va_end(ap); + +// RedisModuleString *ret = RedisModule_CreateString(ctx, (const char *)s, sdslen(s)); +// sdsfree(s); +// return ret; +// } + +int RMUtil_StringEquals(RedisModuleString *s1, RedisModuleString *s2) { + + const char *c1, *c2; + size_t l1, l2; + c1 = RedisModule_StringPtrLen(s1, &l1); + c2 = RedisModule_StringPtrLen(s2, &l2); + if (l1 != l2) return 0; + + return strncmp(c1, c2, l1) == 0; +} + +int RMUtil_StringEqualsC(RedisModuleString *s1, const char *s2) { + + const char *c1; + size_t l1, l2 = strlen(s2); + c1 = RedisModule_StringPtrLen(s1, &l1); + if (l1 != l2) return 0; + + return strncmp(c1, s2, l1) == 0; +} +int RMUtil_StringEqualsCaseC(RedisModuleString *s1, const char *s2) { + + const char *c1; + size_t l1, l2 = strlen(s2); + c1 = RedisModule_StringPtrLen(s1, &l1); + if (l1 != l2) return 0; + + return strncasecmp(c1, s2, l1) == 0; +} + +void RMUtil_StringToLower(RedisModuleString *s) { + + size_t l; + char *c = (char *)RedisModule_StringPtrLen(s, &l); + size_t i; + for (i = 0; i < l; i++) { + *c = tolower(*c); + ++c; + } +} + +void RMUtil_StringToUpper(RedisModuleString *s) { + size_t l; + char *c = (char *)RedisModule_StringPtrLen(s, &l); + size_t i; + for (i = 0; i < l; i++) { + *c = toupper(*c); + ++c; + } +} + +void RMUtil_StringConvert(RedisModuleString **rs, const char **ss, size_t n, int options) { + for (size_t ii = 0; ii < n; ++ii) { + const char *p = RedisModule_StringPtrLen(rs[ii], NULL); + if (options & RMUTIL_STRINGCONVERT_COPY) { + p = strdup(p); + } + ss[ii] = p; + } +} \ No newline at end of file diff --git a/csrc/lib/rmutil/strings.h b/csrc/lib/rmutil/strings.h new file mode 100644 index 0000000..eaef71e --- /dev/null +++ b/csrc/lib/rmutil/strings.h @@ -0,0 +1,38 @@ +#ifndef __RMUTIL_STRINGS_H__ +#define __RMUTIL_STRINGS_H__ + +#include + +/* +* Create a new RedisModuleString object from a printf-style format and arguments. +* Note that RedisModuleString objects CANNOT be used as formatting arguments. +*/ +// DEPRECATED since it was added to the RedisModule API. Replaced with a macro below +// RedisModuleString *RMUtil_CreateFormattedString(RedisModuleCtx *ctx, const char *fmt, ...); +#define RMUtil_CreateFormattedString RedisModule_CreateStringPrintf + +/* Return 1 if the two strings are equal. Case *sensitive* */ +int RMUtil_StringEquals(RedisModuleString *s1, RedisModuleString *s2); + +/* Return 1 if the string is equal to a C NULL terminated string. Case *sensitive* */ +int RMUtil_StringEqualsC(RedisModuleString *s1, const char *s2); + +/* Return 1 if the string is equal to a C NULL terminated string. Case *insensitive* */ +int RMUtil_StringEqualsCaseC(RedisModuleString *s1, const char *s2); + +/* Converts a redis string to lowercase in place without reallocating anything */ +void RMUtil_StringToLower(RedisModuleString *s); + +/* Converts a redis string to uppercase in place without reallocating anything */ +void RMUtil_StringToUpper(RedisModuleString *s); + +// If set, copy the strings using strdup rather than simply storing pointers. +#define RMUTIL_STRINGCONVERT_COPY 1 + +/** + * Convert one or more RedisModuleString objects into `const char*`. + * Both rs and ss are arrays, and should be of length. + * Options may be 0 or `RMUTIL_STRINGCONVERT_COPY` + */ +void RMUtil_StringConvert(RedisModuleString **rs, const char **ss, size_t n, int options); +#endif diff --git a/csrc/lib/rmutil/test.h b/csrc/lib/rmutil/test.h new file mode 100644 index 0000000..a15864a --- /dev/null +++ b/csrc/lib/rmutil/test.h @@ -0,0 +1,69 @@ +#ifndef __TESTUTIL_H__ +#define __TESTUTIL_H__ + +#include +#include +#include + +static int numTests = 0; +static int numAsserts = 0; + +#define TESTFUNC(f) \ + printf(" Testing %s\t\t", __STRING(f)); \ + numTests++; \ + fflush(stdout); \ + if (f()) { \ + printf(" %s FAILED!\n", __STRING(f)); \ + exit(1); \ + } else \ + printf("[PASS]\n"); + +#define ASSERTM(expr, ...) \ + if (!(expr)) { \ + fprintf(stderr, "%s:%d: Assertion '%s' Failed: " __VA_ARGS__ "\n", __FILE__, __LINE__, \ + __STRING(expr)); \ + return -1; \ + } \ + numAsserts++; + +#define ASSERT(expr) \ + if (!(expr)) { \ + fprintf(stderr, "%s:%d Assertion '%s' Failed\n", __FILE__, __LINE__, __STRING(expr)); \ + return -1; \ + } \ + numAsserts++; + +#define ASSERT_STRING_EQ(s1, s2) ASSERT(!strcmp(s1, s2)); + +#define ASSERT_EQUAL(x, y, ...) \ + if (x != y) { \ + fprintf(stderr, "%s:%d: ", __FILE__, __LINE__); \ + fprintf(stderr, "%g != %g: " __VA_ARGS__ "\n", (double)x, (double)y); \ + return -1; \ + } \ + numAsserts++; + +#define FAIL(fmt, ...) \ + { \ + fprintf(stderr, "%s:%d: FAIL: " fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ + return -1; \ + } + +#define RETURN_TEST_SUCCESS return 0; +#define TEST_CASE(x, block) \ + int x { \ + block; \ + return 0 \ + } + +#define PRINT_TEST_SUMMARY printf("\nTotal: %d tests and %d assertions OK\n", numTests, numAsserts); + +#define TEST_MAIN(body) \ + int main(int argc, char **argv) { \ + printf("Starting Test '%s'...\n", argv[0]); \ + body; \ + PRINT_TEST_SUMMARY; \ + printf("\n--------------------\n\n"); \ + return 0; \ + } +#endif \ No newline at end of file diff --git a/csrc/lib/rmutil/test_heap.c b/csrc/lib/rmutil/test_heap.c new file mode 100644 index 0000000..820c299 --- /dev/null +++ b/csrc/lib/rmutil/test_heap.c @@ -0,0 +1,38 @@ +#include +#include "heap.h" +#include "assert.h" + +int cmp(void *a, void *b) { + int *__a = (int *) a; + int *__b = (int *) b; + return *__a - *__b; +} + +int main(int argc, char **argv) { + int myints[] = {10, 20, 30, 5, 15}; + Vector *v = NewVector(int, 5); + for (int i = 0; i < 5; i++) { + Vector_Push(v, myints[i]); + } + + Make_Heap(v, 0, v->top, cmp); + + int n; + Vector_Get(v, 0, &n); + assert(30 == n); + + Heap_Pop(v, 0, v->top, cmp); + v->top = 4; + Vector_Get(v, 0, &n); + assert(20 == n); + + Vector_Push(v, 99); + Heap_Push(v, 0, v->top, cmp); + Vector_Get(v, 0, &n); + assert(99 == n); + + Vector_Free(v); + printf("PASS!\n"); + return 0; +} + diff --git a/csrc/lib/rmutil/test_periodic.c b/csrc/lib/rmutil/test_periodic.c new file mode 100644 index 0000000..a27d7a4 --- /dev/null +++ b/csrc/lib/rmutil/test_periodic.c @@ -0,0 +1,26 @@ +#include +#include +#include +#include "periodic.h" +#include "assert.h" +#include "test.h" + +void timerCb(RedisModuleCtx *ctx, void *p) { + int *x = p; + (*x)++; +} + +int testPeriodic() { + int x = 0; + struct RMUtilTimer *tm = RMUtil_NewPeriodicTimer( + timerCb, NULL, &x, (struct timespec){.tv_sec = 0, .tv_nsec = 10000000}); + + sleep(1); + + ASSERT_EQUAL(0, RMUtilTimer_Terminate(tm)); + ASSERT(x > 0); + ASSERT(x <= 100); + return 0; +} + +TEST_MAIN({ TESTFUNC(testPeriodic); }); diff --git a/csrc/lib/rmutil/test_priority_queue.c b/csrc/lib/rmutil/test_priority_queue.c new file mode 100644 index 0000000..4a21c18 --- /dev/null +++ b/csrc/lib/rmutil/test_priority_queue.c @@ -0,0 +1,37 @@ +#include +#include "assert.h" +#include "priority_queue.h" + +int cmp(void* i1, void* i2) { + int *__i1 = (int*) i1; + int *__i2 = (int*) i2; + return *__i1 - *__i2; +} + +int main(int argc, char **argv) { + PriorityQueue *pq = NewPriorityQueue(int, 10, cmp); + assert(0 == Priority_Queue_Size(pq)); + + for (int i = 0; i < 5; i++) { + Priority_Queue_Push(pq, i); + } + assert(5 == Priority_Queue_Size(pq)); + + Priority_Queue_Pop(pq); + assert(4 == Priority_Queue_Size(pq)); + + Priority_Queue_Push(pq, 10); + Priority_Queue_Push(pq, 20); + Priority_Queue_Push(pq, 15); + int n; + Priority_Queue_Top(pq, &n); + assert(20 == n); + + Priority_Queue_Pop(pq); + Priority_Queue_Top(pq, &n); + assert(15 == n); + + Priority_Queue_Free(pq); + printf("PASS!\n"); + return 0; +} diff --git a/csrc/lib/rmutil/test_util.h b/csrc/lib/rmutil/test_util.h new file mode 100644 index 0000000..39f1575 --- /dev/null +++ b/csrc/lib/rmutil/test_util.h @@ -0,0 +1,70 @@ +#ifndef __TEST_UTIL_H__ +#define __TEST_UTIL_H__ + +#include "util.h" +#include +#include +#include + + + +#define RMUtil_Test(f) \ + if (argc < 2 || RMUtil_ArgExists(__STRING(f), argv, argc, 1)) { \ + int rc = f(ctx); \ + if (rc != REDISMODULE_OK) { \ + char * err = ""; \ + sprintf(err, "Test %s failed ", __STRING(f));\ + RedisModule_ReplyWithError(ctx, err); \ + return REDISMODULE_ERR;\ + }\ + } + + +#define RMUtil_Assert(expr) if (!(expr)) { fprintf (stderr, "Assertion '%s' Failed\n", __STRING(expr)); return REDISMODULE_ERR; } + +#define RMUtil_AssertReplyEquals(rep, cstr) RMUtil_Assert( \ + RMUtil_StringEquals(RedisModule_CreateStringFromCallReply(rep), RedisModule_CreateString(ctx, cstr, strlen(cstr))) \ + ) +# + +/** +* Create an arg list to pass to a redis command handler manually, based on the format in fmt. +* The accepted format specifiers are: +* c - for null terminated c strings +* s - for RedisModuleString* objects +* l - for longs +* +* Example: RMUtil_MakeArgs(ctx, &argc, "clc", "hello", 1337, "world"); +* +* Returns an array of RedisModuleString pointers. The size of the array is store in argcp +*/ +RedisModuleString **RMUtil_MakeArgs(RedisModuleCtx *ctx, int *argcp, const char *fmt, ...) { + + va_list ap; + va_start(ap, fmt); + RedisModuleString **argv = calloc(strlen(fmt), sizeof(RedisModuleString*)); + int argc = 0; + const char *p = fmt; + while(*p) { + if (*p == 'c') { + char *cstr = va_arg(ap,char*); + argv[argc++] = RedisModule_CreateString(ctx, cstr, strlen(cstr)); + } else if (*p == 's') { + argv[argc++] = va_arg(ap,void*);; + } else if (*p == 'l') { + long ll = va_arg(ap,long long); + argv[argc++] = RedisModule_CreateStringFromLongLong(ctx, ll); + } else { + goto fmterr; + } + p++; + } + *argcp = argc; + + return argv; +fmterr: + free(argv); + return NULL; +} + +#endif \ No newline at end of file diff --git a/csrc/lib/rmutil/test_vector.c b/csrc/lib/rmutil/test_vector.c new file mode 100644 index 0000000..c5737b2 --- /dev/null +++ b/csrc/lib/rmutil/test_vector.c @@ -0,0 +1,58 @@ +#include "vector.h" +#include +#include "test.h" + +int testVector() { + + Vector *v = NewVector(int, 1); + ASSERT(v != NULL); + // Vector_Put(v, 0, 1); + // Vector_Put(v, 1, 3); + for (int i = 0; i < 10; i++) { + Vector_Push(v, i); + } + ASSERT_EQUAL(10, Vector_Size(v)); + ASSERT_EQUAL(16, Vector_Cap(v)); + + for (int i = 0; i < Vector_Size(v); i++) { + int n; + int rc = Vector_Get(v, i, &n); + ASSERT_EQUAL(1, rc); + // printf("%d %d\n", rc, n); + + ASSERT_EQUAL(n, i); + } + + Vector_Free(v); + + v = NewVector(char *, 0); + int N = 4; + char *strings[4] = {"hello", "world", "foo", "bar"}; + + for (int i = 0; i < N; i++) { + Vector_Push(v, strings[i]); + } + ASSERT_EQUAL(N, Vector_Size(v)); + ASSERT(Vector_Cap(v) >= N); + + for (int i = 0; i < Vector_Size(v); i++) { + char *x; + int rc = Vector_Get(v, i, &x); + ASSERT_EQUAL(1, rc); + ASSERT_STRING_EQ(x, strings[i]); + } + + int rc = Vector_Get(v, 100, NULL); + ASSERT_EQUAL(0, rc); + + Vector_Free(v); + + return 0; + // Vector_Push(v, "hello"); + // Vector_Push(v, "world"); + // char *x = NULL; + // int rc = Vector_Getx(v, 0, &x); + // printf("rc: %d got %s\n", rc, x); +} + +TEST_MAIN({ TESTFUNC(testVector); }); diff --git a/csrc/lib/rmutil/util.c b/csrc/lib/rmutil/util.c new file mode 100644 index 0000000..886c8b5 --- /dev/null +++ b/csrc/lib/rmutil/util.c @@ -0,0 +1,299 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#define REDISMODULE_EXPERIMENTAL_API +#include +#include "util.h" + +/** +Check if an argument exists in an argument list (argv,argc), starting at offset. +@return 0 if it doesn't exist, otherwise the offset it exists in +*/ +int RMUtil_ArgExists(const char *arg, RedisModuleString **argv, int argc, int offset) { + + size_t larg = strlen(arg); + for (; offset < argc; offset++) { + size_t l; + const char *carg = RedisModule_StringPtrLen(argv[offset], &l); + if (l != larg) continue; + if (carg != NULL && strncasecmp(carg, arg, larg) == 0) { + return offset; + } + } + return 0; +} + +/** +Check if an argument exists in an argument list (argv,argc) +@return -1 if it doesn't exist, otherwise the offset it exists in +*/ +int RMUtil_ArgIndex(const char *arg, RedisModuleString **argv, int argc) { + + size_t larg = strlen(arg); + for (int offset = 0; offset < argc; offset++) { + size_t l; + const char *carg = RedisModule_StringPtrLen(argv[offset], &l); + if (l != larg) continue; + if (carg != NULL && strncasecmp(carg, arg, larg) == 0) { + return offset; + } + } + return -1; +} + +RMUtilInfo *RMUtil_GetRedisInfo(RedisModuleCtx *ctx) { + + RedisModuleCallReply *r = RedisModule_Call(ctx, "INFO", "c", "all"); + if (r == NULL || RedisModule_CallReplyType(r) == REDISMODULE_REPLY_ERROR) { + return NULL; + } + + int cap = 100; // rough estimate of info lines + RMUtilInfo *info = malloc(sizeof(RMUtilInfo)); + info->entries = calloc(cap, sizeof(RMUtilInfoEntry)); + + int i = 0; + size_t sz; + char *text = (char *)RedisModule_CallReplyStringPtr(r, &sz); + + char *line = text; + while (line && line < text + sz) { + char *line = strsep(&text, "\r\n"); + if (line == NULL) break; + + if (!(*line >= 'a' && *line <= 'z')) { // skip non entry lines + continue; + } + + char *key = strsep(&line, ":"); + info->entries[i].key = strdup(key); + info->entries[i].val = strdup(line); + i++; + if (i >= cap) { + cap *= 2; + info->entries = realloc(info->entries, cap * sizeof(RMUtilInfoEntry)); + } + } + info->numEntries = i; + RedisModule_FreeCallReply(r); + return info; +} +void RMUtilRedisInfo_Free(RMUtilInfo *info) { + for (int i = 0; i < info->numEntries; i++) { + free(info->entries[i].key); + free(info->entries[i].val); + } + free(info->entries); + free(info); +} + +int RMUtilInfo_GetInt(RMUtilInfo *info, const char *key, long long *val) { + + const char *p = NULL; + if (!RMUtilInfo_GetString(info, key, &p)) { + return 0; + } + + *val = strtoll(p, NULL, 10); + if ((errno == ERANGE && (*val == LONG_MAX || *val == LONG_MIN)) || (errno != 0 && *val == 0)) { + *val = -1; + return 0; + } + + return 1; +} + +int RMUtilInfo_GetString(RMUtilInfo *info, const char *key, const char **str) { + int i; + for (i = 0; i < info->numEntries; i++) { + if (!strcmp(key, info->entries[i].key)) { + *str = info->entries[i].val; + return 1; + } + } + return 0; +} + +int RMUtilInfo_GetDouble(RMUtilInfo *info, const char *key, double *d) { + const char *p = NULL; + if (!RMUtilInfo_GetString(info, key, &p)) { + printf("not found %s\n", key); + return 0; + } + + *d = strtod(p, NULL); + if ((errno == ERANGE && (*d == HUGE_VAL || *d == -HUGE_VAL)) || (errno != 0 && *d == 0)) { + return 0; + } + + return 1; +} + +/* +c -- pointer to a Null terminated C string pointer. +b -- pointer to a C buffer, followed by pointer to a size_t for its length +s -- pointer to a RedisModuleString +l -- pointer to Long long integer. +d -- pointer to a Double +* -- do not parse this argument at all +*/ +int RMUtil_ParseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int rc = rmutil_vparseArgs(argv, argc, offset, fmt, ap); + va_end(ap); + return rc; +} + +// Internal function that parses arguments based on the format described above +int rmutil_vparseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, va_list ap) { + + int i = offset; + char *c = (char *)fmt; + while (*c && i < argc) { + + // read c string + if (*c == 'c') { + char **p = va_arg(ap, char **); + *p = (char *)RedisModule_StringPtrLen(argv[i], NULL); + } else if (*c == 'b') { + char **p = va_arg(ap, char **); + size_t *len = va_arg(ap, size_t *); + *p = (char *)RedisModule_StringPtrLen(argv[i], len); + } else if (*c == 's') { // read redis string + + RedisModuleString **s = va_arg(ap, void *); + *s = argv[i]; + + } else if (*c == 'l') { // read long + long long *l = va_arg(ap, long long *); + + if (RedisModule_StringToLongLong(argv[i], l) != REDISMODULE_OK) { + return REDISMODULE_ERR; + } + } else if (*c == 'd') { // read double + double *d = va_arg(ap, double *); + if (RedisModule_StringToDouble(argv[i], d) != REDISMODULE_OK) { + return REDISMODULE_ERR; + } + } else if (*c == '*') { // skip current arg + // do nothing + } else { + return REDISMODULE_ERR; // WAT? + } + c++; + i++; + } + // if the format is longer than argc, retun an error + if (*c != 0) { + return REDISMODULE_ERR; + } + return REDISMODULE_OK; +} + +int RMUtil_ParseArgsAfter(const char *token, RedisModuleString **argv, int argc, const char *fmt, + ...) { + + int pos = RMUtil_ArgIndex(token, argv, argc); + if (pos < 0) { + return REDISMODULE_ERR; + } + + va_list ap; + va_start(ap, fmt); + int rc = rmutil_vparseArgs(argv, argc, pos + 1, fmt, ap); + va_end(ap); + return rc; +} + +RedisModuleCallReply *RedisModule_CallReplyArrayElementByPath(RedisModuleCallReply *rep, + const char *path) { + if (rep == NULL) return NULL; + + RedisModuleCallReply *ele = rep; + const char *s = path; + char *e; + long idx; + do { + errno = 0; + idx = strtol(s, &e, 10); + + if ((errno == ERANGE && (idx == LONG_MAX || idx == LONG_MIN)) || (errno != 0 && idx == 0) || + (REDISMODULE_REPLY_ARRAY != RedisModule_CallReplyType(ele)) || (s == e)) { + ele = NULL; + break; + } + s = e; + ele = RedisModule_CallReplyArrayElement(ele, idx - 1); + + } while ((ele != NULL) && (*e != '\0')); + + return ele; +} + +int RedisModule_TryGetValue(RedisModuleKey *key, const RedisModuleType *type, void **out) { + if (key == NULL) { + return RMUTIL_VALUE_MISSING; + } + int keytype = RedisModule_KeyType(key); + if (keytype == REDISMODULE_KEYTYPE_EMPTY) { + return RMUTIL_VALUE_EMPTY; + } else if (keytype == REDISMODULE_KEYTYPE_MODULE && RedisModule_ModuleTypeGetType(key) == type) { + *out = RedisModule_ModuleTypeGetValue(key); + return RMUTIL_VALUE_OK; + } else { + return RMUTIL_VALUE_MISMATCH; + } +} + +RedisModuleString **RMUtil_ParseVarArgs(RedisModuleString **argv, int argc, int offset, + const char *keyword, size_t *nargs) { + if (offset > argc) { + return NULL; + } + + argv += offset; + argc -= offset; + + int ix = RMUtil_ArgIndex(keyword, argv, argc); + if (ix < 0) { + return NULL; + } else if (ix >= argc - 1) { + *nargs = RMUTIL_VARARGS_BADARG; + return argv; + } + + argv += (ix + 1); + argc -= (ix + 1); + + long long n = 0; + RMUtil_ParseArgs(argv, argc, 0, "l", &n); + if (n > argc - 1 || n < 0) { + *nargs = RMUTIL_VARARGS_BADARG; + return argv; + } + + *nargs = n; + return argv + 1; +} + +void RMUtil_DefaultAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value) { + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL); + RedisModuleCallReply *rep = RedisModule_Call(ctx, "DUMP", "s", key); + if (rep != NULL && RedisModule_CallReplyType(rep) == REDISMODULE_REPLY_STRING) { + size_t n; + const char *s = RedisModule_CallReplyStringPtr(rep, &n); + RedisModule_EmitAOF(aof, "RESTORE", "slb", key, 0, s, n); + } else { + RedisModule_Log(RedisModule_GetContextFromIO(aof), "warning", "Failed to emit AOF"); + } + if (rep != NULL) { + RedisModule_FreeCallReply(rep); + } + RedisModule_FreeThreadSafeContext(ctx); +} \ No newline at end of file diff --git a/csrc/lib/rmutil/util.h b/csrc/lib/rmutil/util.h new file mode 100644 index 0000000..cc75de6 --- /dev/null +++ b/csrc/lib/rmutil/util.h @@ -0,0 +1,149 @@ +#ifndef __UTIL_H__ +#define __UTIL_H__ + +#include +#include + +/// make sure the response is not NULL or an error, and if it is sends the error to the client and +/// exit the current function +#define RMUTIL_ASSERT_NOERROR(ctx, r) \ + if (r == NULL) { \ + return RedisModule_ReplyWithError(ctx, "ERR reply is NULL"); \ + } else if (RedisModule_CallReplyType(r) == REDISMODULE_REPLY_ERROR) { \ + RedisModule_ReplyWithCallReply(ctx, r); \ + return REDISMODULE_ERR; \ + } + +#define __rmutil_register_cmd(ctx, cmd, f, mode) \ + if (RedisModule_CreateCommand(ctx, cmd, f, mode, 1, 1, 1) == REDISMODULE_ERR) \ + return REDISMODULE_ERR; + +#define RMUtil_RegisterReadCmd(ctx, cmd, f) __rmutil_register_cmd(ctx, cmd, f, "readonly") + +#define RMUtil_RegisterWriteCmd(ctx, cmd, f) __rmutil_register_cmd(ctx, cmd, f, "write") + +/* RedisModule utilities. */ + +/** DEPRECATED: Return the offset of an arg if it exists in the arg list, or 0 if it's not there */ +int RMUtil_ArgExists(const char *arg, RedisModuleString **argv, int argc, int offset); + +/* Same as argExists but returns -1 if not found. Use this, RMUtil_ArgExists is kept for backwards +compatibility. */ +int RMUtil_ArgIndex(const char *arg, RedisModuleString **argv, int argc); + +/** +Automatically conver the arg list to corresponding variable pointers according to a given format. +You pass it the command arg list and count, the starting offset, a parsing format, and pointers to +the variables. +The format is a string consisting of the following identifiers: + + c -- pointer to a Null terminated C string pointer. + s -- pointer to a RedisModuleString + l -- pointer to Long long integer. + d -- pointer to a Double + * -- do not parse this argument at all + +Example: If I want to parse args[1], args[2] as a long long and double, I do: + double d; + long long l; + RMUtil_ParseArgs(argv, argc, 1, "ld", &l, &d); +*/ +int RMUtil_ParseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, ...); + +/** +Same as RMUtil_ParseArgs, but only parses the arguments after `token`, if it was found. +This is useful for optional stuff like [LIMIT [offset] [limit]] +*/ +int RMUtil_ParseArgsAfter(const char *token, RedisModuleString **argv, int argc, const char *fmt, + ...); + +int rmutil_vparseArgs(RedisModuleString **argv, int argc, int offset, const char *fmt, va_list ap); + +#define RMUTIL_VARARGS_BADARG ((size_t)-1) +/** + * Parse arguments in the form of KEYWORD {len} {arg} .. {arg}_len. + * If keyword is present, returns the position within `argv` containing the arguments. + * Returns NULL if the keyword is not found. + * If a parse error has occurred, `nargs` is set to RMUTIL_VARARGS_BADARG, but + * the return value is not NULL. + */ +RedisModuleString **RMUtil_ParseVarArgs(RedisModuleString **argv, int argc, int offset, + const char *keyword, size_t *nargs); + +/** + * Default implementation of an AoF rewrite function that simply calls DUMP/RESTORE + * internally. To use this function, pass it as the .aof_rewrite value in + * RedisModuleTypeMethods + */ +void RMUtil_DefaultAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value); + +// A single key/value entry in a redis info map +typedef struct { + char *key; + char *val; +} RMUtilInfoEntry; + +// Representation of INFO command response, as a list of k/v pairs +typedef struct { + RMUtilInfoEntry *entries; + int numEntries; +} RMUtilInfo; + +/** +* Get redis INFO result and parse it as RMUtilInfo. +* Returns NULL if something goes wrong. +* The resulting object needs to be freed with RMUtilRedisInfo_Free +*/ +RMUtilInfo *RMUtil_GetRedisInfo(RedisModuleCtx *ctx); + +/** +* Free an RMUtilInfo object and its entries +*/ +void RMUtilRedisInfo_Free(RMUtilInfo *info); + +/** +* Get an integer value from an info object. Returns 1 if the value was found and +* is an integer, 0 otherwise. the value is placed in 'val' +*/ +int RMUtilInfo_GetInt(RMUtilInfo *info, const char *key, long long *val); + +/** +* Get a string value from an info object. The value is placed in str. +* Returns 1 if the key was found, 0 if not +*/ +int RMUtilInfo_GetString(RMUtilInfo *info, const char *key, const char **str); + +/** +* Get a double value from an info object. Returns 1 if the value was found and is +* a correctly formatted double, 0 otherwise. the value is placed in 'd' +*/ +int RMUtilInfo_GetDouble(RMUtilInfo *info, const char *key, double *d); + +/* +* Returns a call reply array's element given by a space-delimited path. E.g., +* the path "1 2 3" will return the 3rd element from the 2 element of the 1st +* element from an array (or NULL if not found) +*/ +RedisModuleCallReply *RedisModule_CallReplyArrayElementByPath(RedisModuleCallReply *rep, + const char *path); + +/** + * Extract the module type from an opened key. + */ +typedef enum { + RMUTIL_VALUE_OK = 0, + RMUTIL_VALUE_MISSING, + RMUTIL_VALUE_EMPTY, + RMUTIL_VALUE_MISMATCH +} RMUtil_TryGetValueStatus; + +/** + * Tries to extract the module-specific type from the value. + * @param key an opened key (may be null) + * @param type the pointer to the type to match to + * @param[out] out if the value is present, will be set to it. + * @return a value in the @ref RMUtil_TryGetValueStatus enum. + */ +int RedisModule_TryGetValue(RedisModuleKey *key, const RedisModuleType *type, void **out); + +#endif diff --git a/csrc/lib/rmutil/vector.c b/csrc/lib/rmutil/vector.c new file mode 100644 index 0000000..25f7fdc --- /dev/null +++ b/csrc/lib/rmutil/vector.c @@ -0,0 +1,88 @@ +#include "vector.h" +#include + +inline int __vector_PushPtr(Vector *v, void *elem) { + if (v->top == v->cap) { + Vector_Resize(v, v->cap ? v->cap * 2 : 1); + } + + __vector_PutPtr(v, v->top, elem); + return v->top; +} + +inline int Vector_Get(Vector *v, size_t pos, void *ptr) { + // return 0 if pos is out of bounds + if (pos >= v->top) { + return 0; + } + + memcpy(ptr, v->data + (pos * v->elemSize), v->elemSize); + return 1; +} + +/* Get the element at the end of the vector, decreasing the size by one */ +inline int Vector_Pop(Vector *v, void *ptr) { + if (v->top > 0) { + if (ptr != NULL) { + Vector_Get(v, v->top - 1, ptr); + } + v->top--; + return 1; + } + return 0; +} + +inline int __vector_PutPtr(Vector *v, size_t pos, void *elem) { + // resize if pos is out of bounds + if (pos >= v->cap) { + Vector_Resize(v, pos + 1); + } + + if (elem) { + memcpy(v->data + pos * v->elemSize, elem, v->elemSize); + } else { + memset(v->data + pos * v->elemSize, 0, v->elemSize); + } + // move the end offset to pos if we grew + if (pos >= v->top) { + v->top = pos + 1; + } + return 1; +} + +int Vector_Resize(Vector *v, size_t newcap) { + int oldcap = v->cap; + v->cap = newcap; + + v->data = realloc(v->data, v->cap * v->elemSize); + + // If we grew: + // put all zeros at the newly realloc'd part of the vector + if (newcap > oldcap) { + int offset = oldcap * v->elemSize; + memset(v->data + offset, 0, v->cap * v->elemSize - offset); + } + return v->cap; +} + +Vector *__newVectorSize(size_t elemSize, size_t cap) { + Vector *vec = malloc(sizeof(Vector)); + vec->data = calloc(cap, elemSize); + vec->top = 0; + vec->elemSize = elemSize; + vec->cap = cap; + + return vec; +} + +void Vector_Free(Vector *v) { + free(v->data); + free(v); +} + + +/* return the used size of the vector, regardless of capacity */ +inline int Vector_Size(Vector *v) { return v->top; } + +/* return the actual capacity */ +inline int Vector_Cap(Vector *v) { return v->cap; } diff --git a/csrc/lib/rmutil/vector.h b/csrc/lib/rmutil/vector.h new file mode 100644 index 0000000..a3b606f --- /dev/null +++ b/csrc/lib/rmutil/vector.h @@ -0,0 +1,73 @@ +#ifndef __VECTOR_H__ +#define __VECTOR_H__ +#include +#include +#include + +/* +* Generic resizable vector that can be used if you just want to store stuff +* temporarily. +* Works like C++ std::vector with an underlying resizable buffer +*/ +typedef struct { + char *data; + size_t elemSize; + size_t cap; + size_t top; + +} Vector; + +/* Create a new vector with element size. This should generally be used + * internall by the NewVector macro */ +Vector *__newVectorSize(size_t elemSize, size_t cap); + +// Put a pointer in the vector. To be used internall by the library +int __vector_PutPtr(Vector *v, size_t pos, void *elem); + +/* +* Create a new vector for a given type and a given capacity. +* e.g. NewVector(int, 0) - empty vector of ints +*/ +#define NewVector(type, cap) __newVectorSize(sizeof(type), cap) + +/* +* get the element at index pos. The value is copied in to ptr. If pos is outside +* the vector capacity, we return 0 +* otherwise 1 +*/ +int Vector_Get(Vector *v, size_t pos, void *ptr); + +/* Get the element at the end of the vector, decreasing the size by one */ +int Vector_Pop(Vector *v, void *ptr); + +//#define Vector_Getx(v, pos, ptr) pos < v->cap ? 1 : 0; *ptr = +//*(typeof(ptr))(v->data + v->elemSize*pos) + +/* +* Put an element at pos. +* Note: If pos is outside the vector capacity, we resize it accordingly +*/ +#define Vector_Put(v, pos, elem) __vector_PutPtr(v, pos, elem ? &(typeof(elem)){elem} : NULL) + +/* Push an element at the end of v, resizing it if needed. This macro wraps + * __vector_PushPtr */ +#define Vector_Push(v, elem) __vector_PushPtr(v, elem ? &(typeof(elem)){elem} : NULL) + +int __vector_PushPtr(Vector *v, void *elem); + +/* resize capacity of v */ +int Vector_Resize(Vector *v, size_t newcap); + +/* return the used size of the vector, regardless of capacity */ +int Vector_Size(Vector *v); + +/* return the actual capacity */ +int Vector_Cap(Vector *v); + +/* free the vector and the underlying data. Does not release its elements if + * they are pointers*/ +void Vector_Free(Vector *v); + +int __vecotr_PutPtr(Vector *v, size_t pos, void *elem); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/Makefile b/csrc/redis-filtered-sort/Makefile new file mode 100644 index 0000000..86231d9 --- /dev/null +++ b/csrc/redis-filtered-sort/Makefile @@ -0,0 +1,32 @@ +include ../variables.include + +#override path defined in variables +JANSSON_LIBDIR=../lib/jansson + +ifndef RM_INCLUDE_DIR + RM_INCLUDE_DIR=../lib/ +endif + +ifndef RMUTIL_LIBDIR + RMUTIL_LIBDIR=../lib/rmutil +endif + +SOURCEDIR=$(shell pwd -P) +CC_SOURCES = $(wildcard $(SOURCEDIR)/*.c) +CC_OBJECTS = $(patsubst $(SOURCEDIR)/%.c, $(SOURCEDIR)/%.o, $(CC_SOURCES)) + +all: rmutil $(CC_OBJECTS) filter_module.so +rmutil: FORCE + $(MAKE) -C $(RMUTIL_LIBDIR) + +filter_module.so: $(CC_OBJECTS) + $(LD) -o $@ $(CC_OBJECTS) $(SHOBJ_LDFLAGS) $(LIBS) -L$(RMUTIL_LIBDIR) -L$(JANSSON_LIBDIR)/src/.libs -lrmutil -lpthread -lc -Bstatic -ljansson + +valgrind: + valgrind --leak-check=full --show-possibly-lost=no redis-server --loadmodule ./filter_module.so --loglevel debug + +clean: + rm -rf *.xo *.so *.o + cd $(RMUTIL_LIBDIR) && make clean + +FORCE: diff --git a/csrc/redis-filtered-sort/filter_module.c b/csrc/redis-filtered-sort/filter_module.c new file mode 100644 index 0000000..c1ba1ec --- /dev/null +++ b/csrc/redis-filtered-sort/filter_module.c @@ -0,0 +1,135 @@ +#include "filter_module.h" +#include "fsort.h" +#include "fsort_utils.h" +#include "thread_pool.h" + +static FSortPool_t *sortPool; + +int FSortBust_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + RedisModule_AutoMemory(ctx); + + int ctxFlags = RedisModule_GetContextFlags(ctx); + int isLua = ctxFlags & REDISMODULE_CTX_FLAGS_LUA; + int isMulti = ctxFlags & REDISMODULE_CTX_FLAGS_MULTI; + + if (isLua || isMulti) { + RedisModule_ReplyWithError(ctx, "Can't work in MULTI mode or LUA script!"); + return REDISMODULE_OK; + } + + if (argc <3 || argc > 4 ) { + return RedisModule_WrongArity(ctx); + } + + pthread_t tid; + RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,NULL,NULL,NULL,0); + + void **targ = RedisModule_Alloc(sizeof(void*)*3); + targ[0] = bc; + targ[1] = (void*)(unsigned long) argc; + targ[2] = argv; + + if (pthread_create(&tid,NULL,fsort_bust_thread,targ) != 0) { + RedisModule_AbortBlock(bc); + return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread"); + } + + return REDISMODULE_OK; +} + +int FSortAggregate_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + RedisModule_AutoMemory(ctx); + + int ctxFlags = RedisModule_GetContextFlags(ctx); + int isLua = ctxFlags & REDISMODULE_CTX_FLAGS_LUA; + int isMulti = ctxFlags & REDISMODULE_CTX_FLAGS_MULTI; + + if (isLua || isMulti) { + RedisModule_ReplyWithError(ctx, "Can't work in MULTI mode or LUA script!"); + return REDISMODULE_OK; + } + + if (argc != 4) { + return RedisModule_WrongArity(ctx); + } + + RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,NULL,NULL,NULL,0); + + void **targ = RedisModule_Alloc(sizeof(void*)*3); + targ[0] = bc; + //targ[1] = argc; + targ[2] = argv; + + tpool_add_work(sortPool, fsort_aggregate_thread, targ); + + return REDISMODULE_OK; +} + +int FSort_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + RedisModule_AutoMemory(ctx); + + int ctxFlags = RedisModule_GetContextFlags(ctx); + int isLua = ctxFlags & REDISMODULE_CTX_FLAGS_LUA; + int isMulti = ctxFlags & REDISMODULE_CTX_FLAGS_MULTI; + + if (isLua || isMulti) { + RedisModule_ReplyWithError(ctx, "Can't work in MULTI mode or LUA script!"); + return REDISMODULE_OK; + } + + if (argc < 7 ) { + return RedisModule_WrongArity(ctx); + } + + RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,NULL,NULL,NULL,0); + + FSortObj_t *sort = fsort_new_fsort(); + sort->ctx = ctx; + int parseRes = fsort_parse_args(sort, ctx, argv, argc); + + fsort_form_keys(sort); + + if (parseRes != REDISMODULE_OK) { + fsort_free_fsort(sort); + RedisModule_UnblockClient(bc,NULL); + return REDISMODULE_ERR; + } + + void **targ = RedisModule_Alloc(sizeof(void*)*2); + targ[0] = bc; + targ[1] = sort; + + tpool_add_work(sortPool, fsort_fsort_thread, targ); + + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + long long poolSize = 4; + if (RedisModule_Init(ctx, "FilterSortModule", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx, "fsort", FSort_RedisCommand, "write deny-script", 1, 2, 1) == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx, "fsortBust", FSortBust_RedisCommand, "write deny-script", 1, 1, 1) == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx, "fsortaggregate", FSortAggregate_RedisCommand, "write deny-script", 1, 2, 1) == REDISMODULE_ERR) { + return REDISMODULE_ERR; + } + + if (argc == 1) { + RedisModule_StringToLongLong(argv[0], &poolSize); + if (poolSize == 0) { + poolSize = 4; + } + } + + + sortPool = tpool_create(poolSize); + return REDISMODULE_OK; +} diff --git a/csrc/redis-filtered-sort/filter_module.h b/csrc/redis-filtered-sort/filter_module.h new file mode 100644 index 0000000..6f775db --- /dev/null +++ b/csrc/redis-filtered-sort/filter_module.h @@ -0,0 +1,11 @@ +#ifndef __FILTERMODULE_H +#define __FILTERMODULE_H 1 + +#include + +#include "fsort.h" + +#include "pthread.h" + + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/filters.c b/csrc/redis-filtered-sort/filters.c new file mode 100644 index 0000000..859d8d1 --- /dev/null +++ b/csrc/redis-filtered-sort/filters.c @@ -0,0 +1,454 @@ +#include "filters.h" +#include "utils.h" +#include "string_match.h" + +void filter_filter_free(Filter_t *filter); + +FilterBlock_t *filter_block_create() { + FilterBlock_t *filterBlock = malloc(sizeof(FilterBlock_t)); + filterBlock->size=0; + filterBlock->fieldsUsed = malloc(sizeof(const char *)); + filterBlock->fieldsCount = 0; + filterBlock->filters = NULL; + filterBlock->err = NULL; + return filterBlock; +} + +void filter_block_free(FilterBlock_t *fb) { + for (size_t i = 0; i < fb->size; i++){ + filter_filter_free(fb->filters[i]); + } + + if (fb->err != NULL) free((char*)fb->err); + + for (size_t i = 0; i < fb->fieldsCount; i++){ + const char *field = fb->fieldsUsed[i]; + free((char *)field); + } + + free(fb->filters); + free((void*)fb->fieldsUsed); + free(fb); +} + +Filter_t *filter_filter_create() { + Filter_t *filter = malloc(sizeof(Filter_t)); + filter->subFilters = filter_block_create(); + filter->value = NULL; + filter->err = NULL; + filter->field = NULL; + return filter; +} + + +void filter_filter_free(Filter_t *filter) { + if (filter->subFilters != NULL) { + filter_block_free(filter->subFilters); + filter->subFilters = NULL; + } + + if (filter->err != NULL) { + free((char*)filter->err); + } + + if (filter->value != NULL) { + free((char *) filter->value); + } + + free((char *)filter->field); + free(filter); +} + +void filter_block_insert(FilterBlock_t *fb, Filter_t *filter) { + fb->size ++; + if (fb->filters != NULL) { + fb->filters = realloc(fb->filters,fb->size * sizeof(Filter_t*)); + } else { + fb->filters = malloc(fb->size * sizeof(Filter_t*)); + } + fb->filters[fb->size-1] = filter; +} + +void filter_block_add_field(FilterBlock_t *fb, const char * field) { + fb->fieldsCount ++; + fb->fieldsUsed = realloc(fb->fieldsUsed, sizeof(const char *) * fb->fieldsCount); + fb->fieldsUsed[fb->fieldsCount -1] = strdup(field); +} + + +int filter_match_string(const char *str, const char * expr) { + MatchResult_t *res = str_match(strlwr(str), expr); + if (res->size > 0 && res->error_count == 0) { + free_match_result(res); + return(1); + } + free_match_result(res); + return(0); +} + +int _filter_eq(const char * str, const char * str2) { + if (strcasecmp(str, str2) == 0) { + return 1; + } else { + return 0; + } +} + +int _filter_ne(const char * str, const char * str2) { + if (strcasecmp(str, str2) == 0) { + return 0; + } else { + return 1; + } +} + +int _filter_lte(const char *str, const char *str2 ) { + long double a = 0,b = 0; + a = atof(str); + b = atof(str2); + + return a <= b ? 1 : 0; +} + +int _filter_gte(const char *str, const char *str2 ) { + long a = 0,b = 0; + a = atof(str); + b = atof(str2); + + return a >= b ? 1 : 0; +} + +// Processes data with selected filter +int _filter_process(HashTable *data, Filter_t *filter, int level) { + int match = 1; + + const char *valueStr = ht_get(data, filter->field); + + switch(filter->filterFunc) { + case FILTER_UNK : + match = 0; + break; + case FILTER_MATCH: { + match = filter_match_string(valueStr, filter->value); + } + break; + case FILTER_EQ: { + match = _filter_eq(valueStr, filter->value); + }; + break; + case FILTER_NE: { + match = _filter_ne(valueStr, filter->value); + } + break; + case FILTER_GTE: { + match = _filter_gte(valueStr, filter->value); + } + break; + case FILTER_LTE: { + match = _filter_lte(valueStr, filter->value); + } + break; + case FILTER_EXISTS: { + + if (strlen(valueStr) == 0) { + match = 0; + } else { + match = 1; + } + + } + break; + case FILTER_ISEMPTY: { + match = strlen(valueStr) == 0 ? 1 : 0; + } + break; + case FILTER_SOME: { + if (strlen(valueStr) == 0) { + match = 0; + } + match = _filter_eq(valueStr, filter->value); + } + break; + case FILTER_ANY: { + if (filter->subFilters == NULL) { + match = 0; + } else { + int andMatched = 1; + level ++; + + for (size_t j = 0; j < filter->subFilters->size; j++) { + + Filter_t *subfilter = filter->subFilters->filters[j]; + int mr = _filter_process(data, subfilter, level); + + if (subfilter->type == FILTERTYPE_SCALAR) { + if (andMatched && mr) andMatched = 1; + if (!(andMatched && mr)) andMatched = 0; + } else { + if (mr == 1) { + andMatched = 1; + break; + } else { + andMatched = 0; + } + } + } + match = andMatched; + } + } + break; + case FILTER_SUBFILTER: + break; + }; + return match; +} + +int _filter_process_obj(HashTable *data, Filter_t *filter) { + int match = 1; + if (filter->filterFunc == FILTER_ANY || filter->filterFunc == FILTER_SOME) { + int filterMatch = 0; + + FilterBlock_t *anyfilters = filter->subFilters; + for (size_t i = 0; i < anyfilters->size; i++){ + Filter_t *subFilter = anyfilters->filters[i]; + filterMatch = _filter_process(data, subFilter, 101); + if (filterMatch == 1) { + match = filterMatch; + break; + } + } + match = filterMatch; + + } else { + + for (size_t i = 0; i < filter->subFilters->size; i++){ + Filter_t *subFilter = filter->subFilters->filters[i]; + int mr = _filter_process(data, subFilter, 1); + if (mr == 0) { + match = 0; + break; + } + + } + } + return match; +} + + +int filter_filter_match(HashTable *data, FilterBlock_t *filterBlock) { + int matched = 1; + for (size_t i = 0; i < filterBlock->size; i++) { + Filter_t *filter = filterBlock->filters[i]; + + if (filter->type == FILTERTYPE_SCALAR) { + matched = _filter_process(data, filter,1); + } else if (filter->type == FILTERTYPE_OBJECT) { + matched = _filter_process_obj(data, filter); + } else if (filter->type == FILTERTYPE_MULTI) { + int anyMatched = 0; + for (size_t j = 0; j < filter->subFilters->size; j++) { + Filter_t *f = filter->subFilters->filters[j]; + anyMatched = _filter_process(data, f,1); + + if (anyMatched == 1) break; + } + matched = anyMatched; + } + if (matched == 0) break; + + } + return matched; +} + +FilterFunc_t filter_function_type(const char *str) { + if (strcasecmp(str, "match") == 0) { + return FILTER_MATCH; + } else if (strcasecmp(str, "gte") == 0) { + return FILTER_GTE; + } else if (strcasecmp(str, "lte") == 0) { + return FILTER_LTE; + } else if (strcasecmp(str, "eq") == 0) { + return FILTER_EQ; + } else if (strcasecmp(str, "ne") == 0) { + return FILTER_NE; + } else if (strcasecmp(str, "exists") == 0) { + return FILTER_EXISTS; + } else if (strcasecmp(str, "isempty") == 0) { + return FILTER_ISEMPTY; + } else if (strcasecmp(str, "some") == 0) { + return FILTER_SOME; + } else if (strcasecmp(str, "any") == 0) { + return FILTER_ANY; + } + return -1; +} + +int filter_create_multi_filter(FilterBlock_t * fb,const char * fieldName, json_t *doc) { + json_t *mfield; + json_t *matchJson = json_object_get(doc, "match"); + + Filter_t *filter = filter_filter_create(); + + filter->field = strdup(fieldName); + filter->type = FILTERTYPE_MULTI; + filter->filterFunc = FILTER_SUBFILTER; + filter->value = strdup(json_string_value(matchJson)); + + json_t *fieldsKey = json_object_get(doc, "fields"); + int index = 0; + + json_array_foreach(fieldsKey, index, mfield) { + const char * mfieldStr = json_string_value(mfield); + filter_block_add_field(fb, mfieldStr); + + Filter_t *mFilter = filter_filter_create(); + + mFilter->field = strdup(mfieldStr); + mFilter->value = strdup(json_string_value(matchJson)); + mFilter->filterFunc = FILTER_MATCH; + mFilter->type = FILTERTYPE_SCALAR; + + filter_block_insert(filter->subFilters, mFilter); + } + + filter_block_insert(fb, filter); + return 0; +} + +Filter_t *filter_create_scalar_filter(const char* fieldName, const char * filterFunction , json_t *doc){ + Filter_t *subFilter = filter_filter_create(); + subFilter->type = FILTERTYPE_SCALAR; + subFilter->field = strdup(fieldName); + filter_block_free(subFilter->subFilters); + subFilter->subFilters = NULL; + + if (json_typeof(doc) == JSON_REAL) { + double v = json_real_value(doc); + subFilter->value = malloc(100); + sprintf((char *)subFilter->value, "%g", v); + + } else if (json_typeof(doc) == JSON_INTEGER){ + long long v = json_integer_value(doc); + subFilter->value = malloc(sizeof(long long) + 1); + sprintf((char *)subFilter->value, "%lli", v); + } else { + subFilter->value = strdup(json_string_value(doc)); + } + + subFilter->filterFunc = filter_function_type(filterFunction); + return subFilter; +} + +int filter_filter_from_object(Filter_t *filter, const char * fieldName, json_t *doc) { + const char *jKey; + json_t *jValue; + filter->field = strdup(fieldName); + + json_object_foreach(doc, jKey, jValue) { + if (json_typeof(jValue) == JSON_OBJECT) { + const char * akey; + json_t * adoc; + filter->filterFunc = filter_function_type(jKey); + + json_object_foreach(jValue, akey, adoc) { + if (json_typeof(adoc) == JSON_OBJECT) { + Filter_t *tempFilter = filter_filter_create(); + tempFilter->filterFunc = filter->filterFunc; + tempFilter->type = filter->type; + + int res = filter_filter_from_object(tempFilter, fieldName, adoc ); + + if (res < 0) { + filter->err = strdup(tempFilter->err); + filter_filter_free(tempFilter); + return res; + } + + filter_block_insert(filter->subFilters, tempFilter); + } else { + Filter_t *subfilter = filter_create_scalar_filter(fieldName, jKey, adoc); + + if (subfilter->filterFunc == FILTER_UNK) { + char * errText = malloc(sizeof(char*)* 512); + filter->err = errText; + + filter_filter_free(subfilter); + return -1; + } + + filter_block_insert(filter->subFilters, subfilter); + } + } + } else { + Filter_t *subfilter = filter_create_scalar_filter(fieldName, jKey, jValue); + + if (subfilter->filterFunc == FILTER_UNK) { + char * errText = malloc(sizeof(char*)* 512); + sprintf(errText, "unknown func: %s", jKey ); + filter->err = errText; + filter_filter_free(subfilter); + + return -1; + } + + filter_block_insert(filter->subFilters, subfilter); + } + } + return 0; +} + +int filter_create_general_filter(FilterBlock_t *fb, const char * fieldName, json_t *doc, int root) { + Filter_t *filter; + + if (root) + filter_block_add_field(fb, fieldName); + + int fieldType = json_typeof(doc); + switch (fieldType) { + default:{ + filter = filter_create_scalar_filter(fieldName,"match", doc); + + if (filter->filterFunc == FILTER_UNK) { + char * errText = malloc(sizeof(char*)* 512); + fb->err = errText; + filter_filter_free(filter); + return -1; + } + } + break; + case JSON_OBJECT: { + //filter from object + filter = filter_filter_create(); + filter->type = FILTERTYPE_OBJECT; + filter->filterFunc = FILTER_SUBFILTER; + + int res = filter_filter_from_object(filter,fieldName, doc); + if (res < 0) { + fb->err = strdup(filter->err); + filter_filter_free(filter); + return -1; + } + }; + break; + } + filter_block_insert(fb, filter); + + return 0; +} + +FilterBlock_t *filter_create_filters(json_t *filter) { + + FilterBlock_t *filtersArray = filter_block_create(); + + const char *fieldName; + json_t *value; + json_object_foreach(filter, fieldName, value) { + if (strcasecmp(fieldName, "#multi") == 0) { + filter_create_multi_filter(filtersArray, fieldName,value); + } else { + filter_create_general_filter(filtersArray, fieldName,value,1); + } + } + + return filtersArray; +} diff --git a/csrc/redis-filtered-sort/filters.h b/csrc/redis-filtered-sort/filters.h new file mode 100644 index 0000000..5d1f7ab --- /dev/null +++ b/csrc/redis-filtered-sort/filters.h @@ -0,0 +1,61 @@ +#ifndef __FILTERS_H +#define __FILTERS_H 1 + +#include "jansson.h" +#include "general.h" + +#include "hashtable.h" +#include "regex.h" + +typedef enum FilterFunc { + FILTER_UNK = -1, + FILTER_SUBFILTER = 0, + FILTER_MATCH, + FILTER_GTE, + FILTER_LTE, + FILTER_EQ, + FILTER_NE, + FILTER_EXISTS, + FILTER_ISEMPTY, + FILTER_SOME, + FILTER_ANY +} FilterFunc_t; + +typedef enum FilterType { + FILTERTYPE_MULTI = 1, + FILTERTYPE_SCALAR, + FILTERTYPE_OBJECT, + FILTERTYPE_ARR, +} FilterType_t; + +struct Filter { + const char * field; + const char * value; + struct FilterBlock * subFilters; + FilterType_t type; + FilterFunc_t filterFunc; + const char * err; +}; + +struct FilterBlock { + size_t size; + size_t fieldsCount; + const char * err; + const char ** fieldsUsed; + struct Filter **filters; +}; + + +typedef struct Filter Filter_t; +typedef struct FilterBlock FilterBlock_t; + +FilterBlock_t *filter_block_create(); +void filter_block_insert(FilterBlock_t *,Filter_t *filter); +void filter_block_add_field(FilterBlock_t *, const char *); +void filter_block_free(FilterBlock_t *); +FilterBlock_t *filter_create_filters(json_t *filter); + +int filter_filter_match(HashTable *data, FilterBlock_t * filters); +int filter_match_string(const char *str, const char * str2); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/fsort.c b/csrc/redis-filtered-sort/fsort.c new file mode 100644 index 0000000..0dce1c9 --- /dev/null +++ b/csrc/redis-filtered-sort/fsort.c @@ -0,0 +1,792 @@ +#include "fsort.h" +#include "fsort_utils.h" +#include "string.h" + +RedisModuleString * fsort_char_rms(FSortObj_t *sort, const char *str) { + return RedisModule_CreateStringPrintf(sort->ctx,"%s", str); +} + + +void fsort_cache_buster(FSortObj_t *sortObj, const char *keyStr) { + long long ttl; + ttl = sortObj->curTime + sortObj->expire; + RedisModule_ThreadSafeContextLock(sortObj->ctx); + RedisModuleCallReply *rep = RedisModule_Call(sortObj->ctx, "PEXPIRE", "cl", keyStr, sortObj->expire); + RedisModule_FreeCallReply(rep); + rep = RedisModule_Call(sortObj->ctx, "ZADD", "clc", sortObj->tempKeysSet,ttl,keyStr); + RedisModule_FreeCallReply(rep); + RedisModule_ThreadSafeContextUnlock(sortObj->ctx); + +} + +void fsort_key_pexpire(FSortObj_t *sort, const char *keyStr) { + RedisModule_ThreadSafeContextLock(sort->ctx); + RedisModuleCallReply *rep = RedisModule_Call(sort->ctx, "PEXPIRE", "cl", keyStr, sort->expire); + RedisModule_FreeCallReply(rep); + RedisModule_ThreadSafeContextUnlock(sort->ctx); +} + +int fsort_parse_args(FSortObj_t *sObj, RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + + sObj->ctx = ctx; + sObj->idSet = fsort_read_string_param(argv[1]); + sObj->metaKey = fsort_read_string_param(argv[2]); + sObj->hashKey = fsort_read_string_param(argv[3]); + sObj->filter = argv[5]; + sObj->curTime = fsort_read_long_param(argv[6], 0); + sObj->filters = NULL; + + const char * orderStr = strdup(RedisModule_StringPtrLen(argv[4], NULL)); + sObj->order = orderStr; + + if (argc > 7 ) sObj->offset = fsort_read_long_param(argv[7], 0); + if (argc > 8 ) sObj->limit = fsort_read_long_param(argv[8], 10); + if (argc > 9 ) sObj->expire = fsort_read_long_param(argv[9], 30000); + + if (argc == 11 && argv[10] != NULL) { + sObj->keyOnly = 1; + } else { + sObj->keyOnly = 0; + } + sObj->tempKeysSet = malloc(strlen(sObj->idSet) + 17 + 1); + sprintf((char*)sObj->tempKeysSet, "%s::fsort_temp_keys", sObj->idSet ); + + size_t json_str_len; + json_error_t *json_err = NULL; + const char * filterString = RedisModule_StringPtrLen(sObj->filter, &json_str_len); + sObj->_json = json_loads(filterString, json_str_len, json_err); + + if (json_err != NULL) { + char* err = malloc(400); + sprintf(err,"Json parse error: %d", json_error_code(json_err)); + RedisModule_ReplyWithError(ctx, err); + free(err); + return REDISMODULE_ERR; + } + + FilterBlock_t *filters = filter_create_filters(sObj->_json); + if (filters->err != NULL) { + RedisModule_ReplyWithError(sObj->ctx, filters->err); + filter_block_free(filters); + return REDISMODULE_ERR; + } + sObj->filters = filters; + + return REDISMODULE_OK; +} + +FSortObj_t *fsort_new_fsort() { + FSortObj_t *sobj = malloc(sizeof(FSortObj_t)); + + sobj->expire = 30000; + sobj->offset = 0; + sobj->limit = 10; + sobj->sortData = str_arr_create(); + sobj->outputData = str_arr_create(); + sobj->filters = NULL; + sobj->order = NULL; + sobj->metaKey = NULL; + sobj->idSet = NULL; + sobj->fflKey = NULL; + sobj->tempKeysSet = NULL; + + return sobj; +} + +void fsort_free_fsort(FSortObj_t *sort) { + if (sort->outputData != NULL) { + str_arr_free(sort->outputData); + } + + if (sort->sortData != NULL) { + str_arr_free(sort->sortData); + } + + json_decref(sort->_json); + if (sort->filters != NULL) filter_block_free(sort->filters); + if (sort->pssKey != NULL) free((char *)sort->pssKey); + if (sort->fflKey != NULL) free((char *) sort->fflKey); + if (sort->tempKeysSet != NULL) free((char *)sort->tempKeysSet); + if (sort->hashKey != NULL) free((char*)sort->hashKey); + if (sort->idSet != NULL) free((char *)sort->idSet); + if (sort->metaKey != NULL) free((char *)sort->metaKey); + if (sort->order != NULL) free((char *)sort->order); + sort->ctx = NULL; + + free(sort); +} + +void fsort_form_keys(FSortObj_t *sort) { + array(const char *) ffKeyOpts = array_init(); + array(const char *) psKeyOpts = array_init(); + + array_push(ffKeyOpts, sort->idSet); + array_push(ffKeyOpts, sort->order); + + array_push(psKeyOpts, sort->idSet); + array_push(psKeyOpts, sort->order); + + size_t filterSize = json_object_size(sort->_json); + + if (sort->metaKey != NULL) { + if (sort->hashKey != NULL) { + array_push(ffKeyOpts, sort->metaKey); + array_push(psKeyOpts, sort->metaKey); + array_push(ffKeyOpts, sort->hashKey); + array_push(psKeyOpts, sort->hashKey); + } + + if (json_object_size(sort->_json) > 0) { + array_push(ffKeyOpts, RedisModule_StringPtrLen(sort->filter, NULL) ); + } + } else if (filterSize >= 1) { + json_t *idFilter = json_object_get(sort->_json, "#"); + if (json_is_string(idFilter)) { + json_t *filterString = json_sprintf("{\"#\":\"%s\"}", json_string_value(idFilter)); + const char * filterChar = json_string_value(filterString); + array_push(ffKeyOpts, filterChar); + json_decref(filterString); + } + } + + const char * charFflKey = joinStrArray(sort->ctx,ffKeyOpts.data,ffKeyOpts.length, ":"); + const char * charPssKey = joinStrArray(sort->ctx, psKeyOpts.data,psKeyOpts.length, ":"); + + sort->fflKey = strdup(charFflKey); + sort->pssKey = strdup(charPssKey); + free((char *)charFflKey); + free((char *)charPssKey); + array_free(ffKeyOpts); + array_free(psKeyOpts); +} + +HashTable **fsort_get_meta_hashtable(FSortObj_t *sort, const char **sData, size_t len, const char **fieldsNeeded, size_t fieldsCount, size_t *resLen){ + HashTable **data = malloc(sizeof(HashTable*)); + size_t hashLen = 0; + for (size_t i = 0; i < len; i++) { + const char *keyName = sort->metaKey; + const char *keyValue = sData[i]; + + char * metaKeyName = repl_str(keyName, "*", keyValue); + size_t metaKeyNameLen = strlen(metaKeyName); + RedisModuleString *metaKeyString = RedisModule_CreateString(sort->ctx, metaKeyName, metaKeyNameLen); + RedisModule_ThreadSafeContextLock(sort->ctx); + RedisModuleKey *metaKey = RedisModule_OpenKey(sort->ctx, metaKeyString , REDISMODULE_READ || REDISMODULE_WRITE); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + if (RedisModule_KeyType(metaKey) == REDISMODULE_KEYTYPE_HASH) { + HashTable *dataHash = ht_create(fieldsCount); + RedisModule_ThreadSafeContextLock(sort->ctx); + for (size_t j = 0; j < fieldsCount; j++) { + const char *dbStr; + size_t dbStrLen = 0; + RedisModuleString *value; + + if (strcmp(fieldsNeeded[j], "#") == 0) { + dbStr = strdup(sData[i]); + } else { + RedisModule_HashGet(metaKey,REDISMODULE_HASH_NONE|REDISMODULE_HASH_CFIELDS, fieldsNeeded[j], &value, NULL); + dbStr = strdup(RedisModule_StringPtrLen(value, &dbStrLen)); + + if (value == NULL) { + free((char *)dbStr); + dbStr = strdup(""); + } + free(value); + } + + ht_put(dataHash, fieldsNeeded[j], dbStr); + free((char*)dbStr); + } + RedisModule_ThreadSafeContextUnlock(sort->ctx); + hashLen += 1; + data = realloc(data, hashLen * sizeof(HashTable*)); + data[hashLen-1] = dataHash; + } + + RedisModule_FreeString(sort->ctx,metaKeyString); + free(metaKeyString); + free(metaKey); + free(metaKeyName); + } + *resLen = hashLen > 0 ? hashLen : 0; + + return data; +} + +KvStr_t ** fsort_get_meta(FSortObj_t *sort, const char **sData, size_t len, size_t *resLen) { + *resLen = 0; + KvStr_t **data = malloc(sizeof(KvStr_t*)); + + RedisModule_ThreadSafeContextLock(sort->ctx); + for (size_t i = 0; i < len; i++) { + const char *keyName = sort->metaKey; + const char *keyValue = sData[i]; + + char * metaKeyName = repl_str(keyName, "*", keyValue); + + RedisModuleString *metaKeyString = RedisModule_CreateString(sort->ctx, metaKeyName, strlen(metaKeyName)); + RedisModuleKey *metaKey = RedisModule_OpenKey(sort->ctx, metaKeyString , REDISMODULE_READ); + + if (RedisModule_KeyType(metaKey) == REDISMODULE_KEYTYPE_HASH) { + const char *dbStr; + size_t dbStrLen = 0; + RedisModuleString *value; + + RedisModule_HashGet(metaKey,REDISMODULE_HASH_NONE|REDISMODULE_HASH_CFIELDS, sort->hashKey, &value, NULL); + + *resLen += 1; + + data = realloc(data, sizeof(KvStr_t*) * *resLen); + + KvStr_t *r = malloc(sizeof(KvStr_t)); + r->key = strdup(sData[i]); + + dbStr = RedisModule_StringPtrLen(value, &dbStrLen); + if (value == NULL) { + dbStr = strdup(""); + } + r->value = strdup(dbStr); + data[*resLen-1] = r; + free(value); + } + RedisModule_CloseKey(metaKey); + free(metaKeyString); + free(metaKeyName); + } + RedisModule_ThreadSafeContextUnlock(sort->ctx); + return data; +} + +KvStr_t **_get_aggregate_fields(json_t *json, size_t *size) { + KvStr_t **records = malloc(json_object_size(json) * sizeof(KvStr_t*)); + const char * field; + json_t * json_aggregate; + size_t arrSize = 0; + json_object_foreach(json, field, json_aggregate) { + KvStr_t * r = malloc(sizeof(KvStr_t)); + const char * val = json_string_value(json_aggregate); + r->key = strdup(field); + + if (val == NULL) { + r->value =strdup(""); + } else { + r->value =strdup(val); + } + + records[arrSize] = r; + arrSize ++; + } + *size = arrSize; + return records; +} + +int _peform_aggr(const char * func,const char * field, json_t *doc, const char * val) { + + if (strcasecmp(func, "sum") == 0) { + json_int_t jval = 0; + json_t *vf = json_object_get(doc, field); + if (vf == NULL) { + vf = json_object(); + json_integer_set(vf, jval); + json_object_set(doc, field, vf); + json_decref(vf); + } else + jval = json_integer_value(vf); + + long long llNew = atoll(val); + jval += llNew; + + json_t *intVal = json_integer(jval); + json_object_set(doc,field,intVal); + json_decref(intVal); + } + return 1; +} + +json_t *fsort_aggregate_data(HashTable **data, KvStr_t **aggr, const char **fneeded, size_t data_size, size_t fneededsize) { + json_t *root = json_object(); + + for (size_t i = 0; i < data_size; i++) { + HashTable *rec = data[i]; + + for (size_t j = 0; j < fneededsize; j++){ + const char *aggrFunc = aggr[j]->value; + const char *recValue = ht_get(rec, fneeded[j]); + _peform_aggr(aggrFunc, fneeded[j], root, recValue); + } + + } + return root; +} + +int fsort_respond_list(FSortObj_t *sort, const char *keyStr) { + fsort_cache_buster(sort, keyStr); + fsort_key_pexpire(sort, keyStr); + if (sort->keyOnly == 1) { + RedisModule_ReplyWithSimpleString(sort->ctx, keyStr); + return REDISMODULE_OK; + } + + RedisModule_ThreadSafeContextLock(sort->ctx); + RedisModuleCallReply *replyCount = RedisModule_Call(sort->ctx, "LLEN", "c", keyStr); + RedisModuleCallReply *reply = RedisModule_Call(sort->ctx, "LRANGE","cll", keyStr, sort->offset, sort->offset + sort->limit -1); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + + if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ARRAY) { + size_t elCount = RedisModule_CallReplyLength(reply); + RedisModule_ReplyWithArray(sort->ctx, REDISMODULE_POSTPONED_ARRAY_LEN); + + for (size_t i = 0; i < elCount; i++) { + RedisModuleCallReply *subreply = RedisModule_CallReplyArrayElement(reply, i); + RedisModule_ReplyWithCallReply(sort->ctx, subreply); + } + + RedisModule_ReplyWithCallReply(sort->ctx, replyCount); + RedisModule_ReplySetArrayLength(sort->ctx, elCount + 1); + + RedisModule_FreeCallReply(reply); + RedisModule_FreeCallReply(replyCount); + + return REDISMODULE_OK; + } else { + RedisModule_FreeCallReply(reply); + RedisModule_FreeCallReply(replyCount); + + return REDISMODULE_ERR; + } +} + +int fsort_sort_data(FSortObj_t *sort) { + const char **dataToSort = sort->sortData->values; + size_t elCount = sort->sortData->size; + + if (sort->metaKey != NULL && sort->hashKey != NULL) { + size_t metaSize = 0; + KvStr_t **data = fsort_get_meta(sort, dataToSort, elCount, &metaSize); + + if (strcasecmp(sort->order, "asc")) { + qsort(data, metaSize, sizeof(KvStr_t*), fsort_compare_meta_asc ); + } else { + qsort(data, metaSize, sizeof(KvStr_t*), fsort_compare_meta_desc ); + } + + for(size_t i = 0; ikey); + kvstr_free(data[i]); + } + free(data); + + } else { + if (strcasecmp(sort->order, "asc") == 0) { + qsort(dataToSort, elCount, sizeof(const char*), fsort_compare_string_asc ); + } else { + qsort(dataToSort, elCount, sizeof(const char*), fsort_compare_string_desc ); + } + } + + return 0; +} + +const char **fsort_load_idset_data(FSortObj_t *sort, size_t *dataSize) { + + RedisModule_ThreadSafeContextLock(sort->ctx); + RedisModuleCallReply *reply = RedisModule_Call(sort->ctx, "SMEMBERS", "c", sort->idSet); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + + size_t elCount = 0; + const char **data = NULL; + + if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ARRAY) { + elCount = RedisModule_CallReplyLength(reply); + if (elCount > 0) { + data = malloc(elCount * sizeof(const char*)); + for (size_t i = 0; i < elCount; i++) { + RedisModuleCallReply *subreply = RedisModule_CallReplyArrayElement(reply, i); + RedisModuleString *strrep = RedisModule_CreateStringFromCallReply(subreply); + size_t loadLen = 0; + data[i] = strdup(RedisModule_StringPtrLen(strrep,&loadLen)); + RedisModule_FreeString(sort->ctx, strrep); + } + *dataSize = elCount; + } + + } else { + *dataSize = elCount; + } + + RedisModule_FreeCallReply(reply); + return data; +} + +const char **fsort_load_pss_data(FSortObj_t *sort,size_t *dataSize) { + const char **data; + + RedisModule_ThreadSafeContextLock(sort->ctx); + RedisModuleCallReply *reply = RedisModule_Call(sort->ctx, "LRANGE", "cll", sort->pssKey, 0, -1); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + + if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ARRAY) { + size_t ds = 0; + ds = RedisModule_CallReplyLength(reply); + if (ds > 0) { + data = malloc(ds * sizeof(const char*)); + + for (size_t i = 0; i < ds; i++) { + RedisModuleCallReply *subreply = RedisModule_CallReplyArrayElement(reply, i); + RedisModuleString *strrep = RedisModule_CreateStringFromCallReply(subreply); + data[i] = strdup(RedisModule_StringPtrLen(strrep,NULL)); + free(strrep); + } + } + + *dataSize = ds; + RedisModule_FreeCallReply(reply); + return ds > 0 ? data: NULL; + + } else { + RedisModule_FreeCallReply(reply); + RedisModule_ReplyWithError(sort->ctx, "Unable to LRANGE PreSortedKey!"); + return NULL; + } +} + +const char **fsort_filter_data(FSortObj_t *sort, size_t *outputDataSize) { + const char **dataToSort = sort->sortData->values; + size_t elCount = sort->sortData->size; + size_t ds = 0; + const char **outputData = malloc(sizeof(const char *)); + + if (!sort->metaKey) { + json_t *idFilter = json_object_get(sort->_json, "#"); + const char * idFilterStr = json_string_value(idFilter); + + for (size_t i = 0; i < elCount; i++){ + if (filter_match_string(dataToSort[i],idFilterStr)==1) { + ds += 1; + outputData = realloc(outputData,ds * sizeof(const char*)); + outputData[ds-1] = strdup(dataToSort[i]); + } + } + } else { + size_t dataHashesLen = 0; + HashTable **dataHashes = fsort_get_meta_hashtable(sort, dataToSort, elCount, sort->filters->fieldsUsed, sort->filters->fieldsCount, &dataHashesLen); + if (dataHashesLen > 0) { + for (size_t i = 0; i < dataHashesLen; i++){ + if (filter_filter_match(dataHashes[i], sort->filters) == 1) { + ds += 1; + outputData = realloc(outputData, ds * sizeof(const char*)); + outputData[ds -1] = strdup(dataToSort[i]); + } + } + + for (size_t i = 0; i < dataHashesLen; i++) { + ht_free(dataHashes[i]); + } + } else { + ds = 0; + } + free(dataHashes); + } + + *outputDataSize = ds; + return outputData; +} + +int fsort_key_type(FSortObj_t *sort, const char *keyStr) { + RedisModuleString *keyName = fsort_char_rms(sort, keyStr); + RedisModuleKey *key = RedisModule_OpenKey(sort->ctx, keyName, REDISMODULE_READ); + int kType = RedisModule_KeyType(key); + RedisModule_CloseKey(key); + RedisModule_FreeString(sort->ctx, keyName); + return kType; +} + +int fsort_fsort(FSortObj_t *sort) { + sort->fflKeyName = fsort_char_rms(sort, sort->fflKey); + sort->pssKeyName = fsort_char_rms(sort, sort->pssKey); + + RedisModule_ThreadSafeContextLock(sort->ctx); + + RedisModuleKey *fflkey = RedisModule_OpenKey(sort->ctx, sort->fflKeyName, REDISMODULE_READ); + RedisModuleKey *psskey = RedisModule_OpenKey(sort->ctx, sort->pssKeyName, REDISMODULE_READ); + + int pssKType = RedisModule_KeyType(psskey); + int fflKType = RedisModule_KeyType(fflkey); + + RedisModule_CloseKey(fflkey); + RedisModule_CloseKey(psskey); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + + if (fflKType == REDISMODULE_KEYTYPE_LIST) { + fsort_key_pexpire(sort, sort->pssKey); + fsort_cache_buster(sort, sort->pssKey); + return fsort_respond_list(sort, sort->fflKey); + } + + if (pssKType == REDISMODULE_KEYTYPE_LIST) { + //Respond pss + if (strcasecmp(sort->fflKey, sort->pssKey) == 0) { + return fsort_respond_list(sort, sort->pssKey); + } + + //Load pss cache if sort requested + size_t sdataSize = 0; + const char **data = fsort_load_pss_data(sort, &sdataSize); + + if (sdataSize > 0) { + sort->sortData->values = data; + } + + sort->sortData->size = sdataSize; + + if (sort->sortData->size == 0) { + return REDISMODULE_OK; + } + + } else { + size_t loadData = 0; + const char **data = fsort_load_idset_data(sort, &loadData); + if (loadData > 0) { + sort->sortData->values = data; + sort->sortData->size = loadData; + } else { + sort->sortData->size = 0; + } + + if (sort->sortData->size > 0) { + fsort_sort_data(sort); + //insert into db for next use + if (fsort_key_type(sort, sort->pssKey) == REDISMODULE_KEYTYPE_EMPTY) { + RedisModule_ThreadSafeContextLock(sort->ctx); + psskey = RedisModule_OpenKey(sort->ctx, sort->pssKeyName, REDISMODULE_WRITE); + for(size_t i = 0; i< sort->sortData->size; i++) { + const char **v = sort->sortData->values; + RedisModuleString *insVal = RedisModule_CreateString(sort->ctx,v[i], strlen(v[i])); + RedisModule_ListPush(psskey, REDISMODULE_LIST_TAIL, insVal); + RedisModule_FreeString(sort->ctx, insVal); + } + RedisModule_CloseKey(psskey); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + } else { + RM_LOG_VERBOSE(sort->ctx,"Key Already created key %s ommiting write", sort->pssKey); + } + fsort_key_pexpire(sort, sort->pssKey); + fsort_cache_buster(sort, sort->pssKey); + } else { + + if (sort->keyOnly == 1) { + RedisModule_ReplyWithSimpleString(sort->ctx, sort->pssKey); + return REDISMODULE_OK; + } + RedisModule_ReplyWithLongLong(sort->ctx, 0); + return REDISMODULE_OK; + } + + if (strcmp(sort->fflKey, sort->pssKey) == 0) { + return fsort_respond_list(sort, sort->pssKey); + } + + fsort_key_pexpire(sort, sort->pssKey); + fsort_cache_buster(sort, sort->pssKey); + } + + + sort->outputData->values = fsort_filter_data(sort, &sort->outputData->size); + size_t outputDataSize = sort->outputData->size; + + const char** outputData = sort->outputData->values; + + if (outputDataSize > 0) { + if (fsort_key_type(sort, sort->fflKey) == REDISMODULE_KEYTYPE_EMPTY) { + RedisModule_ThreadSafeContextLock(sort->ctx); + + fflkey = RedisModule_OpenKey(sort->ctx, sort->fflKeyName, REDISMODULE_WRITE); + + for(size_t i = 0; ictx, strdup(outputData[i]),strlen(outputData[i])); + RedisModule_ListPush(fflkey, REDISMODULE_LIST_TAIL, insStr); + free(insStr); + } + + RedisModule_CloseKey(fflkey); + RedisModule_ThreadSafeContextUnlock(sort->ctx); + } else { + RM_LOG_VERBOSE(sort->ctx,"Filter Key already created created key %s: ommiting write", sort->fflKey); + } + return fsort_respond_list(sort, sort->fflKey); + } + + if (sort->keyOnly == 1) { + RedisModule_ReplyWithSimpleString(sort->ctx, sort->fflKey); + return REDISMODULE_OK; + } + + RedisModule_ReplyWithArray(sort->ctx,1); + RedisModule_ReplyWithLongLong(sort->ctx, 0); + return REDISMODULE_OK; + +} + +int fsort_fsort_aggregate(FSortObj_t *sort) { + size_t dataLen = 0; + const char **data = fsort_load_pss_data(sort, &dataLen); + + if (dataLen >0) { + sort->sortData->values = data; + sort->sortData->size = dataLen; + + size_t fneededLen=0; + KvStr_t **aggregateKv = _get_aggregate_fields(sort->_json, &fneededLen); + const char **fneeded = calloc(fneededLen, sizeof(const char *)); + + for (size_t i = 0; i < fneededLen; i++){ + fneeded[0] = aggregateKv[i]->key; + } + + if (fneededLen >0) { + size_t metaDataLen = 0; + HashTable **metaData = fsort_get_meta_hashtable(sort, data, dataLen, fneeded, fneededLen, &metaDataLen); + json_t *obj = fsort_aggregate_data(metaData, aggregateKv, fneeded, metaDataLen, fneededLen); + + const char *json_encoded = _encode_json(obj); + json_decref(obj); + RedisModule_ReplyWithSimpleString(sort->ctx, json_encoded); + + for (size_t i = 0; i < metaDataLen; i++){ + ht_free(metaData[i]); + } + + for (size_t i = 0; i < fneededLen; i++){ + kvstr_free(aggregateKv[i]); + } + free(metaData); + free(aggregateKv); + free((void*)json_encoded); + + free(fneeded); + return REDISMODULE_OK; + } + } + RedisModule_ReplyWithSimpleString(sort->ctx,"{}"); + return REDISMODULE_ERR; +} + +int fsort_fsort_bust(RedisModuleCtx * ctx, RedisModuleString **argv, int argc) { + RedisModuleString *idSetKey = RedisModule_CreateStringPrintf(ctx, "%s::fsort_temp_keys", RedisModule_StringPtrLen(argv[1], NULL) ); + + long long expire = 30000; + if (argc > 3) fsort_read_long_param(argv[3], 30000); + + long long curtime = fsort_read_long_param(argv[2], 0); + + + RedisModuleCallReply *reply = RedisModule_Call(ctx, "ZRANGEBYSCORE", "slc", idSetKey, curtime-expire, "+inf"); + int rKeyType = RedisModule_CallReplyType(reply); + + if (rKeyType == REDISMODULE_REPLY_ARRAY) { + size_t elCount = RedisModule_CallReplyLength(reply); + if (elCount > 0) { + RedisModule_ThreadSafeContextLock(ctx); + + for (size_t i = 0; i < elCount; i++) { + RedisModuleCallReply *keyNameReply = RedisModule_CallReplyArrayElement(reply, i); + RedisModuleString *keyName = RedisModule_CreateStringFromCallReply(keyNameReply); + RedisModuleCallReply *rDel = RedisModule_Call(ctx, "del", "s", keyName); + RedisModule_FreeCallReply(rDel); + RedisModule_FreeString(ctx, keyName); + } + + RedisModuleCallReply *rDelSet = RedisModule_Call(ctx, "del", "s", idSetKey); + RedisModule_FreeCallReply(rDelSet); + RedisModule_FreeString(ctx, idSetKey); + + RedisModule_ThreadSafeContextUnlock(ctx); + } + } else { + RedisModule_FreeCallReply(reply); + return REDISMODULE_ERR; + } + + RedisModule_FreeCallReply(reply); + RedisModule_ReplyWithLongLong(ctx, 0); + return REDISMODULE_OK; +} + +void *fsort_bust_thread(void *arg) { + void **targ = arg; + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)targ[0]; + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(bc); + + int argc = (unsigned long)targ[1]; + RedisModuleString **argv = targ[2]; + fsort_fsort_bust(ctx, argv, argc); + + RedisModule_UnblockClient(bc,NULL); + RedisModule_FreeThreadSafeContext(ctx); + + return NULL; +} + +void *fsort_aggregate_thread(void *arg) { + void **targ = (void*)arg; + RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)targ[0]; + //int argc = (unsigned long)targ[1]; + RedisModuleString **argv = targ[2]; + + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(bc); + + RedisModuleString *aggregates = argv[3]; + + FSortObj_t *sort = fsort_new_fsort(); + sort->ctx = ctx; + sort->pssKey = strdup(RedisModule_StringPtrLen(argv[1], NULL)); + sort->metaKey = strdup(RedisModule_StringPtrLen(argv[2], NULL)); + sort->idSet = NULL; + sort->fflKey = NULL; + sort->tempKeysSet = NULL; + + sort->hashKey = strdup(""); + size_t json_str_len = 0; + + json_error_t *json_err = NULL; + const char * filterString = strdup(RedisModule_StringPtrLen(aggregates, &json_str_len)); + + if (json_str_len == 0) { + RedisModule_ReplyWithError(sort->ctx, "Incorrect aggregate parameter"); + RedisModule_UnblockClient(bc, NULL); + return NULL; + } + + sort->_json = json_loads(filterString, json_str_len, json_err); + if (json_err != NULL) { + RM_LOG_WARNING(ctx, "Json parse error: %s", json_error_code(json_err)); + } + + fsort_fsort_aggregate(sort); + fsort_free_fsort(sort); + RedisModule_UnblockClient(bc, NULL); + RedisModule_FreeThreadSafeContext(ctx); + + return NULL; +} + +void *fsort_fsort_thread(void *arg) { + void **targ = (void*)arg; + RedisModuleBlockedClient *bc =targ[0]; + FSortObj_t *sort = targ[1]; + + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(bc); + sort->ctx = ctx; + fsort_fsort(sort); + + RedisModule_FreeThreadSafeContext(ctx); + fsort_free_fsort(sort); + + RedisModule_UnblockClient(bc, NULL); + + return NULL; +} + + diff --git a/csrc/redis-filtered-sort/fsort.h b/csrc/redis-filtered-sort/fsort.h new file mode 100644 index 0000000..52247ab --- /dev/null +++ b/csrc/redis-filtered-sort/fsort.h @@ -0,0 +1,34 @@ +#ifndef __FSORT_H +#define __FSORT_H + +#include "general.h" +#include "jansson.h" +#include "fsort_types.h" +#include "hashtable.h" +#include "utils.h" +#include "filters.h" + +void *fsort_fsort_thread(void *arg); + +void *fsort_aggregate_thread(void *arg); +void *fsort_bust_thread(void *arg); + +const char **fsort_load_pss_data(FSortObj_t *sort,size_t *dataSize); +const char **fsort_load_idset_data(FSortObj_t *sort, size_t *dataSize); +int fsort_sort_data(FSortObj_t *sort); +int fsort_respond_list(FSortObj_t *sort, const char *key); +json_t *fsort_aggregate_data(HashTable **data, KvStr_t **aggr, const char **fneeded, size_t data_size, size_t fneededsize); +KvStr_t ** fsort_get_meta(FSortObj_t *sort, const char **sData, size_t len, size_t *resLen); +HashTable **fsort_get_meta_hashtable(FSortObj_t *sort, const char **sData, size_t len, const char **fieldsNeeded, size_t fieldsCount, size_t *resLen); +void fsort_form_keys(FSortObj_t *sort); + +int fsort_parse_args(FSortObj_t *sObj, RedisModuleCtx *ctx, RedisModuleString **argv, int argc); +FSortObj_t *fsort_new_fsort(); + +void fsort_cache_buster(FSortObj_t *sortObj, const char *key); +void fsort_free_fsort(FSortObj_t *sort); + +int fsort_key_type(FSortObj_t *sort, const char *keyName); + + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/fsort_types.h b/csrc/redis-filtered-sort/fsort_types.h new file mode 100644 index 0000000..97c5e15 --- /dev/null +++ b/csrc/redis-filtered-sort/fsort_types.h @@ -0,0 +1,57 @@ +#ifndef __FSORT_TYPES_H +#define __FSORT_TYPES_H +#include "general.h" +#include "jansson.h" +#include "hashtable.h" +#include "filters.h" + +typedef struct KvStr { + const char * key; + const char * value; +} KvStr_t; + +typedef struct KvHashArr { + size_t size; + HashTable **values; +} KvHashArr_t; + +typedef struct KvStrArr { + size_t size; + KvStr_t **values; +} KvStrArr_t; + +typedef struct StrArr { + size_t size; + const char **values; +} StrArr_t; + +typedef struct FSortObj { + const char *fflKey; + const char *pssKey; + RedisModuleString *fflKeyName; + RedisModuleString *pssKeyName; + + const char *tempKeysSet; + const char *idSet; + const char *metaKey; + const char *hashKey; + const char *order; + RedisModuleString *filter; + long long curTime; + long long offset; + long long limit; + long long expire; + long long keyOnly; + + int threaded; + json_t * _json; + + StrArr_t *sortData; + StrArr_t *outputData; + FilterBlock_t *filters; + //Store redis context + RedisModuleCtx *ctx; + +} FSortObj_t; + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/fsort_utils.c b/csrc/redis-filtered-sort/fsort_utils.c new file mode 100644 index 0000000..59b03a6 --- /dev/null +++ b/csrc/redis-filtered-sort/fsort_utils.c @@ -0,0 +1,186 @@ +#include "fsort_utils.h" + +KvStrArr_t *kvstr_arr_create() { + KvStrArr_t *kvstr=malloc(sizeof(KvStrArr_t)); + kvstr->size = 0; + kvstr->values = malloc(sizeof(KvStr_t*)); + return kvstr; +} + +int kvstr_arr_insert(KvStrArr_t *arr, KvStr_t *v) { + arr->size +=1; + arr->values = realloc(arr->values, sizeof(KvStr_t*) * (arr->size)); + arr->values[arr->size-1] = v; + return 1; +} + +int kvstr_arr_free(KvStrArr_t *arr) { + for (size_t i = 0; i < arr->size; i++) { + free(arr->values[i]); + } + free(arr); + arr = NULL; + return 1; +} + +StrArr_t *str_arr_create() { + StrArr_t *strarr=malloc(sizeof(StrArr_t)); + strarr->size = 0; + strarr->values = NULL; + return strarr; +} + +int str_arr_insert(StrArr_t *arr, const char*v) { + arr->size +=1; + if (arr->values == NULL) { + arr->values = malloc(sizeof(const char *) * (arr->size)); + } else { + arr->values = realloc(arr->values, sizeof(const char *) * (arr->size)); + } + arr->values[arr->size-1] = v; + return 1; +} + +int str_arr_free(StrArr_t *arr) { + for (size_t i = 0; i < arr->size; i++) { + if (arr->values[i] != NULL) + free((char *)arr->values[i]); + } + free(arr->values); + free(arr); + arr = NULL; + return 1; +} + + +KvHashArr_t *kvhash_arr_create() { + KvHashArr_t *kvhash=malloc(sizeof(KvHashArr_t)); + kvhash->size = 0; + kvhash->values = malloc(sizeof(HashTable*)); + return kvhash; +} + +int kvhash_arr_insert(KvHashArr_t *arr, HashTable *v) { + arr->size +=1; + arr->values = realloc(arr->values, sizeof(HashTable*) * (arr->size)); + arr->values[arr->size-1] = v; + return 1; +} + +int kvhash_arr_free(KvHashArr_t *arr) { + for (size_t i = 0; i < arr->size; i++) { + ht_free(arr->values[i]); + } + free(arr); + arr = NULL; + return 1; +} + +int kvstr_free(KvStr_t *str) { + free((char*)str->key); + free((char*)str->value); + free(str); + return 1; +} + +const char * GetArgvString(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + + RedisModuleString *s = RedisModule_CreateString(ctx, "", 0); + int i; + for (i = 0; i < argc; i++) { + size_t arglen; + const char *arg = RedisModule_StringPtrLen(argv[i], &arglen); + + if (i >= 1) RedisModule_StringAppendBuffer(ctx, s, "!", 1); + RedisModule_StringAppendBuffer(ctx, s, arg, arglen); + } + + size_t strLen; + return RedisModule_StringPtrLen(s, &strLen); +} + +char *joinStrArray(RedisModuleCtx *ctx, const char **data, size_t len, char *delim) { + + char *result = RedisModule_Strdup(""); + size_t delimLength = strlen(delim); + size_t strLength = 0; + + for (size_t i = 0; i < len; i++) { + const char *el = data[i]; + if (strlen(el)>0) { + if (i > 0) { + strLength = strlen(result)+delimLength+1; + result = RedisModule_Realloc(result, strLength); + strcat(result, delim); + } + strLength = strlen(result)+strlen(el)+1; + result = RedisModule_Realloc(result, strLength); + strcat(result, el); + } + } + result[strLength -1] = '\0'; + return result; +} + +const char *fsort_read_string_param(RedisModuleString *arg) { + size_t _strSize; + const char * resStr; + const char * str = RedisModule_StringPtrLen(arg,&_strSize); + if (_strSize < 1) { + resStr = NULL; + } else { + resStr = strdup(str); + } + return resStr; +} + +long long fsort_read_long_param(RedisModuleString *str, long long def) { + long long num = 0; + + if (str){ + RedisModule_StringToLongLong(str, &num); + } + + return num != 0 ? num : def; +} + +int fsort_compare_string_desc(const void *a, const void *b) { + const char *aStr = *(const char**)a; + const char *bStr = *(const char**)b; + return strcasecmp(bStr, aStr); +} + +int fsort_compare_string_asc(const void *a, const void *b) { + const char *aStr = *(const char**)a; + const char *bStr = *(const char**)b; + return strcasecmp(aStr, bStr); +} + +int fsort_compare_meta_desc(const void *a, const void *b) { + KvStr_t *mrA = *(KvStr_t**)a; + KvStr_t *mrB = *(KvStr_t**)b; + return strcasecmp(mrA->value, mrB->value); +} + + +int fsort_compare_meta_asc(const void *a, const void *b) { + KvStr_t *mrA = *(KvStr_t**)a; + KvStr_t *mrB = *(KvStr_t**)b; + return strcasecmp(mrB->value, mrA->value); +} + +const char * _encode_json(json_t * obj) { + size_t size = json_dumpb(obj, NULL, 0, 0); + if (size == 0) + return NULL; + + char *buf = malloc(size+1); + + size = json_dumpb(obj, buf, size, 0); + if (size == 0) { + free(buf); + return strdup("{}"); + } + buf[size] = '\0'; + return buf; +} diff --git a/csrc/redis-filtered-sort/fsort_utils.h b/csrc/redis-filtered-sort/fsort_utils.h new file mode 100644 index 0000000..2459fe2 --- /dev/null +++ b/csrc/redis-filtered-sort/fsort_utils.h @@ -0,0 +1,33 @@ +#ifndef __FSORT_UTILS_H +#define __FSORT_UTILS_H + +#include "general.h" +#include "fsort_types.h" + +int fsort_compare_string_desc(const void *a, const void *b); +int fsort_compare_string_asc(const void *a, const void *b); +int fsort_compare_meta_desc(const void *a, const void *b); +int fsort_compare_meta_asc(const void *a, const void *b); +const char * GetArgvString(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); +char *joinStrArray(RedisModuleCtx *ctx, const char **data, size_t len, char *delim); + +const char *fsort_read_string_param(RedisModuleString *arg); +long long fsort_read_long_param(RedisModuleString *str, long long def); + +const char * _encode_json(json_t * obj); + +KvStrArr_t *kvstr_arr_create(); +int kvstr_arr_insert(KvStrArr_t *arr, KvStr_t *v); +int kvstr_arr_free(KvStrArr_t *arr); + +KvHashArr_t *kvhash_arr_create(); +int kvhash_arr_insert(KvHashArr_t *arr, HashTable *v); +int kvhash_arr_free(KvHashArr_t *arr); + +StrArr_t *str_arr_create(); +int str_arr_insert(StrArr_t *arr, const char *v); +int str_arr_free(StrArr_t *arr); + +int kvstr_free(KvStr_t *str); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/general.h b/csrc/redis-filtered-sort/general.h new file mode 100644 index 0000000..8442c8e --- /dev/null +++ b/csrc/redis-filtered-sort/general.h @@ -0,0 +1,17 @@ +#ifndef __GENERAL_H +#define __GENERAL_H + +#include +#include +#include + +#define REDISMODULE_EXPERIMENTAL_API +#define REDIS_MODULE_TARGET 1 + +#include "redismodule.h" +#include "rmutil/alloc.h" +// #include "rmutil/util.h" +// #include "rmutil/strings.h" +#include "rmutil/logging.h" + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/hashtable.c b/csrc/redis-filtered-sort/hashtable.c new file mode 100644 index 0000000..47d8d31 --- /dev/null +++ b/csrc/redis-filtered-sort/hashtable.c @@ -0,0 +1,142 @@ +#include "hashtable.h" +#include "general.h" + +HashTable *ht_create(unsigned int size) { + HashTable *ht; + + if (size < 1) { + return NULL; + } + + ht = malloc(sizeof(HashTable)); + if (ht == NULL) { + return (NULL); + } + + ht->array = (List**)malloc(size * sizeof(List*)); + if (ht->array == NULL) { + return NULL; + } + + memset(ht->array, 0, size * sizeof(List*)); + + ht->size = size; + + return ht; +} + +void ht_free(HashTable *hashtable) { + List *tmp; + unsigned int i; + + if (hashtable == NULL) { + return; + } + + for (i = 0; i < hashtable->size; ++i) { + if (hashtable->array[i] != NULL) { + /* Traverse the list and free the nodes. */ + while(hashtable->array[i] != NULL) { + tmp = hashtable->array[i]->next; + free((char*)hashtable->array[i]->key); + free((char*)hashtable->array[i]->value); + free(hashtable->array[i]); + hashtable->array[i] = tmp; + } + free(hashtable->array[i]); + } + } + free(hashtable->array); + free(hashtable); +} + +const char *ht_get(HashTable *hashtable, const char *key) { + char *key_cp; + unsigned int i; + List *tmp; + + if (hashtable == NULL) { + return NULL; + } + key_cp = strdup(key); + i = hash(key, hashtable->size); + tmp = hashtable->array[i]; + + while (tmp != NULL) { + if (strcmp(tmp->key, key_cp) == 0) { + break; + } + tmp = tmp->next; + } + free(key_cp); + + if (tmp == NULL) { + return NULL; + } + return tmp->value; +} + +int ht_put(HashTable *hashtable, const char *key, const char *value) +{ + List *node; + + if (hashtable == NULL) { + return 1; + } + + node = malloc(sizeof(List)); + if (node == NULL) { + return (1); + } + + node->key = strdup(key); + node->value = strdup(value); + + node_handler(hashtable, node); + + return 0; +} + +void node_handler(HashTable *hashtable, List *node){ + unsigned int i = hash(node->key, hashtable->size); + List *tmp = hashtable->array[i]; + + if (hashtable->array[i] != NULL) { + tmp = hashtable->array[i]; + while (tmp != NULL) { + if (strcmp(tmp->key, node->key) == 0) { + break; + } + tmp = tmp->next; + } + if (tmp == NULL) { + node->next = hashtable->array[i]; + hashtable->array[i] = node; + } else { + free((void*)tmp->value); + tmp->value = node->value; + free((void*)node->value); + free((void*)node->key); + free(node); + } + } else { + node->next = NULL; + hashtable->array[i] = node; + } +} + + +unsigned int hash(const char *key, unsigned int size){ + unsigned int hash; + unsigned int i; + + hash = 0; + i = 0; + while (key && key[i]) + { + hash = (hash + key[i]) % size; + ++i; + } + return (hash); +} + diff --git a/csrc/redis-filtered-sort/hashtable.h b/csrc/redis-filtered-sort/hashtable.h new file mode 100644 index 0000000..5d56263 --- /dev/null +++ b/csrc/redis-filtered-sort/hashtable.h @@ -0,0 +1,34 @@ +#ifndef _HASHTABLE_H_ +#define _HASHTABLE_H_ + +#include +#include +#include + +typedef struct List{ + const char *key; + const char *value; + struct List *next; +} List; + +typedef struct HashTable{ + unsigned int size; + List **array; +} HashTable; + + +unsigned int hash(const char *key, unsigned int size); + + +HashTable *ht_create(unsigned int); +int ht_put(HashTable *, const char *, const char *); +int add_begin_list(List **, List *); +int str_cmp(char *, char *); +void node_handler(HashTable *, List *); +const char * ht_get(HashTable *, const char *); +void ht_free(HashTable *); +void print_str(char *str); +int print_char(char c); +void print_num(int n); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/string_match.c b/csrc/redis-filtered-sort/string_match.c new file mode 100644 index 0000000..e19e894 --- /dev/null +++ b/csrc/redis-filtered-sort/string_match.c @@ -0,0 +1,531 @@ +#include "string_match.h" + +static const char *match (MatchState *ms, const char *s, const char *p) ; + + +MatchRange_t *new_match_range(int start, int end) { + MatchRange_t *mr = malloc(sizeof(MatchRange_t)); + mr->start = start; + mr->end = end; + return mr; +} + +MatchString_t *new_match_string(const char *str, int len) { + MatchString_t *ms = malloc(sizeof(MatchString_t)); + ms->value = strdup(str); + ms->size = len; + return ms; +} + +Match_t *new_match(MatchType_t t, void * match) { + Match_t *mt = malloc(sizeof(Match_t)); + mt->type = t; + mt->value = match; + return mt; +} + +int add_match(MatchResult_t *mr, Match_t *m ) { + mr->size ++; + mr->data = realloc(mr->data, sizeof(Match_t*) * mr->size); + mr->data[mr->size-1] = m; + return 0; +} + +MatchResult_t *new_match_result() { + MatchResult_t *mr = malloc(sizeof(MatchResult_t)); + mr->size= 0; + mr->data = malloc(sizeof(Match_t*)); + return mr; +} + +int free_match(Match_t *m) { + if (m->type == MATCH_STRING) { + MatchString_t *msv = match_string_value(m); + free((void *)msv->value); + } + free(m->value); + free(m); + m = NULL; + return 0; +} + +int free_match_result(MatchResult_t *mr) { + + for (size_t i = 0; i < mr->size; i++){ + free_match(mr->data[i]); + } + + for (size_t i = mr->error_count; i >0; i--){ + free((void *)mr->error[i-1]); + } + + free(mr->data); + free(mr->error); + free(mr); + + mr = NULL; + return 0; +} + +int free_match_state(MatchState *ms) { + free(ms); + return 0; +} + +MatchRange_t *match_range_value(Match_t * m) { + MatchRange_t *mr = (MatchRange_t *) m->value; + return mr; +} + +MatchString_t *match_string_value(Match_t *m) { + MatchString_t *ms = (MatchString_t*)m->value; + return ms; +} + +static int seterror(MatchState *ms, const char * fmt, ...) { + + ms->error_count ++; + if (ms->error == NULL) { + ms->error = malloc(sizeof(char*)); + } else { + ms->error = realloc(ms->error, sizeof(char*) * ms->error_count); + } + + va_list argp; + va_start(argp, fmt); + + ms->error[ms->error_count -1 ] = malloc(sizeof(char) * 512); + vsprintf((char *)ms->error[ms->error_count -1], fmt, argp); + + va_end(argp); + + return 1; +} + +static int reprepstate (MatchState *ms) { + ms->level = 0; + //Was assert check. But we don't want process to panic + if (ms->matchdepth != MAXCCALLS) return -1; + return 0; +} + +static void prepstate (MatchState *ms, const char *s, size_t ls, const char *p, size_t lp) { + ms->matchdepth = MAXCCALLS; + ms->src_init = s; + ms->src_end = s + ls; + ms->p_end = p + lp; + ms->error = NULL; + ms->error_count = 0; + ms->result =new_match_result(); +} + +static const char *classend (MatchState *ms, const char *p) { + switch (*p++) { + case ESC_CHAR: { + if (p == ms->p_end) + seterror(ms, "malformed pattern (ends with '%%')"); + return p+1; + } + case '[': { + if (*p == '^') p++; + do { // look for a ']' + if (p == ms->p_end) { + seterror(ms, "malformed pattern (missing ']')"); + return NULL; + } + + if (*(p++) == ESC_CHAR && p < ms->p_end) + p++; // skip escapes (e.g. '%]') + } while (*p != ']'); + return p+1; + } + default: { + return p; + } + } +} + +static int match_class (int c, int cl) { + int res; + switch (tolower(cl)) { + case 'a' : res = isalpha(c); break; + case 'c' : res = iscntrl(c); break; + case 'd' : res = isdigit(c); break; + case 'g' : res = isgraph(c); break; + case 'l' : res = islower(c); break; + case 'p' : res = ispunct(c); break; + case 's' : res = isspace(c); break; + case 'u' : res = isupper(c); break; + case 'w' : res = isalnum(c); break; + case 'x' : res = isxdigit(c); break; + case 'z' : res = (c == 0); break; // deprecated option + default: return (cl == c); + } + return (islower(cl) ? res : !res); +} + +static int matchbracketclass (int c, const char *p, const char *ec) { + int sig = 1; + if (*(p+1) == '^') { + sig = 0; + p++; // skip the '^' + } + while (++p < ec) { + if (*p == ESC_CHAR) { + p++; + if (match_class(c, uchar(*p))) + return sig; + } + else if ((*(p+1) == '-') && (p+2 < ec)) { + p+=2; + if (uchar(*(p-2)) <= c && c <= uchar(*p)) + return sig; + } + else if (uchar(*p) == c) return sig; + } + return !sig; +} + + +static int singlematch (MatchState *ms, const char *s, const char *p, const char *ep) { + if (s >= ms->src_end) + return 0; + else { + int c = uchar(*s); + switch (*p) { + case '.': return 1; // matches any char + case ESC_CHAR: return match_class(c, uchar(*(p+1))); + case '[': return matchbracketclass(c, p, ep-1); + default: return (uchar(*p) == c); + } + } +} + + +static const char *max_expand (MatchState *ms, const char *s, const char *p, const char *ep) { + ptrdiff_t i = 0; // counts maximum expand for item + while (singlematch(ms, s + i, p, ep)) + i++; + //keeps trying to match with the maximum repetitions + while (i>=0) { + const char *res = match(ms, (s+i), ep+1); + if (res) return res; + i--; // else didn't match; reduce 1 repetition to try again + } + return NULL; +} + + +static const char *min_expand (MatchState *ms, const char *s, const char *p, const char *ep) { + for (;;) { + const char *res = match(ms, s, ep+1); + if (res != NULL) + return res; + else if (singlematch(ms, s, p, ep)) + s++; // try with one more repetition + else return NULL; + } +} + +static const char *matchbalance (MatchState *ms, const char *s, const char *p) { + if (p >= ms->p_end - 1) seterror(ms, "malformed pattern (missing arguments to '%%b')"); + if (*s != *p) return NULL; + else { + int b = *p; + int e = *(p+1); + int cont = 1; + while (++s < ms->src_end) { + if (*s == e) { + if (--cont == 0) return s+1; + } + else if (*s == b) cont++; + } + } + return NULL; // string ends out of balance +} + +static int check_capture (MatchState *ms, int l) { + l -= '1'; + if (l < 0 || l >= ms->level || ms->capture[l].len == CAP_UNFINISHED) + return seterror(ms, "invalid capture index %%%d", l + 1); + + return l; +} + +static int capture_to_close (MatchState *ms) { + int level = ms->level; + for (level--; level>=0; level--) + if (ms->capture[level].len == CAP_UNFINISHED) return level; + return seterror(ms, "invalid pattern capture"); +} + +static void push_onecapture (MatchState *ms, int i, const char *s, const char *e) { + if (i >= ms->level) { + if (i == 0) { + + int mstr_size = strlen(s)-strlen(e) ; + if (mstr_size > 0) { + char mstrcp[mstr_size+1]; + strncpy(mstrcp, s, mstr_size); + mstrcp[mstr_size] = '\0'; + MatchString_t *mstr = new_match_string(mstrcp, mstr_size); + + Match_t *m = new_match(MATCH_STRING, mstr); + add_match(ms->result, m); + } + + } else + seterror(ms, "invalid capture index %%%d\n", i + 1); + } + else { + ptrdiff_t l = ms->capture[i].len; + if (l == CAP_UNFINISHED) { + seterror(ms, "unfinished capture"); + return; + } + if (l == CAP_POSITION) { + MatchRange_t *mran = new_match_range(ms->capture[i].init - ms->src_init +1, 0); + Match_t *m = new_match(MATCH_RANGE, mran); + add_match(ms->result, m); + } else { + int mstr_size = ms->capture[i].len ; + char mstrcp[mstr_size+1]; + strncpy(mstrcp, ms->capture[i].init, mstr_size); + mstrcp[mstr_size] = '\0'; + + MatchString_t *mstr = new_match_string(mstrcp,l); + Match_t *m = new_match(MATCH_STRING, mstr); + add_match(ms->result, m); + } + } +} + + +static int push_captures (MatchState *ms, const char *s, const char *e) { + int i; + int nlevels = (ms->level == 0 && s) ? 1 : ms->level; + for (i = 0; i < nlevels; i++) { + push_onecapture(ms, i, s, e); + } + + return nlevels; // number of strings pushed +} + +static const char *start_capture (MatchState *ms, const char *s, const char *p, int what) { + const char *res; + int level = ms->level; + if (level >= MAXCAPTURES) seterror(ms, "too many captures"); + ms->capture[level].init = s; + ms->capture[level].len = what; + ms->level = level+1; + if ((res=match(ms, s, p)) == NULL) // match failed? + ms->level--; // undo capture + return res; +} + + +static const char *end_capture (MatchState *ms, const char *s, const char *p) { + int l = capture_to_close(ms); + const char *res; + ms->capture[l].len = s - ms->capture[l].init; // close capture + if ((res = match(ms, s, p)) == NULL) // match failed? + ms->capture[l].len = CAP_UNFINISHED; // undo capture + return res; +} + + +static const char *match_capture (MatchState *ms, const char *s, int l) { + size_t len; + l = check_capture(ms, l); + len = ms->capture[l].len; + if ((size_t)(ms->src_end-s) >= len && memcmp(ms->capture[l].init, s, len) == 0) { + return s+len; + } + else{ + return NULL; + } +} + + + +static const char *match (MatchState *ms, const char *s, const char *p) { + if (ms->matchdepth-- == 0) { + seterror(ms, "pattern too complex"); + return s; + } + + init: + if (p != ms->p_end) { + switch (*p) { + case '(': { // start capture + if (*(p + 1) == ')') // position capture? + s = start_capture(ms, s, p + 2, CAP_POSITION); + else + s = start_capture(ms, s, p + 1, CAP_UNFINISHED); + break; + } + case ')': { // end capture + s = end_capture(ms, s, p + 1); + break; + } + case '$': { + if ((p + 1) != ms->p_end) // is the `$' the last char in pattern? + goto dflt; // no; go to default + s = (s == ms->src_end) ? s : NULL; // check end of string + break; + } + case ESC_CHAR: { // escaped sequences not in the format class[*+?-]? + switch (*(p + 1)) { + case 'b': { // balanced string? + s = matchbalance(ms, s, p + 2); + if (s != NULL) { + p += 4; goto init; // return match(ms, s, p + 4); + } + break; + } + case 'f': { // frontier? + const char *ep; char previous; + p += 2; + if (*p != '[') seterror(ms, "missing '[' after '%%f' in pattern"); + + ep = classend(ms, p); // points to what is next + if (ep == NULL) { + s = NULL; + break; + } + previous = (s == ms->src_init) ? '\0' : *(s - 1); + if (!matchbracketclass(uchar(previous), p, ep - 1) && + matchbracketclass(uchar(*s), p, ep - 1)) { + p = ep; goto init; // return match(ms, s, ep); + } + s = NULL; // match failed + break; + } + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + case '8': case '9': { // capture results (%0-%9)? + s = match_capture(ms, s, uchar(*(p + 1))); + if (s != NULL) { + p += 2; goto init; // return match(ms, s, p + 2) + } + break; + } + default: goto dflt; + } + break; + } + default: dflt: { // pattern class plus optional suffix + const char *ep = classend(ms, p); // points to optional suffix + if (ep == NULL) { + //ms->matchdepth ++; + return NULL; + } + if (!singlematch(ms, s, p, ep)) { + if (*ep == '*' || *ep == '?' || *ep == '-') { // accept empty? + p = ep + 1; goto init; + } + else + s = NULL; + } + else { // matched once */ + switch (*ep) { // handle optional suffix + case '?': { + const char *res; + if ((res = match(ms, s + 1, ep + 1)) != NULL) + s = res; + else { + p = ep + 1; goto init; + } + break; + } + case '+': + s++; + + case '*': // 0 or more + s = max_expand(ms, s, p, ep); + break; + case '-': // 0 or more min + s = min_expand(ms, s, p, ep); + break; + default: // no suffix + s++; p = ep; goto init; + } + } + break; + } + } + } + ms->matchdepth++; + return s; +} + +static int str_match_aux (MatchState *ms, const char *s, const char *p, int find) { + size_t ls = strlen(s), lp = strlen(p); + int init = 1; + + + const char *s1 = s + init -1; + int anchor = (*p == '^'); + if (anchor) { + p++; lp--; // skip "^" anchor character + } + prepstate(ms, s, ls, p, lp); + do { + const char *res; + if (reprepstate(ms) == -1) { + return 0; //seterror(ms, "too much calls"); + } + if ((res=match(ms, s1, p)) != NULL) { + if (find) { + int start = (s1 - s) + 1; + int stop = (s1 -s ) +lp; + MatchRange_t *mran = new_match_range(start, stop); + Match_t *m = new_match(MATCH_RANGE, mran); + add_match(ms->result, m); + int pushres = push_captures(ms, NULL, 0) + 2; + return pushres; + } + else { + int pushres = push_captures(ms, s1, res); + return pushres; + } + } + } while (s1++ < ms->src_end && !anchor); + + return 0; +} + +MatchResult_t *str_match(const char *s, const char *p) { + MatchState *ms = malloc(sizeof(MatchState)); + str_match_aux(ms, s,p,0); + + MatchResult_t *mr = ms->result; + mr->error_count = ms->error_count; + mr->error = ms->error; + free_match_state(ms); + return mr; +} + +int print_result(MatchResult_t * mr) { + printf("MatchCount: %ld\n", mr->size); + if (mr->error_count > 0) { + for (size_t i = 0; i < mr->error_count; i++) + { + printf("Got %d errors: %s\n", mr->error_count, mr->error[i]); + } + } + for (size_t i = 0; i < mr->size; i++) { + if (mr->data[i]->type == MATCH_STRING) { + MatchString_t *mstr = match_string_value(mr->data[i]); + printf("...Match type %d value '%s'\n", mstr->size, mstr->value); + } + if (mr->data[i]->type == MATCH_RANGE) { + MatchRange_t *mrang = match_range_value(mr->data[i]); + printf("...Match type range %d value %d %d\n",MATCH_RANGE, mrang->start, mrang->end ); + } + } + return 0; +} + + + diff --git a/csrc/redis-filtered-sort/string_match.h b/csrc/redis-filtered-sort/string_match.h new file mode 100644 index 0000000..f2fe985 --- /dev/null +++ b/csrc/redis-filtered-sort/string_match.h @@ -0,0 +1,80 @@ +#ifndef __STRINGMATCH_H +#define __STRINGMATCH_H +#include "general.h" +//emulates lua pattern search +// #include +// #include +// #include +// #include +#include +// #include + +//ptrdiff_t +#include +#include + +#define MAXCAPTURES 32 +#define CAP_POSITION 2 +#define CAP_UNFINISHED -1 +#define ESC_CHAR '%' +#define SPECIALS "^$*+?.([%-" +/* macro to 'unsign' a character */ +#define uchar(c) ((unsigned char)(c)) + +#if !defined(MAXCCALLS) +#define MAXCCALLS 200 +#endif + +typedef enum MatchType { + MATCH_STRING = 101, + MATCH_RANGE +} MatchType_t; + +typedef struct Match { + MatchType_t type; + void * value; +} Match_t; + +typedef struct MatchRange { + int start; + int end; +} MatchRange_t; + +typedef struct MatchString { + const char * value; + int size; +} MatchString_t; + +typedef struct MatchResult { + size_t size; + Match_t **data; + int error_count; + const char **error; + +} MatchResult_t; + +typedef struct MatchState { + int matchdepth; // control for recursive depth (to avoid C stack overflow) + const char *src_init; // init of source string + const char *src_end; // end ('\0') of source string + const char *p_end; // end ('\0') of pattern + + const char **error; + int error_count; + MatchResult_t *result; + + int level; // total number of captures (finished or unfinished) + struct { + const char *init; + ptrdiff_t len; + } capture[MAXCAPTURES]; +} MatchState; + + +int free_match_result(MatchResult_t *mr); +MatchString_t *match_string_value(Match_t *m); +MatchRange_t *match_range_value(Match_t * m); + +MatchResult_t *str_match(const char *s, const char *p); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/thread_pool.c b/csrc/redis-filtered-sort/thread_pool.c new file mode 100644 index 0000000..dad6b18 --- /dev/null +++ b/csrc/redis-filtered-sort/thread_pool.c @@ -0,0 +1,177 @@ +#include "thread_pool.h" +#include + +static FSortWork_t *tpool_work_create(thread_func_t func, void *arg) { + FSortWork_t *work; + + if (func == NULL) + return NULL; + + work = malloc(sizeof(FSortWork_t)); + work->func = func; + work->argv = arg; + work->next = NULL; + return work; +} + +void tpool_work_destroy(FSortWork_t *work) { + if (work == NULL) + return; + free(work); +} + +static FSortWork_t *tpool_work_get(FSortPool_t *tp) { + FSortWork_t *work; + + if (tp == NULL) + return NULL; + + work = tp->work_first; + if (work == NULL) + return NULL; + + if (work->next == NULL) { + tp->work_first = NULL; + tp->work_last = NULL; + } else { + tp->work_first = work->next; + } + + return work; +} + +static void *tpool_worker(void *arg) { + FSortPool_t *tp = arg; + FSortWork_t *work; + + while (1) { + pthread_mutex_lock(&(tp->work_mutex)); + if (tp->stop) { + break; + } + + if (tp->work_first == NULL) { + pthread_cond_wait(&(tp->work_cond), &(tp->work_mutex)); + } + + work = tpool_work_get(tp); + tp->working_cnt++; + pthread_mutex_unlock(&(tp->work_mutex)); + if (work != NULL) { + work->func(work->argv); + if (tp->cb != NULL) { + tp->cb(work); + } + tpool_work_destroy(work); + } + pthread_mutex_lock(&(tp->work_mutex)); + tp->working_cnt--; + + if (!tp->stop && tp->working_cnt == 0 && tp->work_first == NULL){ + pthread_cond_signal(&(tp->working_cond)); + } + + pthread_mutex_unlock(&(tp->work_mutex)); + } + + tp->thread_cnt--; + pthread_cond_signal(&(tp->working_cond)); + pthread_mutex_unlock(&(tp->work_mutex)); + return NULL; +} + +FSortPool_t *tpool_create(size_t num) { + FSortPool_t *tp; + pthread_t thread; + size_t i; + + if (num == 0) + num = 2; + + tp = malloc(sizeof(FSortPool_t)); + tp->thread_cnt = num; + tp->stop = false; + tp->cb = NULL; + + pthread_mutex_init(&(tp->work_mutex), NULL); + pthread_cond_init(&(tp->work_cond), NULL); + pthread_cond_init(&(tp->working_cond), NULL); + + tp->work_first = NULL; + tp->work_last = NULL; + + for (i=0; iwork_mutex)); + work = tp->work_first; + while (work != NULL) { + work2 = work->next; + tpool_work_destroy(work); + work = work2; + } + tp->stop = true; + pthread_cond_broadcast(&(tp->work_cond)); + pthread_mutex_unlock(&(tp->work_mutex)); + + tpool_wait(tp); + + pthread_mutex_destroy(&(tp->work_mutex)); + pthread_cond_destroy(&(tp->work_cond)); + pthread_cond_destroy(&(tp->working_cond)); + + free(tp); +} + +bool tpool_add_work(FSortPool_t *tp, thread_func_t func, void *arg) { + FSortWork_t *work; + + if (tp == NULL) + return false; + + work = tpool_work_create(func, arg); + if (work == NULL) + return false; + + pthread_mutex_lock(&(tp->work_mutex)); + + if (tp->work_first == NULL) { + tp->work_first = work; + tp->work_last = tp->work_first; + } else { + tp->work_last->next = work; + tp->work_last = work; + } + + pthread_cond_broadcast(&(tp->work_cond)); + pthread_mutex_unlock(&(tp->work_mutex)); + + return true; +} + +void tpool_wait(FSortPool_t *tp) { + if (tp == NULL) + return; + + pthread_mutex_lock(&(tp->work_mutex)); + while (1) { + if ((!tp->stop && tp->working_cnt != 0) || (tp->stop && tp->thread_cnt != 0)) { + pthread_cond_wait(&(tp->working_cond), &(tp->work_mutex)); + } else { + break; + } + } + pthread_mutex_unlock(&(tp->work_mutex)); +} \ No newline at end of file diff --git a/csrc/redis-filtered-sort/thread_pool.h b/csrc/redis-filtered-sort/thread_pool.h new file mode 100644 index 0000000..e92154a --- /dev/null +++ b/csrc/redis-filtered-sort/thread_pool.h @@ -0,0 +1,37 @@ +#ifndef __THREAD_POOL_H +#define __THREAD_POOL_H + +#include "general.h" +#include +#include + +typedef void *(*thread_func_t)(void *arg); + +struct FSortWork { + thread_func_t func; + void *argv; + struct FSortWork *next; +}; + +typedef struct FSortWork FSortWork_t; + +typedef int (*tpool_finish_cb_t)(FSortWork_t *work); + +typedef struct FSortPool { + FSortWork_t *work_first; + FSortWork_t *work_last; + pthread_mutex_t work_mutex; + pthread_cond_t working_cond; + size_t working_cnt; + size_t thread_cnt; + tpool_finish_cb_t cb; + bool stop; + pthread_cond_t work_cond; +} FSortPool_t; + +FSortPool_t *tpool_create(size_t num); +bool tpool_add_work(FSortPool_t *tm, thread_func_t func, void *arg); +void tpool_wait(FSortPool_t *tm); +void tpool_work_destroy(FSortWork_t *work); + +#endif \ No newline at end of file diff --git a/csrc/redis-filtered-sort/utils.c b/csrc/redis-filtered-sort/utils.c new file mode 100644 index 0000000..f6472bd --- /dev/null +++ b/csrc/redis-filtered-sort/utils.c @@ -0,0 +1,83 @@ +#include "utils.h" + +char * strlwr(const char *str) { + char *lcStr = (char *)str; + for (size_t i = 0; i < strlen(str); i++) { + lcStr[i] = tolower(str[i]); + } + return lcStr; +} + +char *repl_str(const char *str, const char *from, const char *to) { + size_t cache_sz_inc = 16; + const size_t cache_sz_inc_factor = 3; + const size_t cache_sz_inc_max = 1048576; + + char *pret, *ret = NULL; + const char *pstr2, *pstr = str; + size_t i, count = 0; + + uintptr_t *pos_cache_tmp, *pos_cache = NULL; + + size_t cache_sz = 0; + size_t cpylen, orglen, retlen, tolen, fromlen = strlen(from); + + /* Find all matches and cache their positions. */ + while ((pstr2 = strstr(pstr, from)) != NULL) { + count++; + + /* Increase the cache size when necessary. */ + if (cache_sz < count) { + cache_sz += cache_sz_inc; + pos_cache_tmp = RedisModule_Realloc(pos_cache, sizeof(*pos_cache) * cache_sz); + if (pos_cache_tmp == NULL) { + goto end_repl_str; + } else pos_cache = pos_cache_tmp; + cache_sz_inc *= cache_sz_inc_factor; + if (cache_sz_inc > cache_sz_inc_max) { + cache_sz_inc = cache_sz_inc_max; + } + } + + pos_cache[count-1] = pstr2 - str; + pstr = pstr2 + fromlen; + } + + orglen = pstr - str + strlen(pstr); + + /* Allocate memory for the post-replacement string. */ + if (count > 0) { + tolen = strlen(to); + retlen = orglen + (tolen - fromlen) * count; + } else retlen = orglen; + ret = (char *)RedisModule_Alloc(retlen + 1); + if (ret == NULL) { + goto end_repl_str; + } + + if (count == 0) { + /* If no matches, then just duplicate the string. */ + strcpy(ret, str); + } else { + /* Otherwise, duplicate the string whilst performing + * the replacements using the position cache. */ + pret = ret; + memcpy(pret, str, pos_cache[0]); + pret += pos_cache[0]; + for (i = 0; i < count; i++) { + memcpy(pret, to, tolen); + pret += tolen; + pstr = str + pos_cache[i] + fromlen; + cpylen = (i == count-1 ? orglen : pos_cache[i+1]) - pos_cache[i] - fromlen; + memcpy(pret, pstr, cpylen); + pret += cpylen; + } + ret[retlen] = '\0'; + } + +end_repl_str: + /* Free the cache and return the post-replacement string, + * which will be NULL in the event of an error. */ + RedisModule_Free(pos_cache); + return ret; +} diff --git a/csrc/redis-filtered-sort/utils.h b/csrc/redis-filtered-sort/utils.h new file mode 100644 index 0000000..efb37ba --- /dev/null +++ b/csrc/redis-filtered-sort/utils.h @@ -0,0 +1,44 @@ +#ifndef __UTIL_H +#define __UTIL_H 1 + +#include +#include +#include + +#include "redismodule.h" + +#define array(type) \ + struct { \ + type* data; \ + size_t length; \ + } + +#define array_init() \ + { \ + .data = NULL, \ + .length = 0 \ + }; + +#define array_free(array) \ + do { \ + for (size_t i=0; i < array.length; i ++) { \ + RedisModule_Free((char*)array.data[i]); \ + } \ + RedisModule_Free(array.data); \ + array.data = NULL; \ + array.length = 0; \ + } while (0) + +#define array_push(array, element) \ + do { \ + array.data = RedisModule_Realloc(array.data,sizeof(*array.data) * (array.length + 1)); \ + array.data[array.length] = strdup(element); \ + array.length++; \ + } while (0) + + +char * strlwr(const char *str); +char *repl_str(const char *str, const char *from, const char *to); + + +#endif \ No newline at end of file diff --git a/csrc/variables.include b/csrc/variables.include new file mode 100644 index 0000000..b9e0a6d --- /dev/null +++ b/csrc/variables.include @@ -0,0 +1,31 @@ +DOCKER_IMAGE="makeomatic/redis_filter_mod" + +JANSSON_VERSION=2.12 +JANSSON_TAR_NAME=jansson-$(JANSSON_VERSION).tar.gz +JANSSON_LINK=http://www.digip.org/jansson/releases/$(JANSSON_TAR_NAME) +JANSSON_LIBDIR=lib/jansson + +# find the OS +uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +# Compile flags for linux / osx +ifeq ($(uname_S),Linux) + SHOBJ_CFLAGS ?= -fno-common -g -ggdb + SHOBJ_LDFLAGS ?= -shared -Bsymbolic +else + SHOBJ_CFLAGS ?= -dynamic -fno-common -g -ggdb + SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup +endif + +# Setting the DEBUG env variable to 1 will cause us to build with -O0 +ifndef DEBUG + DEBUG = 0 +endif +DEBUGFLAGS = -g -ggdb -O2 +ifeq ($(DEBUG), 1) + DEBUGFLAGS = -g -ggdb -O0 +endif + +CFLAGS = -I$(RM_INCLUDE_DIR) -I$(JANSSON_LIBDIR)/src -Wall $(DEBUGFLAGS) -fPIC -lc -lm -std=gnu99 +CC=gcc + diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..1903aa6 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,10 @@ +# RedisFilterModule + +Module basically replicates `http://redis.io/commands/sort` but with extra features and ability to run it in clustered mode with +hashed keys, which resolve to the same slot. Works in separate threads, so Redis db stays unblocked. + +* See [HOWTO](./build.md) build module. +* API provided by module [API](./api.md) + +## Load options +Module use 4 threads in it's own processing pool. Thread pool size can be changed in `redis.conf`, eg `loadmodule filter_module.so {POOL_SIZE}` or in Redis args `redis-server --loadmodule {filter_module.so} {POOL_SIZE}` diff --git a/doc/api.md b/doc/api.md new file mode 100644 index 0000000..5004ed8 --- /dev/null +++ b/doc/api.md @@ -0,0 +1,48 @@ +# FilterSortModule redis commands +Module exports this Redis commands + +**Please use `json Objects` instead of `json Arrays`** + +## Redis Cluster NOTE +In cluster environment all keys used by this command must have same *fix with `{}` in key names + +## fsort +Sorts, filters and paginates provided SET `idSetKey` using data from `metaKeyPattern`.`hashKey` using `sortOrder`,`filter` and stores processed data for `expire` seconds + +**Usage**: +```console +127.0.0.1:6379:> cfsort idSetKey metaKeyPattern hashKey sortOrder filter(jsonFormattedString) curTime [offset:int] [limit:int] [expire] [keyOnly] +``` +- **idSetKey**: Redis SET containing id values +- **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce _id-231-metakey_ +- **hashKey**: field name from HASHkey formed from `metakeyPattern` +- **filter**: Json string with filters +- **curTime**: Current time in milliseconds +- **expire** : TTL for generated cacheds in milliseconds(default 30 sec) +- **keyOnly**: Forces command to return Redis `key`, containing result data. Usable for [`fsortBust`](#sortbust) method + + +## fsortBust : +Checking caches for specified `key` that was created in result of [fsort](#fsort) method. Allows forcing cache invalidation. + +**Method usage**: +``` +fsortbust key time [expire] +``` +- **key**: Redis LIST containing cached sorted or filtered values. You can obtain it by passing `keyOnly` parameter into [fsort](#fsort) command +- **time**: Current or desired time in milliseconds +- **expire** : TTL in milliseconds(default 30 sec) + + +## fsortAggregate: +Groups elements by `aggregateParam` from set `idSet` using meta available under `metaKeyPattern` and reports count of each type of criteria. + +Currently supports only `sum`; + +* **Method usage**: +``` +cfsortAggregate idSetKey metaKeyPattern aggregateParam` +``` +- **idSetKey**: Redis SET containing id values +- **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce _id-231-metakey_ +- **aggregateParam** : JsonString containing aggregate criteria. diff --git a/doc/build.md b/doc/build.md new file mode 100644 index 0000000..988b152 --- /dev/null +++ b/doc/build.md @@ -0,0 +1,42 @@ +# Build Redis FilterSortModule +Module works with `redis` v4.0 or higher. +## General Requirements +Cloned module [redis-filtered-sort](https://github.com/makeomatic/redis-filtered-sort) repository +```console +$~: git clone https://github.com/makeomatic/redis-filtered-sort.git +``` + +### Local build dependencies +* For local build you will need general C compiler and linker. +Generally on Ubuntu or other Debian like distros, all you have to do is install `build-essential` meta-package + ```console + $~: sudo apt-get install build-essential + ``` + +### Docker based build +For docker based build, You will need `docker` and `docker-compose` installed. This variant able generate special docker image with module included, or compile module under `alpine` linux. + +## Local build +Enter `csrc` folder, located inside cloned repository. +Call: +```console +$~: make deps && make +``` +Compiled module will be located `csrc/redis-filtered-sort/filter_module.so`. Copy this file into desired location and add `loadmodule {pathto filter_module.so}` option into your `redis.conf`. + +## Docker build +Enter `csrc` folder, located inside cloned repository. +### Compile module +Set desired image name in `csrc\Makefile` and run +``` +$~: make docker-build +``` +If everything goes ok, You'll see **$DOCKER_IMAGE** image in your local *docker images* + +If you want different name for image use `make docker-build IMAGE_NAME=foo/name:tag` + +Push your image to docker with command + +``` +$~: make docker-push +``` diff --git a/doc/rfc/00_design.md b/doc/rfc/00_design.md new file mode 100644 index 0000000..ac61041 --- /dev/null +++ b/doc/rfc/00_design.md @@ -0,0 +1,117 @@ +# Proposal +As described in Redis Module API modules have more benefints than general LUA scripts. In current environment, lua script locking db for a long period. The main reason of this module: provide non db-locking access for time-consuming operations. + +# Design +* Projects consists from one redis module `filtered_sort` and node.js wrapper `redis-filtered-sort`. + +* By default, module going to have 2 build options: Docker based build for redis:5.0-alpine and local system `gcc` + +* Exported functions for dev purpose prefixed with "c" prefix + +* Current module behaviour copied from existing scripts for compat with lua. + +* in Redis Cluster environment: all keys and sets MUST use {prefix} notation, to be stored on same node. Otherwise module won't work correctly. + + +## Expected API and behaviour +Module will export into Redis command namespace these commands: +#### cfsort +Sorts, filters and paginates provided SET `idSetKey` using data from `metaKeyPattern`.`hashKey` using `sortOrder`,`filter` and stores processed data for `expire` seconds + +* **Method usage**: `cfsort idSetKey metaKeyPattern hashKey sortOrder filter(jsonFormattedString) curTime:unixtime [offset:int] [limit:int] [expire:int - ttl in seconds] [keyOnly]`; + - **idSetKey**: Redis SET containing id values + - **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce _id-231-metakey_ + - **hashKey**: field name from HASHkey formed from `metakeyPattern` + - **filter**: Json string with filters +* **General behaviour**: During work process function going to store postprocessed data into 2 temporary sets(sorted and filterd) in format `{idSetKey}:{order}:{metaKey}:{hasKey}:{filterString}` and reuse them while `ttl` not past. + + Creates index of this sets in ZSET `tempKeysSet`. + > _Definitely I don't understand why this needed, but we have cacheBuster_ in lua. #SeeLater + + - If `metaKeyPattern` and `hashKey` not provided, function going to use `idSetKey` values for sorting, otherwise fetch data from `metaKeyPattern.gsub("*", idSetMemberValue).providedHashKey` and use them for sort. + + - Same as prev behaviour going to be used in filtering process. + + - If more than one filter provided: `filterString` will contain only "#" filter; + + +### cfsortBust : +Checking caches for specified `idSetKey` that was created in result of `cfsort` method work. Deletes outdated tempKeys from ZSET `tempKeysSet` + +**Method usage**: `cfsort idSetKey curTime:unixtime [expire:int]` + +### cfsortAggregate: +Groups elements by `aggregateParam` from set `idSet` using meta available under `metaKeyPattern` and reports count of each type of criteria. + +Currently supports only `sum`; + +* **Method usage**: `cfsortAggregate idSetKey metaKeyPattern aggregateParam` + - **idSetKey**: Redis SET containing id values + - **metaKeyPattern**: Pattern to find HASHes of record, "*" is replaced by id from `idSetKey`, eg: _*-metakey_ produce _id-231-metakey_ + - **aggregateParam** : JsonString containing aggregate criteria. + ```Javascript + { + "fieldName": "sum" + } + ``` + + +## Filter Format `filter` parameter: +Accepting `json` formated object: +```Javascript + { + "fieldToFiler": { + //Filter params... + } + "fieldToFilter": "String to find" + } +``` + +- **Available criteria**: + - `lte`,`gte` - (int) value comparison + - `eq`,`ne` - equal or not + - `any` - Object containing criteria list to Exclude + - `some` - Object containing criteria list to Include + - `match` - String contains + - `exists` - if value exists + +- **MultiField(#multi)**: Reserved field name for assigning criterias to multiple fields from `meta` record(currently supports only _String contains check_) + ```Javascript + { + "#multi": { + "fields": [ + //Field list + ... + ], + "match": "criteria" + } + } + ``` + + +**Example** +```Javascript + { + "metaFieldName": "contained string", + "metaFieldName2": { //Filter params object + "gte": 12, + "lte": 13, + }, + "metaFieldAnyParams": { + "any": { + "0": { + "gte": 12, + "lte": 13, + }, + "1": { + "gte": 28, + "lte": 30, + } + } + }, + + + } + ``` + + diff --git a/doc/rfc/02_threading_algo.md b/doc/rfc/02_threading_algo.md new file mode 100644 index 0000000..62675d1 --- /dev/null +++ b/doc/rfc/02_threading_algo.md @@ -0,0 +1,18 @@ +# Multithreading + +OnLoad module allocates pool of workers, default 4 or depends on passed parameter; + +When request arrives, module assigns work into pool queue; + +All tasks processed in FIFO mode. + +**TODO** Find Why RedisModueApi Not allows Block client in transaction scope. Throws segfault from Redis code. + +## FsortAggregate +Client paused and Job assigned into worker pool queue + +## Fsort +Client paused and Job assigned into worker pool queue +When result data being saved, we're checking whether same key already been created. If so, not writing data into result list and continue work. + +**TODO** Think we need add some db level locks. To avoid overprocessing data for sorts and filters. E.g. if one task already started processing command, threads with same command should wait for result. diff --git a/filtered-list-bust.lua b/filtered-list-bust.lua deleted file mode 100644 index d452a9b..0000000 --- a/filtered-list-bust.lua +++ /dev/null @@ -1,15 +0,0 @@ --- must be a set of ids -local idSet = KEYS[1]; --- current time -local curTime = ARGV[1]; --- caching time -local expiration = tonumber(ARGV[2] or 30000); - --- -local tempKeysSet = getIndexTempKeys(idSet); -local keys = redis.call("ZRANGEBYSCORE", tempKeysSet, curTime - expiration, '+inf'); - -if #keys > 0 then - redis.call("DEL", unpack(keys)); - redis.call("DEL", tempKeysSet); -end diff --git a/groupped-list.lua b/groupped-list.lua deleted file mode 100644 index 5af3ee8..0000000 --- a/groupped-list.lua +++ /dev/null @@ -1,79 +0,0 @@ --- cached id list key -local idListKey = KEYS[1]; --- meta key -local metadataKey = KEYS[2]; --- stringified [key]: [aggregateMethod] pairs -local aggregates = ARGV[1]; - --- local cache -local rcall = redis.call; -local tinsert = table.insert; - -local jsonAggregates = cjson.decode(aggregates); -local aggregateKeys = {}; -local result = {}; - -local function try(what) - local status, result = pcall(what[1]); - if not status then - return what[2](result); - end - - return result; -end - -local function catch(what) - return what[1] -end - -local function anynumber(a) - return try { - function() - local num = tonumber(a); - return num ~= nil and num or tonumber(cjson.decode(a)); - end, - - catch { - function() - return nil; - end - } - } -end - -local function aggregateSum(value1, value2) - return value1 + value2; -end - -local aggregateType = { - sum = aggregateSum -}; - -for key, method in pairs(jsonAggregates) do - tinsert(aggregateKeys, key); - result[key] = 0; - - if type(aggregateType[method]) ~= "function" then - return error("not supported op: " .. method); - end -end - -local valuesToGroup = rcall("LRANGE", idListKey, 0, -1); - --- group -for _, id in ipairs(valuesToGroup) do - -- metadata is stored here - local metaKey = metadataKey:gsub("*", id, 1); - -- pull information about required aggregate keys - -- only 1 operation is supported now - sum - -- but we can calculate multiple values - local values = rcall("HMGET", metaKey, unpack(aggregateKeys)); - - for i, aggregateKey in ipairs(aggregateKeys) do - local aggregateMethod = aggregateType[jsonAggregates[aggregateKey]]; - local value = anynumber(values[i]) or 0; - result[aggregateKey] = aggregateMethod(result[aggregateKey], value); - end -end - -return cjson.encode(result); diff --git a/index.js b/index.js index cb8aed8..025c715 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,4 @@ -const fs = require('fs'); -const path = require('path'); - -const camelCase = require('lodash/camelCase'); -const snakeCase = require('lodash/snakeCase'); - -const lua = fs.readFileSync(path.join(__dirname, 'sorted-filtered-list.lua')); -const fsortBust = fs.readFileSync(path.join(__dirname, 'filtered-list-bust.lua')); -const aggregateScript = fs.readFileSync(path.join(__dirname, 'groupped-list.lua')); +const {Pipeline} = require('ioredis'); // cached vars const regexp = /[\^\$\(\)\%\.\[\]\*\+\-\?]/g; @@ -14,35 +6,94 @@ const keys = Object.keys; const stringify = JSON.stringify; const BLACK_LIST_PROPS = ['eq', 'ne']; +// compat fix, cmod's variable hardcoded exports.FSORT_TEMP_KEYSET = 'fsort_temp_keys'; -const luaWrapper = (script) => ` ---- +/** + * Creates new ioredis command + * @param {ioredis} redis + * @param {string} Command name + * @returns {function} New ioredis nonbuffer command + */ +function createModuleCommand(redis, name) { + let funcs = redis.createBuiltinCommand(name); + redis[name] = funcs.string; + redis[name + "Buffer"] = funcs.buffer; +} -local function getIndexTempKeys(index) - return index .. "::${exports.FSORT_TEMP_KEYSET}"; -end +/** + * Sets argument transformer for redis command + * @param {ioredis} redis + * @param {string} name + * @param {int} keyCount + */ +function cmodAttachArgTransformer(redis, name, keyCount) { + let remapFunc = redis[name]; + remapFunc = remapFunc.bind(redis); + + redis[name] = function (...args) { + var keyPrefix = redis.options.keyPrefix; + if (keyPrefix) { + for (let i = 0; i < keyCount; i++) { + if (args[i] !== null) { + args[i] = keyPrefix + args[i]; + } + } + } + return remapFunc(...args); + } +} ---- +/** + * C-Module exported functions + * We have to rebind them manually. IOredis hardcoded on `redis-commands` package. + * Think it's incorrect to change ioredis deps. + */ +const cmodFunctions = { + fsort: 2, + fsortBust: 1, + fsortAggregate: 2 +} + +function cmodWrapCommands(obj) { + Object.keys(cmodFunctions).forEach((fName) => { + createModuleCommand(obj, fName); + cmodAttachArgTransformer(obj, fName, cmodFunctions[fName]); + }) +} -${script.toString('utf-8')} -`; +function cmodWrapPipeline(redis) { + const rc = redis.constructor; + const pipeFunc = rc.prototype.pipeline.bind(redis); + rc.prototype.pipeline = function () { + let newPipe = pipeFunc(...arguments); + cmodWrapCommands(newPipe); + return newPipe; + }; +} /** - * Attached .sortedFilteredList function to ioredis instance - * @param {ioredis} redis + * Currently unsuported. Causes module failure + * @param {ioredis} redis */ -const fsortScript = luaWrapper(lua); -const fsortBustScript = luaWrapper(fsortBust); +function cmodWrapMulti(redis) { + const rc = redis.constructor; + const multiFunc = rc.prototype.multi.bind(redis); + rc.prototype.multi = function () { + let newMulti = multiFunc(...arguments); + cmodWrapCommands(newMulti); + return newMulti; + }; +} -exports.attach = function attachToRedis(redis, _name, useSnakeCase = false) { - const name = _name || 'sortedFilteredList'; - const bustName = (useSnakeCase ? snakeCase : camelCase)(`${name}Bust`); - const aggregateName = (useSnakeCase ? snakeCase : camelCase)(`${name}Aggregate`); - redis.defineCommand(name, { numberOfKeys: 2, lua: fsortScript }); - redis.defineCommand(bustName, { numberOfKeys: 1, lua: fsortBustScript }); - redis.defineCommand(aggregateName, { numberOfKeys: 2, lua: aggregateScript }); +exports.attach = function attachToRedis(redis, _name, useSnakeCase = false) { + //Let this vars be here, for compatibility with previous versions + let _useSnakeCase = useSnakeCase; + let __name = _name; + cmodWrapCommands(redis); + cmodWrapPipeline(redis); + //cmodWrapMulti(redis); }; /** @@ -94,8 +145,3 @@ exports.filter = function filter(obj) { return stringify(iterateOverObject(obj)); }; -/** - * Exports raw script - * @type {Buffer} - */ -exports.script = lua; diff --git a/package.json b/package.json index 2109cc7..2c2dd34 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,8 @@ "url": "git+https://github.com/makeomatic/redis-filtered-sort.git" }, "keywords": [ - "lua", "redis", + "redis-mod", "sort", "filter", "multi-sort" diff --git a/sorted-filtered-list.lua b/sorted-filtered-list.lua deleted file mode 100644 index 549c6d0..0000000 --- a/sorted-filtered-list.lua +++ /dev/null @@ -1,455 +0,0 @@ - --- must be a set of ids -local idSet = KEYS[1]; --- must be a key, which holds metadata for a given id, with a `*` as a substitute for an id from idSet. Optional -local metadataKey = KEYS[2]; --- hashKey to use for sorting, which is taken from `metadataKey` hash, Optional. -local hashKey = ARGV[1]; --- ASC / DESC -local order = ARGV[2]; --- stringified instructions for filtering -local filter = ARGV[3]; --- current time -local curTime = ARGV[4]; --- pagination offset -local offset = tonumber(ARGV[5] or 0); --- limit of items to return in a single call -local limit = tonumber(ARGV[6] or 10); --- caching time -local expiration = tonumber(ARGV[7] or 30000); --- return the key only -local returnKeyOnly = ARGV[8] or false; - --- local caches -local tempKeysSet = getIndexTempKeys(idSet); -local strlower = string.lower; -local strfind = string.find; -local tinsert = table.insert; -local tcontact = table.concat; -local tsort = table.sort; -local rcall = redis.call; -local unpack = unpack; -local pairs = pairs; - --- declaring to use later -local filterType; - -local function catch(what) - return what[1] -end - -local function try(what) - local status, result = pcall(what[1]); - if not status then - return what[2](result); - end - - return result; -end - -local function isempty(s) - return s == nil or s == '' or s == false; -end - -local function isnumber(a) - return try { - function() - local num = tonumber(a); - return num ~= nil and num or tonumber(cjson.decode(a)); - end, - - catch { - function() - return nil; - end - } - } -end - -local function subrange(t, first, last) - local sub = {}; - for i=first + 1,last do - sub[#sub + 1] = t[i]; - end - - return sub; -end - -local function massive_redis_command(command, key, t) - local i = 1; - local temp = {}; - while (i <= #t) do - tinsert(temp, t[i]); - tinsert(temp, t[i + 1]); - if #temp >= 1000 then - rcall(command, key, unpack(temp)); - temp = {}; - end - i = i + 2; - end - if #temp > 0 then - rcall(command, key, unpack(temp)); - end -end - -local function hashmapSize(table) - local numItems = 0; - for k,v in pairs(table) do - numItems = numItems + 1; - end - - return numItems; -end - -local function storeCacheBuster(key) - local tempKeyTTL = curTime + expiration; - - -- store key in temporary zset - rcall("PEXPIRE", tempKeysSet, expiration); - rcall("ZADD", tempKeysSet, tempKeyTTL, key); -end - -local function updateExpireAndReturnWithSize(key) - -- stores key for cache busting - storeCacheBuster(key); - - -- lengthens cache - rcall("PEXPIRE", key, expiration); - - -- returns either results or key where it's stored - if returnKeyOnly ~= false then - return key; - end - - local ret = rcall("LRANGE", key, offset, offset + limit - 1); - tinsert(ret, rcall("LLEN", key)); - return ret; -end - --- create filtered list name -local finalFilteredListKeys = { idSet }; -local preSortedSetKeys = { idSet }; - --- decoded filters -local jsonFilter = cjson.decode(filter); -local totalFilters = hashmapSize(jsonFilter); - --- order always exists -tinsert(finalFilteredListKeys, order); -tinsert(preSortedSetKeys, order); - -if isempty(metadataKey) == false then - if isempty(hashKey) == false then - tinsert(finalFilteredListKeys, metadataKey); - tinsert(finalFilteredListKeys, hashKey); - tinsert(preSortedSetKeys, metadataKey); - tinsert(preSortedSetKeys, hashKey); - end - - -- do we have filter? - if totalFilters > 0 then - tinsert(finalFilteredListKeys, filter); - end -elseif totalFilters >= 1 and type(jsonFilter["#"]) == "string" then - -- the rest of the filters are ignored, since we cant access them! - tinsert(finalFilteredListKeys, "{\"#\":" .. jsonFilter["#"] .. "}"); -end - --- get final filtered key set -local FFLKey = tcontact(finalFilteredListKeys, ":"); -local PSSKey = tcontact(preSortedSetKeys, ":"); - --- do we have existing filtered set? -if rcall("EXISTS", FFLKey) == 1 then - -- also extend live of the underlaying key - rcall("PEXPIRE", PSSKey, expiration); - storeCacheBuster(PSSKey); - -- and ready from existing key now - return updateExpireAndReturnWithSize(FFLKey); -end - --- do we have an existing sorted set? -local valuesToSort; -if rcall("EXISTS", PSSKey) == 0 then - valuesToSort = rcall("SMEMBERS", idSet); - - -- if we sort the given set - if isempty(metadataKey) == false and isempty(hashKey) == false then - local arr = {}; - for i,v in pairs(valuesToSort) do - local metaKey = metadataKey:gsub("*", v, 1); - -- defaults to empty string to avoid sorting problems - arr[v] = rcall("HGET", metaKey, hashKey) or ''; - end - - -- false implies that items stay in place - -- means when they are the same - they must return false - - local function sortFuncASC(a, b) - local sortA = arr[a]; - local sortB = arr[b]; - - if isempty(sortA) and isempty(sortB) then - return false; - elseif isempty(sortA) then - return false; - elseif isempty(sortB) then - return true; - else - local numA = isnumber(sortA); - local numB = isnumber(sortB); - - if numA ~= nil and numB ~= nil then - return numA < numB; - end - - return strlower(sortA) < strlower(sortB); - end - end - - local function sortFuncDESC(a, b) - local sortA = arr[a]; - local sortB = arr[b]; - - if isempty(sortA) and isempty(sortB) then - return false; - elseif isempty(sortA) then - return true; - elseif isempty(sortB) then - return false; - else - local numA = isnumber(sortA); - local numB = isnumber(sortB); - - if numA ~= nil and numB ~= nil then - return numA > numB; - end - - return strlower(sortA) > strlower(sortB); - end - end - - if order == "ASC" then - tsort(valuesToSort, sortFuncASC); - else - tsort(valuesToSort, sortFuncDESC); - end - else - if order == "ASC" then - tsort(valuesToSort, function (a, b) return a < b end); - else - tsort(valuesToSort, function (a, b) return a > b end); - end - end - - if #valuesToSort > 0 then - massive_redis_command("RPUSH", PSSKey, valuesToSort); - rcall("PEXPIRE", PSSKey, expiration); - storeCacheBuster(PSSKey); - else - -- returns either results or key where it's stored - if returnKeyOnly ~= false then - return PSSKey; - end - - return {0}; - end - - if FFLKey == PSSKey then - -- returns either results or key where it's stored - if returnKeyOnly ~= false then - return PSSKey; - end - - -- early return if we have no filter - local ret = subrange(valuesToSort, offset, offset + limit); - tinsert(ret, #valuesToSort); - return ret; - end -else - - if FFLKey == PSSKey then - -- early return if we have no filter - return updateExpireAndReturnWithSize(PSSKey); - end - - -- populate in-memory data - -- update expiration timer - rcall("PEXPIRE", PSSKey, expiration); - storeCacheBuster(PSSKey); - valuesToSort = rcall("LRANGE", PSSKey, 0, -1); -end - --- filtered list holder -local output = {}; - --- filter function -local function filterString(value, filter) - if isempty(value) then - return false; - end - - return strfind(strlower(value), strlower(filter)) ~= nil; -end - --- filter: eq -local function eq(value, filter) - return value == filter; -end - --- filter: some -local function some(value, filter) - if isempty(value) then - return false; - end - - for _, fieldValue in pairs(filter) do - if eq(value, fieldValue) then - return true - end - end - - return false; -end - --- filter: gte -local function gte(value, filter) - if isempty(value) then - return false; - end - - return isnumber(value) >= filter; -end - --- filter: lte -local function lte(value, filter) - if isempty(value) then - return false; - end - - return isnumber(value) <= filter; -end - --- filter: not equal -local function ne(value, filter) - return value ~= filter; -end - --- filter: exists -local function exists(value, filter) - return isempty(value) == false; -end - --- filter: any -local function any(fieldValue, filter) - for _, filterValue in pairs(filter) do - if filterType[type(filterValue)](fieldValue, filterValue) then - return true; - end - end - - return false; -end - --- supported op type table -local opType = { - gte = gte, - lte = lte, - match = filterString, - eq = eq, - ne = ne, - exists = exists, - isempty = isempty, - some = some, - any = any, -}; - -function filter(op, opFilter, fieldValue) - local thunk = opType[op]; - if type(thunk) ~= "function" then - return error("not supported op: " .. op); - end - - return thunk(fieldValue, opFilter); -end - --- when we match against a table -local function tableFilter(valueToFilter, filterValue) - for op, opFilter in pairs(filterValue) do - if filter(op, opFilter, valueToFilter) ~= true then - return false; - end - end - - return true; -end - -filterType = { - table = tableFilter, - string = filterString -}; - --- if no metadata key, but we are still here -if isempty(metadataKey) then - -- only sort by value, which is id - local filterValue = jsonFilter["#"]; - -- iterate over filtered set - for i, idValue in pairs(valuesToSort) do - -- compare strings and insert if they match - if filterString(idValue, filterValue) then - tinsert(output, idValue); - end - end --- we actually have metadata -else - for i, idValue in pairs(valuesToSort) do - local metaKey = metadataKey:gsub("*", idValue, 1); - local matched = true; - - for fieldName, filterValue in pairs(jsonFilter) do - if fieldName == '#multi' then - local fieldValues = rcall('hmget', metaKey, unpack(filterValue["fields"])); - local anyMatched = false; - local matchValue = filterValue["match"]; - - for _, fieldValue in pairs(fieldValues) do - if filterString(fieldValue, matchValue) then - anyMatched = true; - break; - end - end - - if anyMatched == false then - matched = false; - break; - end - else - -- get data that we are filter - local fieldValue = (fieldName == "#") and idValue or rcall("hget", metaKey, fieldName); - - -- traverse filter types and perform filtering - if filterType[type(filterValue)](fieldValue, filterValue) ~= true then - matched = false; - break; - end - end - end - - if matched then - tinsert(output, idValue); - end - end -end - --- if output is more tha 0 - save data and return it -if #output > 0 then - massive_redis_command("RPUSH", FFLKey, output); - return updateExpireAndReturnWithSize(FFLKey); -end - --- returns either results or key where it's stored -if returnKeyOnly ~= false then - return FFLKey; -end - -return {0}; diff --git a/test/docker-compose.yml b/test/docker-compose.yml index f479d55..5d37c2c 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,5 +1,5 @@ redis: - image: redis:3.2.7-alpine + image: makeomatic/redis_filter_mod container_name: redis hostname: redis expose: diff --git a/test/docker.sh b/test/docker.sh index 5c191fd..1fb5276 100755 --- a/test/docker.sh +++ b/test/docker.sh @@ -15,5 +15,7 @@ if [ x"$CI" = x"true" ]; then trap "$COMPOSE stop; $COMPOSE rm -f -v" EXIT fi +make -C csrc/ docker-build + $COMPOSE up -d $COMPOSE exec tester ./node_modules/.bin/mocha diff --git a/test/index.js b/test/index.js index fc863ca..5da1f53 100644 --- a/test/index.js +++ b/test/index.js @@ -72,6 +72,7 @@ describe('filtered sort suite', function suite() { if (field) { // because lua is -1/+1 and js is -1/0/+1 ids can be sorted differently // therefore we compare sort by derivative + // leaving this as default const map = id => metadata[id][field]; expect(copy.map(map)).to.be.deep.eq(ids.map(map)); } else { @@ -85,7 +86,7 @@ describe('filtered sort suite', function suite() { }); before('populate data', function pretest() { - this.timeout(5000); + this.timeout(20000); const promises = ld.times(prepopulateDataLength, generateUser); // alphanum sort, use locale based sorting @@ -166,7 +167,7 @@ describe('filtered sort suite', function suite() { return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) .then(() => redis.zrangebyscore(`${idSetKey}::${mod.FSORT_TEMP_KEYSET}`, '-inf', '+inf')) - .tap(keys => expect(keys.length).to.be.eq(2)) + .tap((keys) => expect(keys.length).to.be.eq(2)) .tap(() => redis.fsortBust(idSetKey, Date.now())) .then(keys => Promise.join( redis.zcard(`${idSetKey}::${mod.FSORT_TEMP_KEYSET}`), @@ -183,7 +184,7 @@ describe('filtered sort suite', function suite() { const offset = 10; const limit = 20; - it('sorts: asc', function test() { + it('C sorts: asc', function test() { return redis.fsort(idSetKey, null, null, 'ASC', '{}', Date.now()) .tap(sortedBy(comparatorASC, prepopulateDataLength)); }); @@ -211,7 +212,7 @@ describe('filtered sort suite', function suite() { }); }); }); - + describe('sort/pagination only, external numeric field', function sortSuite() { const offset = 10; const limit = 20; @@ -310,6 +311,53 @@ describe('filtered sort suite', function suite() { }); }); + describe('filter/unknown command', function sortFilterExistsEmptySuite() { + const fieldName = 'fieldExists'; + + const filter = mod.filter({ + [fieldName]: { exxists: '1' } + }); + + const filterSubCmd = mod.filter({ + [fieldName]: { any: [ + { gte: 10, ltle: 18 }, + { gte: 35, lte: 45 }, + ]} + }) + + it('direct func', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) + .catch((exc)=> { + expect(exc.message).to.equal("unknown func: exxists") + }) + .then(() => {}) + }); + + it('sub func', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filterSubCmd, Date.now()) + .catch((exc)=> { + expect(exc.message).to.equal("unknown func: ltle") + }) + .then(() => {}) + }); + + }); + + describe('filter/incorrect any', function sortFilterExistsEmptySuite() { + const fieldName = 'fieldExists'; + + const filter = mod.filter({ + [fieldName]: { any: ['1',2,3] } + }); + + it('direct func', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) + .then((res) => { + expect(res[0]).to.equal(0); + }) + }); + }); + describe('filter/exists', function sortFilterExistsEmptySuite() { const fieldName = 'fieldExists'; const hasOwnProperty = Object.prototype.hasOwnProperty; @@ -407,6 +455,63 @@ describe('filtered sort suite', function suite() { }); }); }); + + describe('filter/any sub', function sortFilterExistsEmptySuite() { + const fieldName = 'age'; + const hasOwnProperty = Object.prototype.hasOwnProperty; + const jsFilter = id => { + const userAge = metadata[id][fieldName]; + return ( + ( + (userAge >= 10 && userAge <= 18) || ( + (userAge >=60 && userAge <=65) || (userAge >=20 && userAge <=25) + ) + ) || (userAge >= 35 && userAge <= 45)); + }; + const offset = 0; + const limit = 10; + const filter = mod.filter({ + [fieldName]: { any: [ + { any: [ + { gte: 10, lte: 18 }, + {any: [ + { gte: 20, lte: 25 }, + { gte: 60, lte: 65 } + ]} + ]}, + { gte: 35, lte: 45 }, + ]} + }); + + let filteredIds; + let invertedFilteredIds; + let filteredLength; + + before(function pretest() { + filteredIds = insertedIds.filter(jsFilter); + invertedFilteredIds = invertedIds.filter(jsFilter); + filteredLength = filteredIds.length; + }); + + it('sorts: asc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) + .tap(sortedBy(comparatorASC, filteredLength)); + }); + + it('sorts: desc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'DESC', filter, Date.now()) + .tap(sortedBy(comparatorDESC, filteredLength)); + }); + + it('pagination: asc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now(), offset, limit) + .tap(sortedBy(comparatorASC, filteredLength)) + .then(ids => { + expect(ids).to.have.length.of(limit); + expect(ids).to.be.deep.eq(filteredIds.slice(offset, offset + limit)); + }); + }); + }); describe('filter/any', function sortFilterExistsEmptySuite() { const fieldName = 'age'; @@ -464,6 +569,60 @@ describe('filtered sort suite', function suite() { }); }); + describe('filter/some', function sortFilterExistsEmptySuite() { + const fieldName = 'age'; + const ages = ld.range(20, 44); + + const jsFilter = a => { + const meta = metadata[a]; + return ld.indexOf(ages, meta.age) > -1; + }; + const offset = 0; + const limit = 10; + const filter = mod.filter({ + [fieldName]: { some : ages} + }); + + let filteredIds; + let invertedFilteredIds; + let filteredLength; + + before(function pretest() { + filteredIds = insertedIds.filter(jsFilter); + invertedFilteredIds = invertedIds.filter(jsFilter); + filteredLength = filteredIds.length; + }); + + it('sorts: asc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) + .tap(sortedBy(comparatorASC, filteredLength)); + }); + + it('sorts: desc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'DESC', filter, Date.now()) + .tap(sortedBy(comparatorDESC, filteredLength)); + }); + + it('pagination: asc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now(), offset, limit) + .tap(sortedBy(comparatorASC, filteredLength)) + .then(ids => { + expect(ids).to.have.length.of(limit); + expect(ids).to.be.deep.eq(filteredIds.slice(offset, offset + limit)); + }); + }); + + it('pagination: desc', function test() { + return redis.fsort(idSetKey, metaKeyPattern, null, 'DESC', filter, Date.now(), offset, limit) + .tap(sortedBy(comparatorDESC, filteredLength)) + .then(ids => { + expect(ids).to.have.length.of(limit); + expect(ids).to.be.deep.eq(invertedFilteredIds.slice(offset, offset + limit)); + }); + }); + }); + + describe('sort/id filter only', function sortFilterIdSuite() { const filterIdString = 'd-9'; const jsFilter = a => a.indexOf(filterIdString) >= 0; @@ -517,8 +676,8 @@ describe('filtered sort suite', function suite() { const color = 'ue'; const jsFilter = a => { const meta = metadata[a]; - return a.indexOf(filterIdString) >= 0 && - meta.name.toLowerCase().indexOf(name) >= 0 && + return meta.name.toLowerCase().indexOf(name) >= 0 + && a.indexOf(filterIdString) >= 0 && meta.age >= age && meta.age <= age * 3 && ld.some(ld.values(ld.pick(meta, fields)), it => it.indexOf(color) >= 0) @@ -543,8 +702,7 @@ describe('filtered sort suite', function suite() { }); it('sorts: asc', function test() { - return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()) - .tap(sortedBy(comparatorASC, filteredLength)); + return redis.fsort(idSetKey, metaKeyPattern, null, 'ASC', filter, Date.now()).tap(sortedBy(comparatorASC, filteredLength)); }); it('sorts: desc', function test() {