From 8f613ff93e62b8119a79529a9a7f1a6f54c46e07 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 14 May 2013 15:28:54 +0200 Subject: [PATCH 01/84] Sort by timestamp when finding all events not by version, should find a way to ensure replay use the exact same order than first execution --- lib/event_store/mongodb.coffee | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 10387a1..d5a892c 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -71,7 +71,17 @@ class MongoDbEventStore extends Base callback null, uid findAllEvents: (callback) -> - @_find {}, callback + p = new Profiler "MongoDbEventStore#_find(db request)", @logger + p.start() + @eventCollection.find({}).sort("timestamp":1).toArray (err, items) => + p.end() + + if err? + callback err + else if not items? + callback null, [] + else + @_instantiateEventsFromRows items, callback findAllEventsByAggregateUid: (aggregateUid, callback) -> @_find aggregateUid: aggregateUid, callback From d1668b7185505682fb349fbd25ed09eacee19341 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Wed, 29 May 2013 11:34:06 +0200 Subject: [PATCH 02/84] Handle large binary data in an appropriate manner --- lib/command_bus_client.coffee | 6 +++--- lib/command_bus_server.coffee | 20 +++++++++++++++----- package.json | 4 ++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/command_bus_client.coffee b/lib/command_bus_client.coffee index 224611e..3e4ff8f 100644 --- a/lib/command_bus_client.coffee +++ b/lib/command_bus_client.coffee @@ -46,8 +46,7 @@ class CommandBusClient headers = "Content-Disposition": "form-data; name=\"args[]\"" "Content-Type": "application/json" - - if not Buffer.isBuffer arg and not arg.pipe? + if not Buffer.isBuffer(arg) and not arg.pipe? stream.write headers, JSON.stringify arg else headers["Content-Type"] = "application/octet-stream" @@ -74,7 +73,8 @@ class CommandBusClient request processResponse = (response, callback) -> - data = "" + logger = @logger + data = "" response.on "data", (chunk) -> data += chunk diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index 3f67bdd..b1cd46a 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -4,7 +4,8 @@ formidable = require "formidable" class CommandBusServer - JSON_MEDIA_TYPE_REGEXP = /^application\/(.+\+)?json$/i + JSON_MEDIA_TYPE_REGEXP = /^application\/(.+\+)?json$/i + OCTET_STREAM_MEDIA_TYPE_REGEXP = /^application\/octet-stream/i constructor: ({@commandBus, @logger}) -> throw new Error "Missing command bus" unless @commandBus? @@ -50,15 +51,24 @@ class CommandBusServer form = new formidable.IncomingForm(); form.onPart = (part) -> - data = "" + self = this + chunks = [] + chunksLength = 0 part.on "data", (chunk) -> - data += chunk + self.pause() + chunks.push new Buffer chunk + chunksLength += chunk.length + self.resume() part.on "end", -> contentType = part.headers["content-type"] - if contentType and contentType.match JSON_MEDIA_TYPE_REGEXP - data = JSON.parse data + data = Buffer.concat chunks, chunksLength + + if contentType && contentType.match JSON_MEDIA_TYPE_REGEXP + data = JSON.parse data.toString() + else if not contentType + data = data.toString() if part.name is "name" # command name commandName = data diff --git a/package.json b/package.json index 24325ea..1143dc0 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,9 @@ "microtime": "~0.3.3", "mongodb": "~1.2.12", "bson": "~0.1.7", - "multipart": "~0.1.5", + "multipart": "0.1.5", "crypto": "0.0.3", - "formidable": "~1.0.13" + "formidable": "1.0.13" }, "licenses": [ { From 8c55a5e94fd47743e7c2f12c0feb8dbabf4fec8b Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 30 May 2013 11:21:30 +0200 Subject: [PATCH 03/84] Handle undefined argument in command bus client --- lib/command_bus_client.coffee | 4 ++-- lib/command_bus_server.coffee | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/command_bus_client.coffee b/lib/command_bus_client.coffee index 3e4ff8f..096271c 100644 --- a/lib/command_bus_client.coffee +++ b/lib/command_bus_client.coffee @@ -46,8 +46,8 @@ class CommandBusClient headers = "Content-Disposition": "form-data; name=\"args[]\"" "Content-Type": "application/json" - if not Buffer.isBuffer(arg) and not arg.pipe? - stream.write headers, JSON.stringify arg + if not arg? or (not Buffer.isBuffer(arg) and not arg.pipe?) + stream.write headers, JSON.stringify(arg or null) else headers["Content-Type"] = "application/octet-stream" stream.write headers, arg diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index b1cd46a..affa2b0 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -62,6 +62,7 @@ class CommandBusServer self.resume() part.on "end", -> + contentType = part.headers["content-type"] data = Buffer.concat chunks, chunksLength From b9f9a49537c95a2886e415f5d223bc2c8067ba10 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 30 May 2013 12:07:25 +0200 Subject: [PATCH 04/84] Do not overrride 0 command arguments with null --- lib/command_bus_client.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/command_bus_client.coffee b/lib/command_bus_client.coffee index 096271c..9e85ab4 100644 --- a/lib/command_bus_client.coffee +++ b/lib/command_bus_client.coffee @@ -47,7 +47,8 @@ class CommandBusClient "Content-Disposition": "form-data; name=\"args[]\"" "Content-Type": "application/json" if not arg? or (not Buffer.isBuffer(arg) and not arg.pipe?) - stream.write headers, JSON.stringify(arg or null) + arg = null if arg is undefined + stream.write headers, JSON.stringify arg else headers["Content-Type"] = "application/octet-stream" stream.write headers, arg From 0781b3f2edcd469c896dba55e202ba58c0b459ac Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 12 Jul 2013 11:43:00 +0200 Subject: [PATCH 05/84] Add possibility to choose the order when getting aggreate events --- lib/event_store/mongodb.coffee | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index d5a892c..d85f67e 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -83,8 +83,10 @@ class MongoDbEventStore extends Base else @_instantiateEventsFromRows items, callback - findAllEventsByAggregateUid: (aggregateUid, callback) -> - @_find aggregateUid: aggregateUid, callback + findAllEventsByAggregateUid: (aggregateUid, order, callback) -> + [order, callback] = [null, order] unless callback? + + @_find aggregateUid: aggregateUid, order, callback countAllEventsByAggregateUid: (aggregateUid, callback) -> @_count aggregateUid: aggregateUid, callback @@ -184,10 +186,12 @@ class MongoDbEventStore extends Base else @_instantiateEventsFromRows items, callback - _find: (params, callback) -> + _find: (params, order, callback) -> + [order, callback] = [null, order] unless callback? + p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() - @eventCollection.find(params).sort("version":1).toArray (err, items) => + @eventCollection.find(params).sort("version":(order or 1)).toArray (err, items) => p.end() if err? From f547a2826e9e9d9d2e64da2f8c3d44f0761bf7ef Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 26 Jul 2013 18:09:27 +0200 Subject: [PATCH 06/84] Optimize redis event bus by publishing listened events only --- lib/assembler.js | 3 +-- lib/domain_repository.coffee | 4 +++- lib/event_bus/common/receiver.js | 3 +-- lib/event_bus/redis/emitter.js | 41 ++++++++++++++++++++++---------- lib/event_bus/redis/queue.js | 29 ++++++++++++++++++++++ lib/event_bus/redis/receiver.js | 12 ++++++++-- lib/reporter.coffee | 3 +++ 7 files changed, 75 insertions(+), 20 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 1d79aee..bb113c9 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -419,7 +419,6 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { callback(); }; - if (eventStore.initialize) { eventStore.initialize(function(err){ if (err) @@ -431,4 +430,4 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { } }; -module.exports = Assembler; +module.exports = Assembler; \ No newline at end of file diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index e83797f..6db49ec 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -86,6 +86,9 @@ class DomainRepository else callback() + getLastPublishedEvents: () -> + @emitter.lastEmittedEvents + findAllEventsByAggregateUid: (aggregateUid, callback) -> @store.findAllEventsByAggregateUid aggregateUid, callback @@ -184,7 +187,6 @@ class DomainRepository @_publishEventToDirectListeners event, (err) => @logger.warn "publishEvent", "a direct listener failed: #{err}" if err? @logger.log "publishEvent", "publishing \"#{event.name}\" from aggregate #{event.aggregateUid} to event bus" - @lastPublishedEvent = event if @silent callback() else diff --git a/lib/event_bus/common/receiver.js b/lib/event_bus/common/receiver.js index 0be342a..31b96e9 100644 --- a/lib/event_bus/common/receiver.js +++ b/lib/event_bus/common/receiver.js @@ -42,7 +42,6 @@ CommonEventBusReceiver.prototype.onEvent = function (eventName, options, handler } }; } - eventHandlers.push(handler); }; @@ -70,4 +69,4 @@ CommonEventBusReceiver.prototype.stop = function (callback) { this.queue.stop(callback); }; -module.exports = CommonEventBusReceiver; +module.exports = CommonEventBusReceiver; \ No newline at end of file diff --git a/lib/event_bus/redis/emitter.js b/lib/event_bus/redis/emitter.js index 91a0ed4..fbfe497 100644 --- a/lib/event_bus/redis/emitter.js +++ b/lib/event_bus/redis/emitter.js @@ -10,14 +10,15 @@ inherit(RedisEventBusEmitter, CommonEventBusEmitter); function RedisEventBusEmitter(options) { CommonEventBusEmitter.call(this, options); - RedisEventBus = RedisEventBus || require("../redis"); - this.started = false; - this.starting = false; - options.name = "emitter"; - this.queue = new RedisEventBusQueue(options); - this.host = options.host; - this.port = options.port; - this.queuedCalls = []; + RedisEventBus = RedisEventBus || require("../redis"); + this.started = false; + this.starting = false; + options.name = "emitter"; + this.queue = new RedisEventBusQueue(options); + this.host = options.host; + this.port = options.port; + this.queuedCalls = []; + this.lastEmittedEvents = {} }; RedisEventBusEmitter.prototype.emit = function (event, callback) { @@ -31,14 +32,28 @@ RedisEventBusEmitter.prototype.emit = function (event, callback) { if (queueNames.length == 0) return callback(); + var queueNumber = queueNames.length; + var commit = function(){ + queueNumber--; + if (queueNumber == 0) + transaction.exec(function (err, res) { callback(err); }); + }; + var transaction = self.queueWriter.multi(); queueNames.forEach(function (queueName) { - var key = self.queue.getInQueueListKeyPrefix() + queueName; - var value = self.queue.serializeEvent(event); - self.logger.log("RedisEventBusEmitter", "pushing event \"" + event.name + "\" to key \"" + key + "\""); - transaction.lpush(key, value); + var key = self.queue.getInQueueListKeyPrefix() + queueName; + var value = self.queue.serializeEvent(event); + subscribedKey = self.queue.getSubscribedQueueKeyPrefix() + queueName; + self.queueManager.hget(subscribedKey, event.name, function (err, subscribed){ + if (err) return callback(err); + if (subscribed) { + self.logger.log("RedisEventBusEmitter", "pushing event \"" + event.name + "\" to key \"" + key + "\""); + transaction.lpush(key, value); + self.lastEmittedEvents[queueName] = event; + } + commit(); + }); }); - transaction.exec(function (err, _) { callback(err); }); }); }); }; diff --git a/lib/event_bus/redis/queue.js b/lib/event_bus/redis/queue.js index e93893b..7a21efc 100644 --- a/lib/event_bus/redis/queue.js +++ b/lib/event_bus/redis/queue.js @@ -14,6 +14,7 @@ var QUEUE_KEY_PREFIX = "event-bus" + QUEUE_KEY_SEPARATOR + "queues" + QUEUE_ var IN_QUEUE_KEY_PREFIX = "in" + QUEUE_KEY_SEPARATOR; var OUT_QUEUE_KEY_PREFIX = "out" + QUEUE_KEY_SEPARATOR; var LAST_KEY_INDEX = -1; +var QUEUE_KEY_SUBSCRIBED = "events" + QUEUE_KEY_SEPARATOR; function RedisEventBusQueue(options) { CommonEventBusQueue.call(this, options); @@ -38,6 +39,14 @@ RedisEventBusQueue.prototype.getInQueueKeyPrefix = function () { return IN_QUEUE_KEY_PREFIX; }; +RedisEventBusQueue.prototype.getSubscribedQueueKeyPrefix = function () { + return this.getQueueKeyPrefix() + QUEUE_KEY_SUBSCRIBED; +}; + +RedisEventBusQueue.prototype.getSubscribedQueueKey = function () { + return this.getQueueKeyPrefix() + QUEUE_KEY_SUBSCRIBED + this.name; +}; + RedisEventBusQueue.prototype.getInQueueListKeyPrefix = function () { return this.getQueueKeyPrefix() + this.getInQueueKeyPrefix(); }; @@ -164,6 +173,26 @@ RedisEventBusQueue.prototype.empty = function (callback) { }); }; +RedisEventBusQueue.prototype.subscribeEvents = function (eventNames, callback) { + var self = this; + self.queueManager.hkeys(self.getSubscribedQueueKey(), function(err, oldEventNames){ + if (err) return callback(err); + var transaction = self.queueManager.multi(); + eventNames.forEach(function(eventName){ + transaction.hset(self.getSubscribedQueueKey(), eventName, true); + }); + oldEventNames.forEach(function(oldEventName){ + oldEventName = oldEventName.toString(); + if (eventNames.indexOf(oldEventName) == -1) { + transaction.hdel(self.getSubscribedQueueKey(), oldEventName); + } + }); + transaction.exec(function (err, _) { + callback(err); + }); + }); +}; + RedisEventBusQueue.prototype.serializeEvent = function (event) { var res = BSON.serialize(event); return res; diff --git a/lib/event_bus/redis/receiver.js b/lib/event_bus/redis/receiver.js index 83417ed..8c3f721 100644 --- a/lib/event_bus/redis/receiver.js +++ b/lib/event_bus/redis/receiver.js @@ -26,7 +26,10 @@ RedisEventBusReceiver.prototype.initialize = function (callback) { self._handleEvent(event, callback); }); - self.queue.initialize(callback); + self.queue.initialize(function(err){ + if (err) return callback(err); + self._subscribeEvents(callback); + }); }; RedisEventBusReceiver.prototype.start = function (callback) { @@ -37,4 +40,9 @@ RedisEventBusReceiver.prototype.emptyQueue = function (callback) { this.queue.empty(callback); }; -module.exports = RedisEventBusReceiver; +RedisEventBusReceiver.prototype._subscribeEvents = function (callback) { + var eventNames = Object.keys(this.eventHandlers); + this.queue.subscribeEvents(eventNames, callback); +}; + +module.exports = RedisEventBusReceiver; \ No newline at end of file diff --git a/lib/reporter.coffee b/lib/reporter.coffee index e5fb544..8adad12 100644 --- a/lib/reporter.coffee +++ b/lib/reporter.coffee @@ -9,4 +9,7 @@ class Reporter @logger = null callback() + getQueueName: () -> + @eventBusReceiver.queueName + module.exports = Reporter From a1f63849bbedc7e449fee2cb0d429f5383a7638d Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 10 Sep 2013 10:03:52 +0200 Subject: [PATCH 07/84] Add CommonEventBusReceiver#onEvents method in order to specify the same handler for multiple events --- lib/event_bus/common/receiver.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/event_bus/common/receiver.js b/lib/event_bus/common/receiver.js index 31b96e9..11803e9 100644 --- a/lib/event_bus/common/receiver.js +++ b/lib/event_bus/common/receiver.js @@ -13,6 +13,19 @@ function CommonEventBusReceiver(options) { this.eventHandlers = {}; } +CommonEventBusReceiver.prototype.onEvents = function (eventNames, options, handler) { + var self = this; + + if (!handler) { + handler = options; + options = {}; + } + + eventNames.forEach(function(eventName) { + self.onEvent(eventName, options, handler) + }); +} + CommonEventBusReceiver.prototype.onEvent = function (eventName, options, handler) { var self = this; From dbbcd4f2baee63662cb14d37a5cdf470279c7af2 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 12 Sep 2013 11:11:53 +0200 Subject: [PATCH 08/84] Inject app in the reporters --- lib/assembler.js | 3 ++- lib/domain_repository.coffee | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index bb113c9..4e4121d 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -153,7 +153,8 @@ Assembler.prototype.loadReporters = function (options, callback) { var eventBusReceiver = new EventBusReceiver(eventBusReceiverConfig); var reporterOptions = { eventBusReceiver: eventBusReceiver, - logger: logger + logger: logger, + app: app }; var reporter = new reporterSettings.ctor(reporterOptions); self.reporters.push(reporter); diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 6db49ec..d2d98c2 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -209,4 +209,4 @@ class DomainRepository callback null unless queuedListeners -module.exports = DomainRepository +module.exports = DomainRepository \ No newline at end of file From d001c9d4dadc6cac385e7d3654a3e8513e594e30 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 12 Sep 2013 14:55:16 +0200 Subject: [PATCH 09/84] Store app at reporter instantiation --- lib/assembler.js | 2 +- lib/reporter.coffee | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 4e4121d..7a6b959 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -154,7 +154,7 @@ Assembler.prototype.loadReporters = function (options, callback) { var reporterOptions = { eventBusReceiver: eventBusReceiver, logger: logger, - app: app + app: self.app }; var reporter = new reporterSettings.ctor(reporterOptions); self.reporters.push(reporter); diff --git a/lib/reporter.coffee b/lib/reporter.coffee index 8adad12..9734e3a 100644 --- a/lib/reporter.coffee +++ b/lib/reporter.coffee @@ -1,8 +1,9 @@ class Reporter - constructor: ({@eventBusReceiver, @logger}) -> + constructor: ({@eventBusReceiver, @logger, @app}) -> throw new Error "Missing event bus receiver" unless @eventBusReceiver? throw new Error "Missing logger" unless @logger? + throw new Error "Missing app" unless @app? destroy: (callback) -> @eventBusReceiver = null From 15023cb1077ba0b7aaca59cdf8207c858ba71d6d Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 18 Feb 2014 16:55:59 +0100 Subject: [PATCH 10/84] Use a cursor to fetch all events from mongodb database, remove async from redis event bus reciever --- lib/domain_repository.coffee | 7 +++--- lib/event_bus/common/receiver.js | 37 ++++++++++++++++---------------- lib/event_store/mongodb.coffee | 24 ++++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 67ca0ae..3988323 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -75,16 +75,17 @@ class DomainRepository entityInstantiator.findByUid uid, callback replayAllEvents: (callback) -> - @store.findAllEvents (err, events) => + @store.findAllEvents (err, events, batchCallback) => if events.length > 0 eventQueue = async.queue (event, eventTaskCallback) => event.replayed = true @_publishEvent event, eventTaskCallback , 1 - eventQueue.drain = callback + eventQueue.drain = batchCallback eventQueue.push events else - callback() + batchCallback() + , callback getLastPublishedEvents: () -> @emitter.lastEmittedEvents diff --git a/lib/event_bus/common/receiver.js b/lib/event_bus/common/receiver.js index adf1c52..d118421 100644 --- a/lib/event_bus/common/receiver.js +++ b/lib/event_bus/common/receiver.js @@ -69,25 +69,26 @@ CommonEventBusReceiver.prototype._handleEvent = function (event, callback) { return callback(); } - var parallelHandling = async.queue(function (eventHandler, callback) { - eventHandler(event, function (err) { - if (err) { - self.logger.warning("CommonEventBusReceiver#_handleEvent", "an error occurred in an event handler: " + (err.stack || err.message || err)); - errors.push(err); + var pendingHandlers = 0; + eventHandlers.forEach(function(eventHandler){ + pendingHandlers++ + eventHandler(event, function (err) { + if (err) { + self.logger.warning("CommonEventBusReceiver#_handleEvent", "an error occurred in an event handler: " + (err.stack || err.message || err)); + errors.push(err); + } + pendingHandlers--; + if (pendingHandlers == 0) { + if (errors.length > 0) { + var error = new Error("Some errors occured when handling the event: [" + errors.join(" | ") + "]"); + callback(error); + } else { + self.lastEvent = event; + callback(null); } - callback(err); - }); - }, Infinity); - - parallelHandling.drain = function () { - if (errors.length > 0) { - var err = new Error("Some errors occured when handling the event: [" + errors.join(" | ") + "]"); - return callback(err); - } - self.lastEvent = event; - callback(); - }; - parallelHandling.push(eventHandlers); + } + }); + }); }; CommonEventBusReceiver.prototype.stop = function (callback) { diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 96c3886..b6c3ce7 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -70,18 +70,22 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEvents: (callback) -> + findAllEvents: (batchCallback, callback) -> p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() - @eventCollection.find({}).sort("timestamp":1).toArray (err, items) => - p.end() - - if err? - callback err - else if not items? - callback null, [] - else - @_instantiateEventsFromRows items, callback + cursor = @eventCollection.find({}).sort("timestamp":1) + retrieve = => + cursor.nextObject (err, item) => + return callback err if err? + if item? + @_instantiateEventsFromRows [item], (err, events) -> + batchCallback err, events, (err) -> + return callback err if err? + retrieve() + else + p.end() + callback null + retrieve() findAllEventsByEntityUid: (entityUid, order, callback) -> [order, callback] = [null, order] unless callback? From 5802868afe6bf5af05288183e7615f3248a881f4 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 18 Feb 2014 17:10:41 +0100 Subject: [PATCH 11/84] Refactor replay mechanism --- lib/domain_repository.coffee | 12 ++------ lib/event_store/base.coffee | 5 +++- lib/event_store/mongodb.coffee | 53 +++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 3988323..4070f1c 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -75,16 +75,8 @@ class DomainRepository entityInstantiator.findByUid uid, callback replayAllEvents: (callback) -> - @store.findAllEvents (err, events, batchCallback) => - if events.length > 0 - eventQueue = async.queue (event, eventTaskCallback) => - event.replayed = true - @_publishEvent event, eventTaskCallback - , 1 - eventQueue.drain = batchCallback - eventQueue.push events - else - batchCallback() + @store.findAllEventsOneByOne (err, event, eventHandlerCallback) => + @_publishEvent event, eventHandlerCallback , callback getLastPublishedEvents: () -> diff --git a/lib/event_store/base.coffee b/lib/event_store/base.coffee index 600e6c0..39e7b9c 100644 --- a/lib/event_store/base.coffee +++ b/lib/event_store/base.coffee @@ -9,6 +9,9 @@ class BaseEventStore findAllEvents: (options, callback) -> throw new Error "implement me" + findAllEventsOneByOne: (options, callback) -> + throw new Error "implement me" + findAllEventsByEntityUid: (entityUid, options, callback) -> throw new Error "Implement me" @@ -23,4 +26,4 @@ class BaseEventStore # implement me if you want snapshots in your store callback? null -module.exports = BaseEventStore +module.exports = BaseEventStore \ No newline at end of file diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index b6c3ce7..e387620 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -70,7 +70,7 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEvents: (batchCallback, callback) -> + findAllEventsOneByOne: (eventHandler, callback) -> p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() cursor = @eventCollection.find({}).sort("timestamp":1) @@ -78,8 +78,8 @@ class MongoDbEventStore extends Base cursor.nextObject (err, item) => return callback err if err? if item? - @_instantiateEventsFromRows [item], (err, events) -> - batchCallback err, events, (err) -> + @_instantiateEventFromRow item, (err, event) -> + eventHandler err, event, (err) -> return callback err if err? retrieve() else @@ -215,27 +215,8 @@ class MongoDbEventStore extends Base return callback null, events if rows.length is 0 rowsQueue = async.queue (row, rowCallback) => - uid = row.uid - name = row.name - entityUid = row.entityUid - data = row.data - timestamp = row.timestamp - version = row.version - - @_loadAttachmentsFromRow row, (err, attachments) -> - return rowCallback err if err? - - for attachmentName, attachmentBody of attachments - data[attachmentName] = attachmentBody - - event = new Event - name: name - data: data - uid: uid - entityUid: entityUid - timestamp: timestamp - version: version - + @_instantiateEventFromRow row, (err, event) -> + return callback err if err? events.push event defer rowCallback , 1 @@ -245,6 +226,30 @@ class MongoDbEventStore extends Base rowsQueue.push rows + _instantiateEventFromRow: (row, callback) -> + uid = row.uid + name = row.name + entityUid = row.entityUid + data = row.data + timestamp = row.timestamp + version = row.version + + @_loadAttachmentsFromRow row, (err, attachments) -> + return rowCallback err if err? + + for attachmentName, attachmentBody of attachments + data[attachmentName] = attachmentBody + + event = new Event + name: name + data: data + uid: uid + entityUid: entityUid + timestamp: timestamp + version: version + + callback null, event + _loadAttachmentsFromRow: (row, callback) -> attachments = {} for attachmentName, attachmentBody of row._attachments From 3f13432445b1a668f8828376e7a5cdf29a6c9f69 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 20 Feb 2014 11:39:15 +0100 Subject: [PATCH 12/84] Update coffee script --- lib/main.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/main.js b/lib/main.js index b56cdbd..fc14563 100644 --- a/lib/main.js +++ b/lib/main.js @@ -1,4 +1,4 @@ -require("coffee-script"); +require("coffee-script/register"); module.exports = { Assembler: require("./assembler"), diff --git a/package.json b/package.json index b50e3e7..d1bb5e8 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "framework" ], "dependencies": { - "coffee-script": "1.3.3", + "coffee-script": "1.7.1", "async": "0.1.22", "node-uuid": "1.3.3", "redis": "0.9.0", From faf9561a70b7e70fd626872ef3c5b5afd16ce90c Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 20 Feb 2014 11:54:53 +0100 Subject: [PATCH 13/84] Bump hiredis --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d1bb5e8..ec1457e 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "async": "0.1.22", "node-uuid": "1.3.3", "redis": "0.9.0", - "hiredis": "~0.1.14", + "hiredis": "0.1.16", "nano": "~3.3.6", "devnull": "0.0.10", "mongodb": "~1.2.12", From 71b06817fb9513993eccc5686847f9dc85fe5566 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 21 Feb 2014 13:57:07 +0100 Subject: [PATCH 14/84] Callback executeCommand even when a validation exists on the command handler and the repository is alted oterwise it could create deadlocks --- lib/assembler.js | 1 - lib/command_bus.coffee | 1 + lib/command_bus_client.coffee | 5 ++--- lib/command_bus_server.coffee | 1 - lib/domain_repository.coffee | 1 + 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 02465d2..bdd3e41 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -241,7 +241,6 @@ Assembler.prototype.tearDownApp = function (callback) { var self = this; // TODO: review this, anything else to unload/destroy? - async.series([ function (next) { self.destroyEventBusEmitter(next); diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index e62d401..fc8b2ac 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -57,6 +57,7 @@ class CommandBus p2.end() p.end() done args... + transaction.callback = callback else transaction = (done) -> p.start() diff --git a/lib/command_bus_client.coffee b/lib/command_bus_client.coffee index a7d0159..1b99deb 100644 --- a/lib/command_bus_client.coffee +++ b/lib/command_bus_client.coffee @@ -36,8 +36,7 @@ class CommandBusClient request = @_makeRequest path: "/commands" stream = request.stream - request.on "error", (err) -> - callback err + request.on "error", callback request.on "response", (response) => @_processResponse response, callback @@ -96,4 +95,4 @@ class CommandBusClient logger.error "CommandBusClient", "command failed remotely: #{error.stack || error.message || error}" callback error -module.exports = CommandBusClient +module.exports = CommandBusClient \ No newline at end of file diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index 79094de..4f38c72 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -87,7 +87,6 @@ class CommandBusServer logger.log "CommandBusServer", "deserialize command \"#{commandName}\"" @commandBus.deserializeCommand commandName, payload, (err, command) => - logger.log "CommandBusServer", "start command \"#{commandName}\"" @commandBus.executeCommand command, (err) -> if err? logger.alert "CommandBusServer", "error while executing command (#{err})" diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 4070f1c..469c358 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -22,6 +22,7 @@ class DomainRepository @transactionQueue = async.queue (transaction, done) => if @halted @logger.warning "transaction", "skipped (#{@transactionQueue.length()} more transaction(s) in queue)" + transaction.callback() if transaction.callback? done() else @transacting = true From 80eb6283ecbae3d639dec9ffe433f14ba1642c6b Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 21 Feb 2014 14:03:26 +0100 Subject: [PATCH 15/84] Properly handle errors in async.queue and fix closeConnectionAndReturn in mongoDb event store --- lib/assembler.js | 30 +++++++++++++++--------------- lib/event_store/mongodb.coffee | 9 +++++++-- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index bdd3e41..52b1bd4 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -276,19 +276,19 @@ Assembler.prototype.destroyReporters = function (callback) { if (self.reporters.length == 0) return callback(); - var queue = async.queue(function (reporter, callback) { + var queue = async.queue(function (reporter, taskCallback) { reporter.destroy(function (err) { - if (err) + if (err) { self.logger.error("Assembler#unloadReporters", "destroying reporter failed: " + err); - callback(err); + return callback(err); + } + taskCallback(); }); }, Infinity); - queue.drain = function (err) { - if (err) - self.logger.error("Assembler#unloadReporters", "error: " + err); + queue.drain = function() { self.reporters = []; - callback(err); + callback(); }; queue.push(self.reporters); }; @@ -299,23 +299,23 @@ Assembler.prototype.destroyReportStores = function (callback) { if (self.reportStores.length == 0) return callback(); - var queue = async.queue(function (reportStore, callback) { + var queue = async.queue(function (reportStore, taskCallback) { if (reportStore.destroy) { reportStore.destroy(function (err) { - if (err) + if (err) { self.logger.error("Assembler#destroyReportStores", "destroying report store failed: " + err); - callback(err); + return callback(err); + } + taskCallback(); }); } else { - callback(); + taskCallback(); } }, Infinity); - queue.drain = function (err) { - if (err) - self.logger.error("Assembler#destroyReportStores", "error: " + err); + queue.drain = function() { self.reportStores = []; - callback(err); + callback(); }; queue.push(self.reportStores); }; diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index e387620..ec27502 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -43,14 +43,19 @@ class MongoDbEventStore extends Base destroy: (callback) -> @_closeConnectionAndReturn @db, null, (err) => + return callback err if err? @db = null @eventCollection = null @snapshotCollection = null callback null _closeConnectionAndReturn: (db, err, callback) -> - db.close() if db? - callback err + if db? + db.close (closeErr) -> + return callback closeErr if closeErr? + callback err + else + callback err setup: (callback) -> async.series [ From d92c18125b22276d3fc2ed8a925536704eaa2a95 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 24 Feb 2014 10:50:38 +0100 Subject: [PATCH 16/84] Set replayed flag on events during replay --- lib/domain_repository.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 469c358..d4de410 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -77,6 +77,7 @@ class DomainRepository replayAllEvents: (callback) -> @store.findAllEventsOneByOne (err, event, eventHandlerCallback) => + event.replayed = true @_publishEvent event, eventHandlerCallback , callback From f212ec72b47ca8bf23ac40ef99e5a1811a904c50 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 24 Feb 2014 15:01:53 +0100 Subject: [PATCH 17/84] Use Entity instead of finalCtor to initialize Entity bases, otherwize may be undefined --- lib/entity.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index e1d8a4b..04f8d57 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -35,9 +35,9 @@ Entity = (name, finalCtor, Ctor) -> Base::_initializeAtVersion = (version) -> @$version = version @$appliedEvents = [] - @$domainRepository = finalCtor.$domainRepository - @$commandBus = finalCtor.$commandBus - @$logger = finalCtor.$logger + @$domainRepository = Entity.domainRepository + @$commandBus = Entity.commandBus + @$logger = Entity.logger Base::_serialize = (contained) -> throw new Error "References between entity are forbidden" if contained From 17f3fb64cd16a2303365c8544aeb2f08ace2ce92 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 25 Feb 2014 10:26:17 +0100 Subject: [PATCH 18/84] Initialize domain objects after app instantiation --- lib/assembler.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 52b1bd4..52b2b7e 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -100,7 +100,7 @@ Assembler.prototype.assembleApp = function (callback) { } } - var createApp = function createApp() { + var createApp = function createApp(callback) { self.app = new self.App(appOptions); callback(); }; @@ -111,10 +111,15 @@ Assembler.prototype.assembleApp = function (callback) { return callback(err); appOptions.domainRepository = self.domainRepository; appOptions.commandBus = self.commandBus; - createApp(); + createApp(function(){ + DomainObject.initializeConstructors(); + callback(); + }); }); } else { - createApp(); + createApp(function() { + callback(); + }); } }; @@ -410,8 +415,6 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { logger: self.logger }); - DomainObject.initializeConstructors(); - self.eventBusEmitter = eventBusEmitter; self.domainRepository = domainRepository; self.commandBus = commandBus; From 620a40c6eee0493b09aa63677cfbddd0644c0e3e Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 6 Mar 2014 21:05:02 +0100 Subject: [PATCH 19/84] Add logger in CommandHandler --- lib/command_handler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/command_handler.js b/lib/command_handler.js index 03408c0..86f2501 100644 --- a/lib/command_handler.js +++ b/lib/command_handler.js @@ -28,6 +28,7 @@ CommandHandler.registerAllInDirectoryOnBus = function (path, options) { var logger = options.logger; fs.readdirSync(path).forEach(function (fileName) { var CommandHandler = require(path + "/" + fileName); + CommandHandler.logger = logger; logger.log("CommandHandler", "loading command handler for command \"" + CommandHandler.getCommandName() + "\" (" + fileName + ")"); commandBus.registerCommandHandler(CommandHandler); }); From a7a8792671db899f96c67d6d0c6a6c38f5af75da Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 15 Mar 2014 09:16:16 +0100 Subject: [PATCH 20/84] Avoid returning an error when entity could not be found --- lib/entity.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index 04f8d57..dcfb5e1 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -96,12 +96,12 @@ Entity = (name, finalCtor, Ctor) -> deferred = Q.defer() deferred.promise.nodeify callback if uid? - @$domainRepository.findEntityByUid @, uid, (err, entity) -> + @$domainRepository.findEntityByUid @, uid, (err, entity) => if err? deferred.reject err - else if not entity - deferred.reject new Error "Could not find entity with UID " + uid else + if not entity + @$logger.error "findByUid", "Could not find entity with UID #{uid}" deferred.resolve entity else deferred.reject(new Error 'Please provide a UID to find') From 5769cdceca8197b402b739d13b940c6cdfbb2585 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 15 Mar 2014 20:21:39 +0100 Subject: [PATCH 21/84] Refactor condition statement --- lib/entity.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index dcfb5e1..0e635db 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -100,8 +100,7 @@ Entity = (name, finalCtor, Ctor) -> if err? deferred.reject err else - if not entity - @$logger.error "findByUid", "Could not find entity with UID #{uid}" + @$logger.error "findByUid", "Could not find entity with UID #{uid}" if not entity deferred.resolve entity else deferred.reject(new Error 'Please provide a UID to find') From f7b9cd7fe425e3b7ffecf352172e52ead1585941 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 18 Feb 2014 16:55:59 +0100 Subject: [PATCH 22/84] Use a cursor to fetch all events from mongodb database, remove async from redis event bus reciever --- lib/domain_repository.coffee | 7 +++--- lib/event_bus/common/receiver.js | 37 ++++++++++++++++---------------- lib/event_store/mongodb.coffee | 24 ++++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 67ca0ae..3988323 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -75,16 +75,17 @@ class DomainRepository entityInstantiator.findByUid uid, callback replayAllEvents: (callback) -> - @store.findAllEvents (err, events) => + @store.findAllEvents (err, events, batchCallback) => if events.length > 0 eventQueue = async.queue (event, eventTaskCallback) => event.replayed = true @_publishEvent event, eventTaskCallback , 1 - eventQueue.drain = callback + eventQueue.drain = batchCallback eventQueue.push events else - callback() + batchCallback() + , callback getLastPublishedEvents: () -> @emitter.lastEmittedEvents diff --git a/lib/event_bus/common/receiver.js b/lib/event_bus/common/receiver.js index adf1c52..d118421 100644 --- a/lib/event_bus/common/receiver.js +++ b/lib/event_bus/common/receiver.js @@ -69,25 +69,26 @@ CommonEventBusReceiver.prototype._handleEvent = function (event, callback) { return callback(); } - var parallelHandling = async.queue(function (eventHandler, callback) { - eventHandler(event, function (err) { - if (err) { - self.logger.warning("CommonEventBusReceiver#_handleEvent", "an error occurred in an event handler: " + (err.stack || err.message || err)); - errors.push(err); + var pendingHandlers = 0; + eventHandlers.forEach(function(eventHandler){ + pendingHandlers++ + eventHandler(event, function (err) { + if (err) { + self.logger.warning("CommonEventBusReceiver#_handleEvent", "an error occurred in an event handler: " + (err.stack || err.message || err)); + errors.push(err); + } + pendingHandlers--; + if (pendingHandlers == 0) { + if (errors.length > 0) { + var error = new Error("Some errors occured when handling the event: [" + errors.join(" | ") + "]"); + callback(error); + } else { + self.lastEvent = event; + callback(null); } - callback(err); - }); - }, Infinity); - - parallelHandling.drain = function () { - if (errors.length > 0) { - var err = new Error("Some errors occured when handling the event: [" + errors.join(" | ") + "]"); - return callback(err); - } - self.lastEvent = event; - callback(); - }; - parallelHandling.push(eventHandlers); + } + }); + }); }; CommonEventBusReceiver.prototype.stop = function (callback) { diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 96c3886..b6c3ce7 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -70,18 +70,22 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEvents: (callback) -> + findAllEvents: (batchCallback, callback) -> p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() - @eventCollection.find({}).sort("timestamp":1).toArray (err, items) => - p.end() - - if err? - callback err - else if not items? - callback null, [] - else - @_instantiateEventsFromRows items, callback + cursor = @eventCollection.find({}).sort("timestamp":1) + retrieve = => + cursor.nextObject (err, item) => + return callback err if err? + if item? + @_instantiateEventsFromRows [item], (err, events) -> + batchCallback err, events, (err) -> + return callback err if err? + retrieve() + else + p.end() + callback null + retrieve() findAllEventsByEntityUid: (entityUid, order, callback) -> [order, callback] = [null, order] unless callback? From 291ba7055d45cd1bcd9bd7dc354aecd0266c33bc Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 1 Apr 2014 11:50:51 +0200 Subject: [PATCH 23/84] Refactor replay mechanism Fetch events domain events one by one instead all in memory --- lib/domain_repository.coffee | 12 ++------ lib/event_store/base.coffee | 5 +++- lib/event_store/mongodb.coffee | 53 +++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 3988323..4070f1c 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -75,16 +75,8 @@ class DomainRepository entityInstantiator.findByUid uid, callback replayAllEvents: (callback) -> - @store.findAllEvents (err, events, batchCallback) => - if events.length > 0 - eventQueue = async.queue (event, eventTaskCallback) => - event.replayed = true - @_publishEvent event, eventTaskCallback - , 1 - eventQueue.drain = batchCallback - eventQueue.push events - else - batchCallback() + @store.findAllEventsOneByOne (err, event, eventHandlerCallback) => + @_publishEvent event, eventHandlerCallback , callback getLastPublishedEvents: () -> diff --git a/lib/event_store/base.coffee b/lib/event_store/base.coffee index 600e6c0..39e7b9c 100644 --- a/lib/event_store/base.coffee +++ b/lib/event_store/base.coffee @@ -9,6 +9,9 @@ class BaseEventStore findAllEvents: (options, callback) -> throw new Error "implement me" + findAllEventsOneByOne: (options, callback) -> + throw new Error "implement me" + findAllEventsByEntityUid: (entityUid, options, callback) -> throw new Error "Implement me" @@ -23,4 +26,4 @@ class BaseEventStore # implement me if you want snapshots in your store callback? null -module.exports = BaseEventStore +module.exports = BaseEventStore \ No newline at end of file diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index b6c3ce7..e387620 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -70,7 +70,7 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEvents: (batchCallback, callback) -> + findAllEventsOneByOne: (eventHandler, callback) -> p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() cursor = @eventCollection.find({}).sort("timestamp":1) @@ -78,8 +78,8 @@ class MongoDbEventStore extends Base cursor.nextObject (err, item) => return callback err if err? if item? - @_instantiateEventsFromRows [item], (err, events) -> - batchCallback err, events, (err) -> + @_instantiateEventFromRow item, (err, event) -> + eventHandler err, event, (err) -> return callback err if err? retrieve() else @@ -215,27 +215,8 @@ class MongoDbEventStore extends Base return callback null, events if rows.length is 0 rowsQueue = async.queue (row, rowCallback) => - uid = row.uid - name = row.name - entityUid = row.entityUid - data = row.data - timestamp = row.timestamp - version = row.version - - @_loadAttachmentsFromRow row, (err, attachments) -> - return rowCallback err if err? - - for attachmentName, attachmentBody of attachments - data[attachmentName] = attachmentBody - - event = new Event - name: name - data: data - uid: uid - entityUid: entityUid - timestamp: timestamp - version: version - + @_instantiateEventFromRow row, (err, event) -> + return callback err if err? events.push event defer rowCallback , 1 @@ -245,6 +226,30 @@ class MongoDbEventStore extends Base rowsQueue.push rows + _instantiateEventFromRow: (row, callback) -> + uid = row.uid + name = row.name + entityUid = row.entityUid + data = row.data + timestamp = row.timestamp + version = row.version + + @_loadAttachmentsFromRow row, (err, attachments) -> + return rowCallback err if err? + + for attachmentName, attachmentBody of attachments + data[attachmentName] = attachmentBody + + event = new Event + name: name + data: data + uid: uid + entityUid: entityUid + timestamp: timestamp + version: version + + callback null, event + _loadAttachmentsFromRow: (row, callback) -> attachments = {} for attachmentName, attachmentBody of row._attachments From c8cadbec7d626b6f9f0a3ad8022e08bf69cd3426 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 1 Apr 2014 11:52:37 +0200 Subject: [PATCH 24/84] Bump coffee-script --- lib/main.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/main.js b/lib/main.js index b56cdbd..fc14563 100644 --- a/lib/main.js +++ b/lib/main.js @@ -1,4 +1,4 @@ -require("coffee-script"); +require("coffee-script/register"); module.exports = { Assembler: require("./assembler"), diff --git a/package.json b/package.json index b50e3e7..d1bb5e8 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "framework" ], "dependencies": { - "coffee-script": "1.3.3", + "coffee-script": "1.7.1", "async": "0.1.22", "node-uuid": "1.3.3", "redis": "0.9.0", From 897a44d3b5471a6b0106ef1272c001ce1d53e4d7 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 20 Feb 2014 11:54:53 +0100 Subject: [PATCH 25/84] Bump hiredis --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d1bb5e8..ec1457e 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "async": "0.1.22", "node-uuid": "1.3.3", "redis": "0.9.0", - "hiredis": "~0.1.14", + "hiredis": "0.1.16", "nano": "~3.3.6", "devnull": "0.0.10", "mongodb": "~1.2.12", From 2df27bb24bb77587c4e53aa87b85c84741c73d10 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 21 Feb 2014 13:57:07 +0100 Subject: [PATCH 26/84] Callback executeCommand even when a validation exists on the command handler and the repository is alted oterwise it could create deadlocks --- lib/assembler.js | 1 - lib/command_bus.coffee | 1 + lib/command_bus_client.coffee | 5 ++--- lib/command_bus_server.coffee | 1 - lib/domain_repository.coffee | 1 + 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 02465d2..bdd3e41 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -241,7 +241,6 @@ Assembler.prototype.tearDownApp = function (callback) { var self = this; // TODO: review this, anything else to unload/destroy? - async.series([ function (next) { self.destroyEventBusEmitter(next); diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index e62d401..fc8b2ac 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -57,6 +57,7 @@ class CommandBus p2.end() p.end() done args... + transaction.callback = callback else transaction = (done) -> p.start() diff --git a/lib/command_bus_client.coffee b/lib/command_bus_client.coffee index a7d0159..1b99deb 100644 --- a/lib/command_bus_client.coffee +++ b/lib/command_bus_client.coffee @@ -36,8 +36,7 @@ class CommandBusClient request = @_makeRequest path: "/commands" stream = request.stream - request.on "error", (err) -> - callback err + request.on "error", callback request.on "response", (response) => @_processResponse response, callback @@ -96,4 +95,4 @@ class CommandBusClient logger.error "CommandBusClient", "command failed remotely: #{error.stack || error.message || error}" callback error -module.exports = CommandBusClient +module.exports = CommandBusClient \ No newline at end of file diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index 79094de..4f38c72 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -87,7 +87,6 @@ class CommandBusServer logger.log "CommandBusServer", "deserialize command \"#{commandName}\"" @commandBus.deserializeCommand commandName, payload, (err, command) => - logger.log "CommandBusServer", "start command \"#{commandName}\"" @commandBus.executeCommand command, (err) -> if err? logger.alert "CommandBusServer", "error while executing command (#{err})" diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 4070f1c..469c358 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -22,6 +22,7 @@ class DomainRepository @transactionQueue = async.queue (transaction, done) => if @halted @logger.warning "transaction", "skipped (#{@transactionQueue.length()} more transaction(s) in queue)" + transaction.callback() if transaction.callback? done() else @transacting = true From 8c43181b6cddfed92ecea6de05d4bac0b4366faf Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 1 Apr 2014 12:07:16 +0200 Subject: [PATCH 27/84] Handle errors appropriately in mongoDB event store closeConnectionAndReturn() --- lib/event_store/mongodb.coffee | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index e387620..ec27502 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -43,14 +43,19 @@ class MongoDbEventStore extends Base destroy: (callback) -> @_closeConnectionAndReturn @db, null, (err) => + return callback err if err? @db = null @eventCollection = null @snapshotCollection = null callback null _closeConnectionAndReturn: (db, err, callback) -> - db.close() if db? - callback err + if db? + db.close (closeErr) -> + return callback closeErr if closeErr? + callback err + else + callback err setup: (callback) -> async.series [ From 5fd0201814e727b842d85bddaad619b603c94677 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 1 Apr 2014 12:08:24 +0200 Subject: [PATCH 28/84] Handle erors appropriately in async queues for Assembler --- lib/assembler.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index bdd3e41..52b1bd4 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -276,19 +276,19 @@ Assembler.prototype.destroyReporters = function (callback) { if (self.reporters.length == 0) return callback(); - var queue = async.queue(function (reporter, callback) { + var queue = async.queue(function (reporter, taskCallback) { reporter.destroy(function (err) { - if (err) + if (err) { self.logger.error("Assembler#unloadReporters", "destroying reporter failed: " + err); - callback(err); + return callback(err); + } + taskCallback(); }); }, Infinity); - queue.drain = function (err) { - if (err) - self.logger.error("Assembler#unloadReporters", "error: " + err); + queue.drain = function() { self.reporters = []; - callback(err); + callback(); }; queue.push(self.reporters); }; @@ -299,23 +299,23 @@ Assembler.prototype.destroyReportStores = function (callback) { if (self.reportStores.length == 0) return callback(); - var queue = async.queue(function (reportStore, callback) { + var queue = async.queue(function (reportStore, taskCallback) { if (reportStore.destroy) { reportStore.destroy(function (err) { - if (err) + if (err) { self.logger.error("Assembler#destroyReportStores", "destroying report store failed: " + err); - callback(err); + return callback(err); + } + taskCallback(); }); } else { - callback(); + taskCallback(); } }, Infinity); - queue.drain = function (err) { - if (err) - self.logger.error("Assembler#destroyReportStores", "error: " + err); + queue.drain = function() { self.reportStores = []; - callback(err); + callback(); }; queue.push(self.reportStores); }; From c9605f291f7c13c60b66dce2f645f286b3e4893f Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 24 Feb 2014 10:50:38 +0100 Subject: [PATCH 29/84] Set replayed flag on events during replay --- lib/domain_repository.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 469c358..d4de410 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -77,6 +77,7 @@ class DomainRepository replayAllEvents: (callback) -> @store.findAllEventsOneByOne (err, event, eventHandlerCallback) => + event.replayed = true @_publishEvent event, eventHandlerCallback , callback From aaa20bdc2df04c9da905409767fd02529fbffd07 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 24 Feb 2014 15:01:53 +0100 Subject: [PATCH 30/84] Use Entity instead of finalCtor to initialize Entity bases, otherwize may be undefined --- lib/entity.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index e1d8a4b..04f8d57 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -35,9 +35,9 @@ Entity = (name, finalCtor, Ctor) -> Base::_initializeAtVersion = (version) -> @$version = version @$appliedEvents = [] - @$domainRepository = finalCtor.$domainRepository - @$commandBus = finalCtor.$commandBus - @$logger = finalCtor.$logger + @$domainRepository = Entity.domainRepository + @$commandBus = Entity.commandBus + @$logger = Entity.logger Base::_serialize = (contained) -> throw new Error "References between entity are forbidden" if contained From 7a4ca9942d93fbb75719924948a885efdcb29613 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 25 Feb 2014 10:26:17 +0100 Subject: [PATCH 31/84] Initialize domain objects after app instantiation --- lib/assembler.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 52b1bd4..52b2b7e 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -100,7 +100,7 @@ Assembler.prototype.assembleApp = function (callback) { } } - var createApp = function createApp() { + var createApp = function createApp(callback) { self.app = new self.App(appOptions); callback(); }; @@ -111,10 +111,15 @@ Assembler.prototype.assembleApp = function (callback) { return callback(err); appOptions.domainRepository = self.domainRepository; appOptions.commandBus = self.commandBus; - createApp(); + createApp(function(){ + DomainObject.initializeConstructors(); + callback(); + }); }); } else { - createApp(); + createApp(function() { + callback(); + }); } }; @@ -410,8 +415,6 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { logger: self.logger }); - DomainObject.initializeConstructors(); - self.eventBusEmitter = eventBusEmitter; self.domainRepository = domainRepository; self.commandBus = commandBus; From 6264ee5763d316eb7f059bb1f8aedbc16e2a5822 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 6 Mar 2014 21:05:02 +0100 Subject: [PATCH 32/84] Add logger in CommandHandler --- lib/command_handler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/command_handler.js b/lib/command_handler.js index 03408c0..86f2501 100644 --- a/lib/command_handler.js +++ b/lib/command_handler.js @@ -28,6 +28,7 @@ CommandHandler.registerAllInDirectoryOnBus = function (path, options) { var logger = options.logger; fs.readdirSync(path).forEach(function (fileName) { var CommandHandler = require(path + "/" + fileName); + CommandHandler.logger = logger; logger.log("CommandHandler", "loading command handler for command \"" + CommandHandler.getCommandName() + "\" (" + fileName + ")"); commandBus.registerCommandHandler(CommandHandler); }); From 9fbad9b651f14fc6941402751d6da78d58bebf01 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 15 Mar 2014 09:16:16 +0100 Subject: [PATCH 33/84] Avoid returning an error when entity could not be found --- lib/entity.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index 04f8d57..dcfb5e1 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -96,12 +96,12 @@ Entity = (name, finalCtor, Ctor) -> deferred = Q.defer() deferred.promise.nodeify callback if uid? - @$domainRepository.findEntityByUid @, uid, (err, entity) -> + @$domainRepository.findEntityByUid @, uid, (err, entity) => if err? deferred.reject err - else if not entity - deferred.reject new Error "Could not find entity with UID " + uid else + if not entity + @$logger.error "findByUid", "Could not find entity with UID #{uid}" deferred.resolve entity else deferred.reject(new Error 'Please provide a UID to find') From 16e67744fb9984333c8986378aa7bffa157f6d7f Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 15 Mar 2014 20:21:39 +0100 Subject: [PATCH 34/84] Refactor condition statement --- lib/entity.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index dcfb5e1..0e635db 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -100,8 +100,7 @@ Entity = (name, finalCtor, Ctor) -> if err? deferred.reject err else - if not entity - @$logger.error "findByUid", "Could not find entity with UID #{uid}" + @$logger.error "findByUid", "Could not find entity with UID #{uid}" if not entity deferred.resolve entity else deferred.reject(new Error 'Please provide a UID to find') From 0a9eb4ab70bd022a475affe8b5c17e281bfc25fa Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 28 Apr 2014 10:31:59 +0200 Subject: [PATCH 35/84] Add timestamp to logs --- lib/logger.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/logger.js b/lib/logger.js index 0d25dc2..7df2d6b 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -2,7 +2,7 @@ var Logger = require("devnull"); var logLevel = process.env.LOG_LEVEL != null ? process.env.LOG_LEVEL : 6; var logger = new Logger({ - timestamp: false, + timestamp: true, level: logLevel }); @@ -11,13 +11,13 @@ logger.http = function(req, res, next) { req._startTime = process.hrtime(); res.end = function(chunk, encoding) { - var status = res.statusCode; - var len = parseInt(res.getHeader("Content-Length"), 10); - len = isNaN(len) ? "" : "- " + len; + var status = res.statusCode; + var len = parseInt(res.getHeader("Content-Length"), 10); + len = isNaN(len) ? "" : "- " + len; - var diff = process.hrtime(req._startTime); - var duration = (diff[0] * 1e9 + diff[1]) / 1000000; - var line = "" + req.method + " " + req.originalUrl + " " + res.statusCode + " " + duration + "ms " + len; + var diff = process.hrtime(req._startTime); + var duration = (diff[0] * 1e9 + diff[1]) / 1000000; + var line = "" + req.method + " " + req.originalUrl + " " + res.statusCode + " " + duration + "ms " + len; logger.log("http", line); res.end = end; From 001c17c3d3e65dc9858919b78143ca5c215d09ca Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 9 May 2014 15:45:54 +0200 Subject: [PATCH 36/84] Add replaying flag to assembler and event bus --- lib/assembler.js | 18 ++++++++++----- lib/event_bus/redis/emitter.js | 2 ++ lib/event_bus/redis/queue.js | 39 +++++++++++++++++++-------------- lib/event_bus/redis/receiver.js | 10 +++++---- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 52b2b7e..57508fa 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -8,12 +8,15 @@ var AggregateRoot = require("./aggregate_root"); var Service = require("./service"); var logger = require("./logger"); -function Assembler(App, configurationPath) { +function Assembler(App, configurationPath, options) { if (!App) throw new Error("Missing app constructor"); if (!configurationPath) throw new Error("Missing configuration file path"); + if (!options) + options = {}; + this.App = App; this.configurationPath = configurationPath; this.logger = logger; @@ -22,6 +25,7 @@ function Assembler(App, configurationPath) { this.reportStores = []; this.eventBusReceivers = []; this.env = process.env.NODE_ENV || "development"; + this.replaying = !!options.replaying; } // Assemble an app, load the reporters. @@ -30,8 +34,9 @@ Assembler.prototype.loadCompleteApp = function (options, callback) { if (!callback) { callback = options; - options = {} + options = {}; } + if (typeof options.clearQueues == "undefined") options.clearQueues = false; if (typeof options.setupEventStore == "undefined") @@ -146,7 +151,8 @@ Assembler.prototype.loadReporters = function (options, callback) { var eventBusReceiverConfig = { queueName: reporterSettings.eventBusReceiver.queue, logger: self.logger, - scope: self.configuration.scope + scope: self.configuration.scope, + replaying: self.replaying }; if (reporterSettings.eventBusReceiver.options) { @@ -369,7 +375,8 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { var eventBusEmitterOptions = { logger: self.logger, - scope: self.configuration.scope + scope: self.configuration.scope, + replaying: self.replaying }; if (self.configuration.domain.eventBusEmitter.options) { @@ -383,7 +390,8 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { var domainRepository = new DomainRepository({ store: eventStore, emitter: eventBusEmitter, - logger: self.logger + logger: self.logger, + replaying: self.replaying }); var commandBus = new CommandBus({ domainRepository: domainRepository, diff --git a/lib/event_bus/redis/emitter.js b/lib/event_bus/redis/emitter.js index 42d9438..4097a76 100644 --- a/lib/event_bus/redis/emitter.js +++ b/lib/event_bus/redis/emitter.js @@ -19,6 +19,8 @@ function RedisEventBusEmitter(options) { this.port = options.port; this.queuedCalls = []; this.lastEmittedEvents = {} + console.log(options) + this.replaying = !!options.replaying; }; RedisEventBusEmitter.prototype.emit = function (event, callback) { diff --git a/lib/event_bus/redis/queue.js b/lib/event_bus/redis/queue.js index 40c6ecd..9da9743 100644 --- a/lib/event_bus/redis/queue.js +++ b/lib/event_bus/redis/queue.js @@ -9,29 +9,36 @@ var RedisEventBus; inherit(RedisEventBusQueue, CommonEventBusQueue); -var QUEUE_KEY_SEPARATOR = ":"; -var QUEUE_KEY_PREFIX = "event-bus" + QUEUE_KEY_SEPARATOR + "queues" + QUEUE_KEY_SEPARATOR; -var IN_QUEUE_KEY_PREFIX = "in" + QUEUE_KEY_SEPARATOR; -var OUT_QUEUE_KEY_PREFIX = "out" + QUEUE_KEY_SEPARATOR; -var LAST_KEY_INDEX = -1; -var QUEUE_KEY_SUBSCRIBED = "events" + QUEUE_KEY_SEPARATOR; +var QUEUE_KEY_SEPARATOR = ":"; +var QUEUE_KEY_PREFIX = "event-bus" + QUEUE_KEY_SEPARATOR + "queues" + QUEUE_KEY_SEPARATOR; +var IN_QUEUE_KEY_PREFIX = "in" + QUEUE_KEY_SEPARATOR; +var OUT_QUEUE_KEY_PREFIX = "out" + QUEUE_KEY_SEPARATOR; +var LAST_KEY_INDEX = -1; +var QUEUE_KEY_SUBSCRIBED = "events" + QUEUE_KEY_SEPARATOR; +var QUEUE_KEY_REPLAYING_PREFIX = "replay" + QUEUE_KEY_SEPARATOR; function RedisEventBusQueue(options) { CommonEventBusQueue.call(this, options); - RedisEventBus = RedisEventBus || require("../redis"); - this.retries = 0; - this.scope = options.scope; - this.host = options.host; - this.port = options.port; - this.stopped = true; - this.canStart = true; + RedisEventBus = RedisEventBus || require("../redis"); + this.retries = 0; + this.scope = options.scope; + this.host = options.host; + this.port = options.port; + this.stopped = true; + this.canStart = true; + this.replaying = !!options.replaying; } RedisEventBusQueue.prototype.getQueueKeyPrefix = function () { + var result = QUEUE_KEY_PREFIX; + if (this.scope) - return this.scope + QUEUE_KEY_SEPARATOR + QUEUE_KEY_PREFIX; - else - return QUEUE_KEY_PREFIX + QUEUE_KEY_SEPARATOR; + result = this.scope + QUEUE_KEY_SEPARATOR + result; + + if (this.replaying) + result += QUEUE_KEY_REPLAYING_PREFIX; + + return result; }; RedisEventBusQueue.prototype.getQueueSetKey = RedisEventBusQueue.prototype.getQueueKeyPrefix; diff --git a/lib/event_bus/redis/receiver.js b/lib/event_bus/redis/receiver.js index 8c3f721..b83c802 100644 --- a/lib/event_bus/redis/receiver.js +++ b/lib/event_bus/redis/receiver.js @@ -6,9 +6,10 @@ inherit(RedisEventBusReceiver, CommonEventBusReceiver); function RedisEventBusReceiver(options) { CommonEventBusReceiver.call(this, options); - this.scope = options.scope; - this.host = options.host; - this.port = options.port; + this.scope = options.scope; + this.host = options.host; + this.port = options.port; + this.replaying = !!options.replaying; } RedisEventBusReceiver.prototype.initialize = function (callback) { @@ -19,7 +20,8 @@ RedisEventBusReceiver.prototype.initialize = function (callback) { host: self.host, port: self.port, logger: self.logger, - scope: self.scope + scope: self.scope, + replaying : self.replaying }); self.queue.registerHandler(function (event, callback) { From 563ee345bba9a3d01b938e2f8236c3d4f817abfd Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 9 May 2014 17:05:36 +0200 Subject: [PATCH 37/84] Handle two-steps replay in domain repository --- lib/domain_repository.coffee | 16 ++++++++--- lib/event_store/mongodb.coffee | 51 ++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index d4de410..7b3683e 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -9,7 +9,7 @@ REDIS_STORE = "redis" class DomainRepository - constructor: ({@store, @emitter, @logger}) -> + constructor: ({@store, @emitter, @logger, @replaying}) -> throw new Error "Missing store" unless @store? throw new Error "Missing event bus emitter" unless @emitter? throw new Error "Missing logger" unless @logger? @@ -75,11 +75,18 @@ class DomainRepository entityInstantiator = new EntityInstantiator store: @store, Entity: Entity, logger: @logger entityInstantiator.findByUid uid, callback - replayAllEvents: (callback) -> - @store.findAllEventsOneByOne (err, event, eventHandlerCallback) => + replayAllEvents: (options, callback) -> + return callback new Error("Replay mode not set") unless @replaying + [options, callback] = [{}, options] unless callback? + + @store.findAllEventsOneByOne options, (err, event, eventHandlerCallback) => + return callback err if err? event.replayed = true + lastEvent = event @_publishEvent event, eventHandlerCallback - , callback + , (err) -> + return callback err if err? + callback null, lastEvent getLastPublishedEvents: () -> @emitter.lastEmittedEvents @@ -88,6 +95,7 @@ class DomainRepository @store.findAllEventsByEntityUid entityUid, callback add: (entity) -> + throw new Error "Cannot write during replay" if @replaying throw new Error "Entity is missing its UID" unless entity.uid? @entityEvents[entity.uid] ?= [] addedEvents = @entityEvents[entity.uid] diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index ec27502..ec75232 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -75,22 +75,43 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEventsOneByOne: (eventHandler, callback) -> - p = new Profiler "MongoDbEventStore#_find(db request)", @logger - p.start() - cursor = @eventCollection.find({}).sort("timestamp":1) - retrieve = => - cursor.nextObject (err, item) => + findAllEventsOneByOne: (options, eventHandler, callback) -> + [options, eventHandler, callback] = [{}, options, eventHandler] unless callback? + + query = {} + sortQuery = + timestamp: 1 + uid: 1 + + doFindAll = => + p = new Profiler "MongoDbEventStore#_find(db request)", @logger + p.start() + cursor = @eventCollection.find(query).sort(sortQuery) + retrieve = => + cursor.nextObject (err, item) => + return callback err if err? + if item? + @_instantiateEventFromRow item, (err, event) -> + eventHandler err, event, (err) -> + return callback err if err? + retrieve() + else + p.end() + callback null + retrieve() + + if options.startUid? + @eventCollection.findOne({uid: options.startUid}).sort sortQuery, (err, event) -> return callback err if err? - if item? - @_instantiateEventFromRow item, (err, event) -> - eventHandler err, event, (err) -> - return callback err if err? - retrieve() - else - p.end() - callback null - retrieve() + startTimestamp = event.timestamp + query = + timestamp: + $gte: startTimestamp + uid: + $ne: startUid + doFindAll() + else + doFindAll() findAllEventsByEntityUid: (entityUid, order, callback) -> [order, callback] = [null, order] unless callback? From 6d1f4b82af724c4469e092f3c995d7c44e8b2504 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 12 May 2014 11:14:33 +0200 Subject: [PATCH 38/84] Do not start command bus server when replaying --- lib/assembler.js | 3 ++- lib/command_bus.coffee | 4 ++-- lib/event_bus/redis/emitter.js | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 57508fa..cef6cab 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -396,7 +396,8 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { var commandBus = new CommandBus({ domainRepository: domainRepository, port: self.configuration.domain.commandBus && self.configuration.domain.commandBus.port, - logger: self.logger + logger: self.logger, + replaying: self.replaying }); // These initializations take place on the constructor, not instances. diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index fc8b2ac..1528805 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -4,11 +4,11 @@ Profiler = require "./profiler" class CommandBus - constructor: ({@domainRepository, @logger, @port}) -> + constructor: ({@domainRepository, @logger, @port, @replaying}) -> throw new Error "Missing domain repository" unless @domainRepository? throw new Error "Missing logger" unless @logger? - if @port + if @port && !@replaying @server = new CommandBusServer commandBus: @, port: @port, logger: @logger @server.listen @port diff --git a/lib/event_bus/redis/emitter.js b/lib/event_bus/redis/emitter.js index 4097a76..340199a 100644 --- a/lib/event_bus/redis/emitter.js +++ b/lib/event_bus/redis/emitter.js @@ -18,8 +18,7 @@ function RedisEventBusEmitter(options) { this.host = options.host; this.port = options.port; this.queuedCalls = []; - this.lastEmittedEvents = {} - console.log(options) + this.lastEmittedEvents = {}; this.replaying = !!options.replaying; }; From 5f2aa4522924cf2a04add6ed80f75fa2c94a0699 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 17 May 2014 19:55:12 +0200 Subject: [PATCH 39/84] Add information about entity name --- lib/entity.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/entity.coffee b/lib/entity.coffee index 0e635db..152597c 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -100,10 +100,10 @@ Entity = (name, finalCtor, Ctor) -> if err? deferred.reject err else - @$logger.error "findByUid", "Could not find entity with UID #{uid}" if not entity + @$logger.error "findByUid", "Could not find entity <#{name}> with UID #{uid}" if not entity deferred.resolve entity else - deferred.reject(new Error 'Please provide a UID to find') + deferred.reject(new Error "Please provide a UID to find") deferred.promise Base.findAllEvents = (uid, callback) -> From c3c79372017fa9d3ac10dd15db2fb4aca404f5ff Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 20 May 2014 15:37:13 +0200 Subject: [PATCH 40/84] Add index on mongo event store to support two step replays (non backward-compatible) Run on your event database --- lib/event_store/mongodb.coffee | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index ec75232..d6d89a3 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -62,13 +62,15 @@ class MongoDbEventStore extends Base (next) => @eventCollection.remove next (next) => - @eventCollection.ensureIndex {"entityUid": 1}, next + @eventCollection.ensureIndex {entityUid: 1}, next (next) => - @eventCollection.ensureIndex {"entityUid": 1, "version": 1}, { unique: true }, next + @eventCollection.ensureIndex {entityUid: 1, version: 1}, { unique: true }, next + (next) => + @eventCollection.ensureIndex {timestamp: 1, uid: 1}, next (next) => @snapshotCollection.remove next (next) => - @snapshotCollection.ensureIndex {"entityUid": 1}, next + @snapshotCollection.ensureIndex {entityUid: 1}, next ], callback createNewUid: (callback) -> From e735a347ce912c886c6739c5f680d15f12a497b2 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 20 May 2014 15:38:49 +0200 Subject: [PATCH 41/84] Pass replaying mode to reports --- lib/assembler.js | 5 +++-- lib/domain_repository.coffee | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index cef6cab..45e9a6d 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -200,7 +200,8 @@ Assembler.prototype.initializeReports = function (callback) { var ReportStore = reportSettings.store.ctor; var reportStore = new ReportStore({ uri: reportSettings.store.uri, - logger: self.logger + logger: self.logger, + replaying: self.replaying }); Report.initialize({store: reportStore, logger: self.logger}); self.reports.push(Report); @@ -443,4 +444,4 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { } }; -module.exports = Assembler; +module.exports = Assembler; \ No newline at end of file diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 7b3683e..0dc2f90 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -78,6 +78,7 @@ class DomainRepository replayAllEvents: (options, callback) -> return callback new Error("Replay mode not set") unless @replaying [options, callback] = [{}, options] unless callback? + lastEvent = null @store.findAllEventsOneByOne options, (err, event, eventHandlerCallback) => return callback err if err? From 8fa1b969af5c430fe65548cf299f1544fd382e37 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Tue, 20 May 2014 17:45:52 +0200 Subject: [PATCH 42/84] Pass replaying option to reporters --- lib/assembler.js | 3 ++- lib/reporter.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/assembler.js b/lib/assembler.js index 45e9a6d..2558f5c 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -165,7 +165,8 @@ Assembler.prototype.loadReporters = function (options, callback) { var reporterOptions = { eventBusReceiver: eventBusReceiver, logger: logger, - app: self.app + app: self.app, + replaying: self.replaying }; var reporter = new reporterSettings.ctor(reporterOptions); diff --git a/lib/reporter.js b/lib/reporter.js index 9915eec..fcadfbb 100644 --- a/lib/reporter.js +++ b/lib/reporter.js @@ -5,6 +5,7 @@ function Reporter(attributes) { this.app = attributes.app; this.logger = attributes.logger; this.eventBusReceiver = attributes.eventBusReceiver; + this.replaying = !!attributes.replaying; } Reporter.prototype.destroy = function(callback) { From 4c6a8bfaeb4d5045f1c8bd9cc6ad7cf5052db9b7 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Thu, 5 Jun 2014 15:34:14 +0200 Subject: [PATCH 43/84] Do not erroneous sort after findOne in findAllEventsOneByOne --- lib/event_store/mongodb.coffee | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index d6d89a3..442bed9 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -88,6 +88,7 @@ class MongoDbEventStore extends Base doFindAll = => p = new Profiler "MongoDbEventStore#_find(db request)", @logger p.start() + cursor = @eventCollection.find(query).sort(sortQuery) retrieve = => cursor.nextObject (err, item) => @@ -102,8 +103,9 @@ class MongoDbEventStore extends Base callback null retrieve() - if options.startUid? - @eventCollection.findOne({uid: options.startUid}).sort sortQuery, (err, event) -> + startUid = options.startUid + if startUid? + @eventCollection.findOne uid: startUid, (err, event) -> return callback err if err? startTimestamp = event.timestamp query = From 37202270b022499273a09a295fb4217e3f093f7b Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Fri, 6 Jun 2014 11:52:47 +0200 Subject: [PATCH 44/84] Fix commandName var in command bus --- lib/command_bus.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index 1528805..4d675bd 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -16,7 +16,7 @@ class CommandBus registerCommandHandler: (commandHandler) -> commandName = commandHandler.getCommandName() - throw new Error "A command and its handler for command named \"#{command}\" were already registered" if @commandHandlers[commandName]? + throw new Error "A command and its handler for command named \"#{commandName}\" were already registered" if @commandHandlers[commandName]? @commandHandlers[commandName] = commandHandler createNewUid: (callback) -> From fe8389357dabaf11c70cbb50c053cd36465970fd Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Wed, 21 May 2014 23:56:30 +0200 Subject: [PATCH 45/84] Postgres EventStore: WIP --- lib/domain_repository.coffee | 17 +- lib/entity_instantiator.js | 28 ++- lib/event_store.coffee | 10 +- lib/event_store/mongodb.coffee | 2 +- lib/event_store/postgresql.coffee | 393 ++++++++++++++++++++++++++++++ lib/snapshot.js | 2 +- package.json | 1 + 7 files changed, 430 insertions(+), 23 deletions(-) create mode 100644 lib/event_store/postgresql.coffee diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 0dc2f90..bd65234 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -39,7 +39,8 @@ class DomainRepository done() else @logger.log "transaction", "succeeded, comitting" - @_commit => + @_commit (err) => + throw err if err? @logger.log "transaction", "committed (#{@transactionQueue.length()} more transaction(s) in queue)" @transacting = false done() @@ -149,16 +150,15 @@ class DomainRepository if firstEvent? queue = async.queue (event, eventTaskCallback) => + console.log "@@@ commit", event nextEvent = entityAppliedEvents.shift() queue.push nextEvent if nextEvent? @logger.log "commit", "saving event \"#{event.name}\" for entity #{event.entityUid}" - @store.saveEvent event, (err, event) => - savedEvents.push(event); - if err? - eventTaskCallback err - else - eventTaskCallback null + @store.saveEvent event, (err, event) -> + savedEvents.push event + return callback err if err? + eventTaskCallback null , 1 queue.drain = entityTaskCallback @@ -168,8 +168,7 @@ class DomainRepository , Infinity # TODO determine if it is safe to treat all entitys in parallel? - entityQueue.drain = (err) => - return callback err if err? + entityQueue.drain = => return callback null unless savedEvents.length > 0 publicationQueue = async.queue (event, publicationCallback) => @_publishEvent event, publicationCallback diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index 0062e17..f41adf9 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -40,13 +40,17 @@ EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, call if (err) return callback(err); entity.applyEvents(events, function (err) { if (err) return callback(err); + self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); var snapshotAge = entity.$version - version; if (snapshotAge > SNAPSHOT_MAX_AGE) { self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \""+ uid + "\""); - self._snapEntity(entity); + self._snapEntity(entity, function(err) { + if (err) callback(err); + callback(null, entity); + }); + } else { + callback(null, entity); } - self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); - callback(null, entity); }); }); }; @@ -62,21 +66,29 @@ EntityInstantiator.prototype._instantiateThroughEventsFromUid = function (uid, c } else { self.Entity.buildFromEvents(events, function (err, entity) { if (err) return callback(err); - if (entity.$version > SNAPSHOT_MAX_AGE) - self._snapEntity(entity); self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" at version " + entity.$version + " from " + events.length + " events"); - callback(null, entity); + if (entity.$version > SNAPSHOT_MAX_AGE) { + self._snapEntity(entity, function(err) { + if (err) return callback(err); + callback(null, entity); + }); + } else { + callback(null, entity); + } }); } }); }; -EntityInstantiator.prototype._snapEntity = function (entity) { +EntityInstantiator.prototype._snapEntity = function (entity, callback) { var self = this; self.logger.log("EntityInstantiator", "snapping entity \"" + entity.uid +"\" at version " + entity.$version); var snapshot = Snapshot.makeFromEntity(entity); - self.store.saveSnapshot(snapshot); + self.store.saveSnapshot(snapshot, function(err) { + if (err) return callback(err); + callback(); + }); }; module.exports = EntityInstantiator; diff --git a/lib/event_store.coffee b/lib/event_store.coffee index 6f4110b..4747f28 100644 --- a/lib/event_store.coffee +++ b/lib/event_store.coffee @@ -1,6 +1,8 @@ -CouchDbEventStore = require "./event_store/couchdb" -MongoDbEventStore = require "./event_store/mongodb" +CouchDbEventStore = require "./event_store/couchdb" +MongoDbEventStore = require "./event_store/mongodb" +PostgresqlEventStore = require "./event_store/postgresql" module.exports = - CouchDb: CouchDbEventStore - MongoDb: MongoDbEventStore + CouchDb : CouchDbEventStore + MongoDb : MongoDbEventStore + Postgresql : PostgresqlEventStore diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 442bed9..a90563c 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -205,7 +205,7 @@ class MongoDbEventStore extends Base @logger.alert "MongoDbEventStore#saveSnapshot", "failed to save snapshot of entity \"#{snapshot.entityUid}\": #{err}" else @logger.log "MongoDbEventStore#saveSnapshot", "saved snapshot for entity \"#{snapshot.entityUid}\"" - callback? err + callback err _findLimited: (params, eventCount, callback) -> p = new Profiler "MongoDbEventStore#_findLimited(db request)", @logger diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee new file mode 100644 index 0000000..16e9db2 --- /dev/null +++ b/lib/event_store/postgresql.coffee @@ -0,0 +1,393 @@ +url = require "url" +async = require "async" +uuid = require "node-uuid" +Event = require "../event" +Snapshot = require "../snapshot" +Base = require "./base" +Profiler = require "../profiler" +pg = require("pg").native +defer = require "../defer" +format = require("util").format + +class PostgresqlEventStore extends Base + + constructor: ({@uri, @logger}) -> + throw new Error "Missing URI" unless @uri + throw new Error "Missing logger" unless @logger + + @eventTableName = "events" + @snapshotTableName = "snapshots" + + createNewUid: (callback) -> + uid = uuid.v4() + callback null, uid + + initialize: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + done() + callback null + + _handleError: (err, pgClient, pgCallback, callback) -> + pgCallback(pgClient) + callback(err) + + _createEventTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + version integer, + name varchar(255), + data text, + entity_uid varchar(255), + timestamp bigint, + attachments text);" + query = format query, @eventTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback() + + _createIndexOnEventTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + indexName = "#{@eventTableName}_entity_uid_idx" + + query = "DROP INDEX IF EXISTS %s; + CREATE INDEX ON %s (entity_uid);" + query = format query, indexName, @eventTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback() + + _createSnapshotTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + contents text, + entity_uid varchar(255), + version integer);" + query = format query, @snapshotTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback null + + _createIndexOnSnapshotTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + indexName = "#{@snapshotTableName}_entity_uid_idx" + + query = "DROP INDEX IF EXISTS %s; + CREATE INDEX ON %s (entity_uid);" + query = format query, indexName, @snapshotTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback null + + _emptySnapshotTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "TRUNCATE TABLE %s RESTART IDENTITY;" + query = format query, @snapshotTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback null + + _addAutoIncrementOnEventVersion: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "CREATE OR REPLACE FUNCTION events_version_auto_increment() + RETURNS trigger AS $$ + DECLARE + _rel_id constant int := 'events'::regclass::int; + BEGIN + PERFORM pg_advisory_xact_lock(_rel_id); + + SELECT COALESCE(MAX(version) + 1, 1) + INTO NEW.version + FROM events + WHERE entity_uid = NEW.entity_uid; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql STRICT; + + DROP TRIGGER IF EXISTS events_version_auto_increment on events; + + CREATE TRIGGER events_version_auto_increment + BEFORE INSERT ON events + FOR EACH ROW WHEN (NEW.version IS NULL) + EXECUTE PROCEDURE events_version_auto_increment();" + #query = format query, @eventTableName, @eventTableName, @eventTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback null + + destroy: (callback) -> + callback null + + setup: (callback) -> + console.log "Setup postgresql event store" + async.series [ + (next) => + console.log "Creating event table" + @_createEventTable next + (next) => + console.log "Creating index on event table" + @_createIndexOnEventTable next + (next) => + console.log "Adding auto increment on event table" + @_addAutoIncrementOnEventVersion next + (next) => + console.log "Creating snapshot table" + @_createSnapshotTable next + (next) => + console.log "Creating index on snapshot table" + @_createIndexOnSnapshotTable next + (next) => + console.log "Emptying snapshot table" + @_emptySnapshotTable next + ], callback + + findAllEventsOneByOne: (eventHandler, callback) -> + p = new Profiler "PostgresqlEventStore#_findAllEventsOneByOne (db request)", @logger + p.start() + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "FIND * FROM %s ORDER BY id ASC;" + query = format query, @eventTableName + + clientReceiver = client.query query + + clientReceiver.on "error", (err) => + return @_handleError(err, client, done, callback) if err? + + clientReceiver.on "row", (row) => + @_instantiateEventFromRow row, (err, event) -> + eventHandler err, event, (err) -> + return @_handleError(err, client, done, callback) if err? + + clientReceiver.on "end", (results) => + p.end() + done() + callback null + + findAllEvents: (options, callback) -> + params = true + order = "ASC" + + @_find params, order, callback + + _find: (params, order, limit, callback) -> + [limit, callback] = [null, limit] unless callback? + + p = new Profiler "PostgresqlEventStore#_find(db request)", @logger + p.start() + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "SELECT * FROM %s WHERE %s ORDER BY id %s" + query += " LIMIT %s" if limit? + query += ";" + + if limit? + query = format query, @eventTableName, params, order, limit + else + query = format query, @eventTableName, params, order + + clientReceiver = client.query query, (err, results) => + p.end() + return @_handleError(err, client, done, callback) if err? + done() + @_instantiateEventsFromRows results.rows, (err, events) -> + console.log "-- _find", query, err, events + callback err, events + + _count: (params, callback) -> + p = new Profiler "PostgresqlEventStore#_count(db request)", @logger + p.start() + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "SELECT COUNT(*) FROM %s WHERE %s;" + query = format query, @eventTableName, params + + clientReceiver = client.query query, (err, results) => + p.end() + return @_handleError(err, client, done, callback) if err? + done() + callback null, results.count + + findAllEventsByEntityUid: (entityUid, order, callback) -> + [order, callback] = [null, order] unless callback? + + params = "entity_uid='#{entityUid}'" + order ?= "ASC" + + @_find params, order, callback + + countAllEventsByEntityUid: (entityUid, callback) -> + params = "entity_uid='#{entityUid}'" + + @_count params, callback + + findSomeEventsByEntityUidBeforeVersion: (entityUid, version, eventCount, callback) -> + params = "entity_uid='#{{entityUid}}' AND version <= #{version}" + order = "ASC" + + @_find params, order, eventCount, callback + + findAllEventsByEntityUidAfterVersion: (entityUid, version, callback) -> + params = "entity_uid='#{entityUid}' AND version > #{version}" + order = "ASC" + + @_find params, order, callback + + saveEvent: (event, callback) => + console.log "&&&", "saveEvent", event + p = new Profiler "PostgresqlEventStore#saveEvent (db request)", @logger + p.start() + + data = {} + attachments = {} + + for key, value of event.data + if value instanceof Buffer + attachments[key] = value + else + data[key] = value + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "INSERT INTO %s (name, entity_uid, timestamp, data, attachments) VALUES ('%s', '%s', %d, '%j', '%j');" + query = format query, @eventTableName, event.name, event.entityUid, event.timestamp, data, attachments + + clientReceiver = client.query query, (err, results) => + p.end() + return @_handleError(err, client, done, callback) if err? + done() + callback null, event + + loadSnapshotForEntityUid: (uid, callback) -> + p = new Profiler "PostgresqlEventStore#loadSnapshotForEntityUid (db request)", @logger + p.start() + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "SELECT * FROM %s WHERE %s;" + params = "entity_uid='#{uid}'" + + query = format query, @snapshotTableName, params + + clientReceiver = client.query query, (err, results) => + p.end() + return @_handleError(err, client, done, callback) if err? + done() + + rawSnapshot = results.rows[0] + snapshot = null + + if rawSnapshot? + console.log "-- raw snapshot --", typeof rawSnapshot.contents, rawSnapshot.contents + + snapshotAttributes = + version : rawSnapshot.version + entityUid : rawSnapshot.entity_uid + contents : JSON.parse(rawSnapshot.contents) + + snapshot = new Snapshot snapshotAttributes + + callback null, snapshot + + saveSnapshot: (snapshot, callback) -> + console.log "-- saveSnapshot", snapshot + + p = new Profiler "PostgresqlEventStore#saveSnapshot (db request)", @logger + p.start() + + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "WITH upsert AS (UPDATE %s SET version=%d, contents='%j' WHERE entity_uid='%s' RETURNING *) INSERT INTO %s (version, contents, entity_uid) SELECT %d, '%j', '%s' WHERE NOT EXISTS (SELECT * FROM upsert);" + query = format query, @snapshotTableName, snapshot.version, snapshot.contents, snapshot.entityUid, @snapshotTableName, snapshot.version, snapshot.contents, snapshot.entityUid + + console.log "--- query", query + + clientReceiver = client.query query, (err, results) => + console.log "--- saveSnapshot err", err + p.end() + return @_handleError(err, client, done, callback) if err? + done() + callback null + + _instantiateEventsFromRows: (rows, callback) -> + events = [] + return callback null, events if rows.length is 0 + + rowsQueue = async.queue (row, rowCallback) => + @_instantiateEventFromRow row, (err, event) -> + return callback err if err? + events.push event + defer rowCallback + , 1 + + rowsQueue.drain = -> + callback null, events + + rowsQueue.push rows + + _instantiateEventFromRow: (row, callback) -> + console.log "--- instantiate event ---", typeof row.data, row.data + + data = + uid : row.id + name : row.name + entityUid : row.entity_uid + data : JSON.parse(row.data) + timestamp : row.timestamp + version : row.version + + @_loadAttachmentsFromRow row, (err, attachments) -> + return rowCallback err if err? + + for attachmentName, attachmentBody of attachments + data[attachmentName] = attachmentBody + + event = new Event data + + callback null, event + + _loadAttachmentsFromRow: (row, callback) -> + attachments = {} + for attachmentName, attachmentBody of JSON.parse(row.attachments) + attachments[attachmentName] = attachmentBody.buffer + + callback null, attachments + +module.exports = PostgresqlEventStore diff --git a/lib/snapshot.js b/lib/snapshot.js index f3787be..d7c6738 100644 --- a/lib/snapshot.js +++ b/lib/snapshot.js @@ -17,4 +17,4 @@ Snapshot.makeFromEntity = function (entity) { return snapshot; } -module.exports = Snapshot; \ No newline at end of file +module.exports = Snapshot; diff --git a/package.json b/package.json index ec1457e..845041a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "nano": "~3.3.6", "devnull": "0.0.10", "mongodb": "~1.2.12", + "pg": "~1.0.4", "bson": "~0.1.7", "multipart": "0.1.5", "crypto": "0.0.3", From 1ebf98ba9e8218863789300f28fbd37cba2a3ff5 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Wed, 21 May 2014 23:59:59 +0200 Subject: [PATCH 46/84] Re add createNewUid method in mongodb event store --- lib/domain_repository.coffee | 2 +- lib/event_store/mongodb.coffee | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index bd65234..5e150aa 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -156,8 +156,8 @@ class DomainRepository @logger.log "commit", "saving event \"#{event.name}\" for entity #{event.entityUid}" @store.saveEvent event, (err, event) -> - savedEvents.push event return callback err if err? + savedEvents.push event eventTaskCallback null , 1 diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index a90563c..003b6e9 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -22,6 +22,10 @@ class MongoDbEventStore extends Base @eventCollectionName = "events" @snapshotCollectionName = "snapshots" + createNewUid: (callback) -> + uid = uuid.v4() + callback null, uid + initialize: (callback) -> async.waterfall [ (next) => From 4ae6c1bed772aad7b017915c030b3a32e9eda6d6 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 22 May 2014 00:18:29 +0200 Subject: [PATCH 47/84] Fix postgres event store setup. --- lib/event_store/postgresql.coffee | 46 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index 16e9db2..029a722 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -33,6 +33,18 @@ class PostgresqlEventStore extends Base pgCallback(pgClient) callback(err) + _dropEventTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "DROP TABLE IF EXISTS %s;" + query = format query, @eventTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback() + _createEventTable: (callback) -> pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? @@ -67,6 +79,18 @@ class PostgresqlEventStore extends Base done() callback() + _dropSnapshotTable: (callback) -> + pg.connect @uri, (err, client, done) => + return @_handleError(err, client, done, callback) if err? + + query = "DROP TABLE IF EXISTS %s;" + query = format query, @snapshotTableName + + client.query query, (err) => + return @_handleError(err, client, done, callback) if err? + done() + callback() + _createSnapshotTable: (callback) -> pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? @@ -98,18 +122,6 @@ class PostgresqlEventStore extends Base done() callback null - _emptySnapshotTable: (callback) -> - pg.connect @uri, (err, client, done) => - return @_handleError(err, client, done, callback) if err? - - query = "TRUNCATE TABLE %s RESTART IDENTITY;" - query = format query, @snapshotTableName - - client.query query, (err) => - return @_handleError(err, client, done, callback) if err? - done() - callback null - _addAutoIncrementOnEventVersion: (callback) -> pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? @@ -136,7 +148,6 @@ class PostgresqlEventStore extends Base BEFORE INSERT ON events FOR EACH ROW WHEN (NEW.version IS NULL) EXECUTE PROCEDURE events_version_auto_increment();" - #query = format query, @eventTableName, @eventTableName, @eventTableName client.query query, (err) => return @_handleError(err, client, done, callback) if err? @@ -149,6 +160,9 @@ class PostgresqlEventStore extends Base setup: (callback) -> console.log "Setup postgresql event store" async.series [ + (next) => + console.log "Dropping event table" + @_dropEventTable next (next) => console.log "Creating event table" @_createEventTable next @@ -158,15 +172,15 @@ class PostgresqlEventStore extends Base (next) => console.log "Adding auto increment on event table" @_addAutoIncrementOnEventVersion next + (next) => + console.log "Dropping snapshot table" + @_dropSnapshotTable next (next) => console.log "Creating snapshot table" @_createSnapshotTable next (next) => console.log "Creating index on snapshot table" @_createIndexOnSnapshotTable next - (next) => - console.log "Emptying snapshot table" - @_emptySnapshotTable next ], callback findAllEventsOneByOne: (eventHandler, callback) -> From 7a7349bba9217463e7850f39a6a0daa053330951 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 22 May 2014 09:46:38 +0200 Subject: [PATCH 48/84] Escape strings before inserting them into the event store --- lib/event_store/postgresql.coffee | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index 029a722..e8b8eb3 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -281,6 +281,14 @@ class PostgresqlEventStore extends Base @_find params, order, callback + escapeString: (string) -> + return "NULL" if string is null + hasBackSlash = ~string.indexOf "\\" + prefix = if hasBackSlash then "E" else "" + string = string.replace /'/g, "''" + string = string.replace /\\/g, "\\\\" + prefix + "'" + string + "'" + saveEvent: (event, callback) => console.log "&&&", "saveEvent", event p = new Profiler "PostgresqlEventStore#saveEvent (db request)", @logger @@ -295,11 +303,14 @@ class PostgresqlEventStore extends Base else data[key] = value + data = JSON.stringify data + attachments = JSON.stringify attachments + pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? - query = "INSERT INTO %s (name, entity_uid, timestamp, data, attachments) VALUES ('%s', '%s', %d, '%j', '%j');" - query = format query, @eventTableName, event.name, event.entityUid, event.timestamp, data, attachments + query = "INSERT INTO %s (name, entity_uid, timestamp, data, attachments) VALUES (%s, %s, %d, %s, %s);" + query = format query, @eventTableName, @escapeString(event.name), @escapeString(event.entityUid), event.timestamp, @escapeString(data), @escapeString(attachments) clientReceiver = client.query query, (err, results) => p.end() @@ -348,8 +359,12 @@ class PostgresqlEventStore extends Base pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? - query = "WITH upsert AS (UPDATE %s SET version=%d, contents='%j' WHERE entity_uid='%s' RETURNING *) INSERT INTO %s (version, contents, entity_uid) SELECT %d, '%j', '%s' WHERE NOT EXISTS (SELECT * FROM upsert);" - query = format query, @snapshotTableName, snapshot.version, snapshot.contents, snapshot.entityUid, @snapshotTableName, snapshot.version, snapshot.contents, snapshot.entityUid + version = snapshot.version + contents = JSON.stringify(snapshot.contents) + entityUid = snapshot.entityUid + + query = "WITH upsert AS (UPDATE %s SET version=%d, contents=%s WHERE entity_uid=%s RETURNING *) INSERT INTO %s (version, contents, entity_uid) SELECT %d, %s, %s WHERE NOT EXISTS (SELECT * FROM upsert);" + query = format query, @snapshotTableName, version, @escapeString(contents), @escapeString(entityUid), @snapshotTableName, version, @escapeString(contents), @escapeString(entityUid) console.log "--- query", query From 8f3c848497bcc7ae3619b61de415e5e138eadb24 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 23 May 2014 14:49:39 +0200 Subject: [PATCH 49/84] Refactor event store interface --- lib/domain_repository.coffee | 20 ++++++----- lib/entity_instantiator.js | 38 +++++++++++---------- lib/event_store/base.coffee | 4 +-- lib/event_store/mongodb.coffee | 55 ++++++++++++++++--------------- lib/event_store/postgresql.coffee | 32 ++++++++++-------- 5 files changed, 80 insertions(+), 69 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 5e150aa..3234c08 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -76,13 +76,13 @@ class DomainRepository entityInstantiator = new EntityInstantiator store: @store, Entity: Entity, logger: @logger entityInstantiator.findByUid uid, callback - replayAllEvents: (options, callback) -> + replayAllEvents: (callback) -> return callback new Error("Replay mode not set") unless @replaying + [options, callback] = [{}, options] unless callback? - lastEvent = null + lastEvent = null - @store.findAllEventsOneByOne options, (err, event, eventHandlerCallback) => - return callback err if err? + @store.iterateOverAllEvents options, (event, eventHandlerCallback) => event.replayed = true lastEvent = event @_publishEvent event, eventHandlerCallback @@ -150,7 +150,6 @@ class DomainRepository if firstEvent? queue = async.queue (event, eventTaskCallback) => - console.log "@@@ commit", event nextEvent = entityAppliedEvents.shift() queue.push nextEvent if nextEvent? @@ -188,23 +187,26 @@ class DomainRepository defer => @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => - @logger.warn "publishEvent", "a direct listener failed: #{err}" if err? + @logger.error "publishEvent", "a direct listener failed: #{err}" if err? @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" if @silent callback() else @emitter.emit event, (err) => - @logger.log "publishEvent", "event publication failed: #{err}" if err? + @logger.error "publishEvent", "event publication failed: #{err}" if err? callback err _publishEventToDirectListeners: (event, callback) -> directListeners = @directListeners[event.name] - queue = async.queue (directListener, callback) -> - directListener event, callback + queue = async.queue (directListener, taskCallback) -> + directListener event, (err) -> + return callback err if err? + taskCallback() , Infinity queue.drain = callback + queuedListeners = false for _, directListener of directListeners queuedListeners = true unless queuedListeners diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index f41adf9..d74f4d5 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -27,31 +27,33 @@ EntityInstantiator.prototype.findByUid = function (uid, callback) { }; EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, callback) { - var self = this; - - var entity = DomainObject.deserialize(snapshot.contents); + var self = this; + var entity = DomainObject.deserialize(snapshot.contents); var uid = snapshot.entityUid; var version = snapshot.version; + entity._initializeAtVersion(version); self.logger.log("EntityInstantiator", "looking for events of entity \""+ uid + "\" after version " + version); - self.store.findAllEventsByEntityUidAfterVersion(uid, version, function (err, events) { - self.logger.log("EntityInstantiator", "found " + events.length + " event(s) of entity \""+ uid + "\" after version " + version); + + var eventHandler = function(event, callback) { + entity.applyEvent(event, callback); + }; + + self.store.iterateOverEntityEventsAfterVersion(uid, version, eventHandler, function (err) { if (err) return callback(err); - entity.applyEvents(events, function (err) { - if (err) return callback(err); - self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); - var snapshotAge = entity.$version - version; - if (snapshotAge > SNAPSHOT_MAX_AGE) { - self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \""+ uid + "\""); - self._snapEntity(entity, function(err) { - if (err) callback(err); - callback(null, entity); - }); - } else { + self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); + + var snapshotAge = entity.$version - version; + if (snapshotAge > SNAPSHOT_MAX_AGE) { + self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \""+ uid + "\""); + self._snapEntity(entity, function(err) { + if (err) callback(err); callback(null, entity); - } - }); + }); + } else { + callback(null, entity); + } }); }; diff --git a/lib/event_store/base.coffee b/lib/event_store/base.coffee index 39e7b9c..ce8c548 100644 --- a/lib/event_store/base.coffee +++ b/lib/event_store/base.coffee @@ -9,7 +9,7 @@ class BaseEventStore findAllEvents: (options, callback) -> throw new Error "implement me" - findAllEventsOneByOne: (options, callback) -> + iterateOverAllEvents: (options, callback) -> throw new Error "implement me" findAllEventsByEntityUid: (entityUid, options, callback) -> @@ -26,4 +26,4 @@ class BaseEventStore # implement me if you want snapshots in your store callback? null -module.exports = BaseEventStore \ No newline at end of file +module.exports = BaseEventStore diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 003b6e9..42cbf0c 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -81,45 +81,49 @@ class MongoDbEventStore extends Base uid = uuid.v4() callback null, uid - findAllEventsOneByOne: (options, eventHandler, callback) -> + iterateOverAllEvents: (options, eventHandler, callback) -> [options, eventHandler, callback] = [{}, options, eventHandler] unless callback? - query = {} + query = {} sortQuery = timestamp: 1 uid: 1 - doFindAll = => - p = new Profiler "MongoDbEventStore#_find(db request)", @logger - p.start() - - cursor = @eventCollection.find(query).sort(sortQuery) - retrieve = => - cursor.nextObject (err, item) => - return callback err if err? - if item? - @_instantiateEventFromRow item, (err, event) -> - eventHandler err, event, (err) -> - return callback err if err? - retrieve() - else - p.end() - callback null - retrieve() - startUid = options.startUid if startUid? @eventCollection.findOne uid: startUid, (err, event) -> return callback err if err? - startTimestamp = event.timestamp + query = timestamp: - $gte: startTimestamp + $gte: event.timestamp uid: $ne: startUid - doFindAll() + + @_iterateOverEvents query, sortQuery, eventHandler, callback else - doFindAll() + @_iterateOverEvents query, sortQuery, eventHandler, callback + + iterateOverEntityEventsAfterVersion: (entityUid, version, eventHandler, callback) -> + @_iterateOverEvents {entityUid: entityUid, version: {$gt: version}}, {version:1}, eventHandler, callback + + _iterateOverEvents: (params, order, eventHandler, callback) -> + p = new Profiler "MongoDbEventStore#_iterateOverEvents(db request)", @logger + p.start() + cursor = @eventCollection.find(params).sort(order) + retrieve = => + cursor.nextObject (err, item) => + return callback err if err? + if item? + @_instantiateEventFromRow item, (err, event) -> + return callback err if err? + eventHandler event, (err) -> + return callback err if err? + retrieve() + else + p.end() + callback null + retrieve() findAllEventsByEntityUid: (entityUid, order, callback) -> [order, callback] = [null, order] unless callback? @@ -132,9 +136,6 @@ class MongoDbEventStore extends Base findSomeEventsByEntityUidBeforeVersion: (entityUid, version, eventCount, callback) -> @_findLimited { entityUid: entityUid, version: { "$lte": version } }, eventCount, callback - findAllEventsByEntityUidAfterVersion: (entityUid, version, callback) -> - @_find entityUid: entityUid, version: { $gt: version }, callback - saveEvent: (event, callback) => @createNewUid (err, eventUid) => return callback err if err? diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index e8b8eb3..0001643 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -183,15 +183,27 @@ class PostgresqlEventStore extends Base @_createIndexOnSnapshotTable next ], callback - findAllEventsOneByOne: (eventHandler, callback) -> - p = new Profiler "PostgresqlEventStore#_findAllEventsOneByOne (db request)", @logger + iterateOverAllEvents: (eventHandler, callback) -> + @_iterateOverEvents null, eventHandler, callback + + iterateOverEntityEventsAfterVersion: (entityUid, version, eventHandler, callback) -> + @_iterateOverEvents "entity_uid='#{entityUid}' AND version > #{version}", eventHandler, callback + + _iterateOverEvents: (params, eventHandler, callback) -> + p = new Profiler "PostgresqlEventStore#_iterateOverEvents (db request)", @logger p.start() pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? - query = "FIND * FROM %s ORDER BY id ASC;" - query = format query, @eventTableName + query = "FIND * FROM %s" + query += " WHERE %s" if params? + query += " ORDER BY id ASC;" + + if params? + query = format query, @eventTableName, params + else + query = format query, @eventTableName clientReceiver = client.query query @@ -200,7 +212,9 @@ class PostgresqlEventStore extends Base clientReceiver.on "row", (row) => @_instantiateEventFromRow row, (err, event) -> - eventHandler err, event, (err) -> + return @_handleError(err, client, done, callback) if err? + + eventHandler event, (err) -> return @_handleError(err, client, done, callback) if err? clientReceiver.on "end", (results) => @@ -275,12 +289,6 @@ class PostgresqlEventStore extends Base @_find params, order, eventCount, callback - findAllEventsByEntityUidAfterVersion: (entityUid, version, callback) -> - params = "entity_uid='#{entityUid}' AND version > #{version}" - order = "ASC" - - @_find params, order, callback - escapeString: (string) -> return "NULL" if string is null hasBackSlash = ~string.indexOf "\\" @@ -339,8 +347,6 @@ class PostgresqlEventStore extends Base snapshot = null if rawSnapshot? - console.log "-- raw snapshot --", typeof rawSnapshot.contents, rawSnapshot.contents - snapshotAttributes = version : rawSnapshot.version entityUid : rawSnapshot.entity_uid From f0ca471465e05cfcd6e9128f2463ef91712f5302 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 23 May 2014 15:25:45 +0200 Subject: [PATCH 50/84] Use iterators in entity instantiator where possible --- lib/entity_instantiator.js | 36 ++++++++++++++++--------------- lib/event_store/mongodb.coffee | 3 +++ lib/event_store/postgresql.coffee | 3 +++ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index d74f4d5..7b14f23 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -58,26 +58,28 @@ EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, call }; EntityInstantiator.prototype._instantiateThroughEventsFromUid = function (uid, callback) { - var self = this; + var self = this; + var entity = new self.Entity(); + + var eventHandler = function(event, callback) { + entity.applyEvent(event, callback); + }; - self.store.findAllEventsByEntityUid(uid, function (err, events) { - if (err) { - callback(err); - } else if (events.length == 0) { + self.store.iterateOverEntityEvents(uid, eventHandler, function (err) { + if (err) return callback(err); + + if (entity.$version === 0) callback(null, null); - } else { - self.Entity.buildFromEvents(events, function (err, entity) { - if (err) return callback(err); - self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" at version " + entity.$version + " from " + events.length + " events"); - if (entity.$version > SNAPSHOT_MAX_AGE) { - self._snapEntity(entity, function(err) { - if (err) return callback(err); - callback(null, entity); - }); - } else { + else { + self.logger.log("EntityInstantiator", "instantiated entity \"" + uid + "\" at version " + entity.$version); + if (entity.$version > SNAPSHOT_MAX_AGE) { + self._snapEntity(entity, function(err) { + if (err) return callback(err); callback(null, entity); - } - }); + }); + } else { + callback(null, entity); + } } }); }; diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 42cbf0c..ec0f679 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -107,6 +107,9 @@ class MongoDbEventStore extends Base iterateOverEntityEventsAfterVersion: (entityUid, version, eventHandler, callback) -> @_iterateOverEvents {entityUid: entityUid, version: {$gt: version}}, {version:1}, eventHandler, callback + iterateOverEntityEvents: (entityUid, eventHandler, callback) -> + @_iterateOverEvents {entityUid: entityUid}, {version:1}, eventHandler, callback + _iterateOverEvents: (params, order, eventHandler, callback) -> p = new Profiler "MongoDbEventStore#_iterateOverEvents(db request)", @logger p.start() diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index 0001643..4f27fa1 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -189,6 +189,9 @@ class PostgresqlEventStore extends Base iterateOverEntityEventsAfterVersion: (entityUid, version, eventHandler, callback) -> @_iterateOverEvents "entity_uid='#{entityUid}' AND version > #{version}", eventHandler, callback + iterateOverEntityEvents: (entityUid, eventHandler, callback) -> + @_iterateOverEvents "entity_uid='#{entityUid}'", eventHandler, callback + _iterateOverEvents: (params, eventHandler, callback) -> p = new Profiler "PostgresqlEventStore#_iterateOverEvents (db request)", @logger p.start() From 514fb414c614b0ab118b37ffb34e856e8bf71fde Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 23 May 2014 15:53:05 +0200 Subject: [PATCH 51/84] Ensure inner callback have returned in iterateOverX methods --- lib/event_store/postgresql.coffee | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index 4f27fa1..c255503 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -196,10 +196,13 @@ class PostgresqlEventStore extends Base p = new Profiler "PostgresqlEventStore#_iterateOverEvents (db request)", @logger p.start() + treatedRows = 0 + affectedRows = undefined + pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? - query = "FIND * FROM %s" + query = "SELECT * FROM %s" query += " WHERE %s" if params? query += " ORDER BY id ASC;" @@ -220,10 +223,17 @@ class PostgresqlEventStore extends Base eventHandler event, (err) -> return @_handleError(err, client, done, callback) if err? + treatedRows++ + if treatedRows is affectedRows + return callback null + clientReceiver.on "end", (results) => - p.end() + affectedRows = results.rowCount done() - callback null + + if treatedRows is affectedRows + p.end() + return callback null findAllEvents: (options, callback) -> params = true From 26f78f429b0e8f7b6180bbcc666cbb65df5a79e4 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 23 May 2014 16:05:24 +0200 Subject: [PATCH 52/84] Remove useless log in entity instantiator --- lib/entity_instantiator.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index 7b14f23..aedb837 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -34,8 +34,6 @@ EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, call entity._initializeAtVersion(version); - self.logger.log("EntityInstantiator", "looking for events of entity \""+ uid + "\" after version " + version); - var eventHandler = function(event, callback) { entity.applyEvent(event, callback); }; From 1a602a68f85b2de82036c557662e7de515a2d474 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 16:43:06 +0200 Subject: [PATCH 53/84] Handle errors --- lib/domain_repository.coffee | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 3234c08..5e037ec 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -13,6 +13,7 @@ class DomainRepository throw new Error "Missing store" unless @store? throw new Error "Missing event bus emitter" unless @emitter? throw new Error "Missing logger" unless @logger? + @entityEvents = {} @directListeners = {} @nextDirectListenerKey = 0 @@ -169,14 +170,19 @@ class DomainRepository entityQueue.drain = => return callback null unless savedEvents.length > 0 + publicationQueue = async.queue (event, publicationCallback) => - @_publishEvent event, publicationCallback + @_publishEvent event, (err) -> + return callback err if err? + publicationCallback() , Infinity + publicationQueue.drain = callback publicationQueue.push savedEvents for entityUid, entityEvents of @entityEvents entityQueue.push [entityEvents] + @entityEvents = {} _rollback: (callback) -> @@ -187,7 +193,8 @@ class DomainRepository defer => @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => - @logger.error "publishEvent", "a direct listener failed: #{err}" if err? + return callback err if err? + @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" if @silent callback() From 6b9262ddc53098d7a68e0e40bd2e200159412c6d Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 16:44:45 +0200 Subject: [PATCH 54/84] Re-add log on error --- lib/domain_repository.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 5e037ec..1b10ecf 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -193,7 +193,9 @@ class DomainRepository defer => @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => - return callback err if err? + if err? + @logger.log "publishEvent", "a direct listener failed: #{err}" if err? + return callback err @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" if @silent From 628711d5690275e9896f0348905f6841548de857 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 16:45:27 +0200 Subject: [PATCH 55/84] Use logger warn level --- lib/domain_repository.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 1b10ecf..6c621a3 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -194,7 +194,7 @@ class DomainRepository @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => if err? - @logger.log "publishEvent", "a direct listener failed: #{err}" if err? + @logger.warn "publishEvent", "a direct listener failed: #{err}" if err? return callback err @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" @@ -202,7 +202,7 @@ class DomainRepository callback() else @emitter.emit event, (err) => - @logger.error "publishEvent", "event publication failed: #{err}" if err? + @logger.warn "publishEvent", "event publication failed: #{err}" if err? callback err _publishEventToDirectListeners: (event, callback) -> From db2fcf22743df85cce048df6b111b624d963f319 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 17:23:52 +0200 Subject: [PATCH 56/84] Remove useless logs --- lib/domain_repository.coffee | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 6c621a3..eb0895d 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -193,16 +193,13 @@ class DomainRepository defer => @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => - if err? - @logger.warn "publishEvent", "a direct listener failed: #{err}" if err? - return callback err + return callback err if err? @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" if @silent callback() else @emitter.emit event, (err) => - @logger.warn "publishEvent", "event publication failed: #{err}" if err? callback err _publishEventToDirectListeners: (event, callback) -> From fb21a76a609a5943bef51255ac5dd9b28a1a0960 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 17:24:58 +0200 Subject: [PATCH 57/84] Re add options arguments in replayAllEvents --- lib/domain_repository.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index eb0895d..c3a402d 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -77,7 +77,7 @@ class DomainRepository entityInstantiator = new EntityInstantiator store: @store, Entity: Entity, logger: @logger entityInstantiator.findByUid uid, callback - replayAllEvents: (callback) -> + replayAllEvents: (options, callback) -> return callback new Error("Replay mode not set") unless @replaying [options, callback] = [{}, options] unless callback? From 7ade2fb9b2e0812c06e24bcc6a62641fc258cdb5 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 17:35:26 +0200 Subject: [PATCH 58/84] Snap entity asynchronously --- lib/entity_instantiator.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index aedb837..d5b33d0 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -42,15 +42,14 @@ EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, call if (err) return callback(err); self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); + callback(null, entity); + var snapshotAge = entity.$version - version; if (snapshotAge > SNAPSHOT_MAX_AGE) { - self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \""+ uid + "\""); + self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \"" + uid + "\""); self._snapEntity(entity, function(err) { - if (err) callback(err); - callback(null, entity); + if (err) self.logger.error("EntityInstantiator", "instantiate from snapshot failed", err); }); - } else { - callback(null, entity); } }); }; From 0e5b08f8d011179f5d3e09fe604077121a60f185 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 30 May 2014 17:37:55 +0200 Subject: [PATCH 59/84] Snap entity asynchronously in instantiateFromEvents --- lib/entity_instantiator.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index d5b33d0..6768052 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -69,13 +69,12 @@ EntityInstantiator.prototype._instantiateThroughEventsFromUid = function (uid, c callback(null, null); else { self.logger.log("EntityInstantiator", "instantiated entity \"" + uid + "\" at version " + entity.$version); + callback(null, entity); + if (entity.$version > SNAPSHOT_MAX_AGE) { self._snapEntity(entity, function(err) { - if (err) return callback(err); - callback(null, entity); + if (err) self.logger.error("EntityInstantiator", "instantiate through events from uid failed", err); }); - } else { - callback(null, entity); } } }); From 77667a0ff4697051b06f5c17174a5a2c213e7009 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Wed, 4 Jun 2014 14:18:44 +0200 Subject: [PATCH 60/84] Fix interface in base event store --- lib/event_store/base.coffee | 2 +- lib/event_store/postgresql.coffee | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/event_store/base.coffee b/lib/event_store/base.coffee index ce8c548..619648e 100644 --- a/lib/event_store/base.coffee +++ b/lib/event_store/base.coffee @@ -9,7 +9,7 @@ class BaseEventStore findAllEvents: (options, callback) -> throw new Error "implement me" - iterateOverAllEvents: (options, callback) -> + iterateOverAllEvents: (options, eventHandler, callback) -> throw new Error "implement me" findAllEventsByEntityUid: (entityUid, options, callback) -> diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index c255503..ff1c1e6 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -198,6 +198,7 @@ class PostgresqlEventStore extends Base treatedRows = 0 affectedRows = undefined + returned = false pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? @@ -224,15 +225,18 @@ class PostgresqlEventStore extends Base return @_handleError(err, client, done, callback) if err? treatedRows++ - if treatedRows is affectedRows + if treatedRows is affectedRows and not returned + p.end() + returned = true return callback null clientReceiver.on "end", (results) => affectedRows = results.rowCount done() - if treatedRows is affectedRows + if treatedRows is affectedRows and not returned p.end() + returned = true return callback null findAllEvents: (options, callback) -> From 037e7dac74dfbfee65c4f71469b7ebd832f2510d Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 5 Jun 2014 11:39:26 +0200 Subject: [PATCH 61/84] Refactor entity instantiator and remove console.log --- lib/entity_instantiator.js | 28 ++++++++++++++++------------ lib/event_store/postgresql.coffee | 9 --------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/lib/entity_instantiator.js b/lib/entity_instantiator.js index 6768052..d35cdd3 100644 --- a/lib/entity_instantiator.js +++ b/lib/entity_instantiator.js @@ -40,17 +40,15 @@ EntityInstantiator.prototype._instantiateFromSnapshot = function (snapshot, call self.store.iterateOverEntityEventsAfterVersion(uid, version, eventHandler, function (err) { if (err) return callback(err); - self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); + self.logger.log("EntityInstantiator", "instantiated entity \""+ uid + "\" from snapshot at version " + version); callback(null, entity); var snapshotAge = entity.$version - version; - if (snapshotAge > SNAPSHOT_MAX_AGE) { - self.logger.log("EntityInstantiator", "updating outdated snapshot (" + snapshotAge + " versions) of entity \"" + uid + "\""); - self._snapEntity(entity, function(err) { - if (err) self.logger.error("EntityInstantiator", "instantiate from snapshot failed", err); - }); - } + + self._snapEntityIfNeeded(entity, snapshotAge, function(err) { + if (err) self.logger.error("EntityInstantiator", "Snap entity failed", err); + }); }); }; @@ -71,15 +69,21 @@ EntityInstantiator.prototype._instantiateThroughEventsFromUid = function (uid, c self.logger.log("EntityInstantiator", "instantiated entity \"" + uid + "\" at version " + entity.$version); callback(null, entity); - if (entity.$version > SNAPSHOT_MAX_AGE) { - self._snapEntity(entity, function(err) { - if (err) self.logger.error("EntityInstantiator", "instantiate through events from uid failed", err); - }); - } + self._snapEntityIfNeeded(entity, entity.$version, function(err) { + if (err) self.logger.error("EntityInstantiator", "Snap entity failed", err); + }); } }); }; +EntityInstantiator.prototype._snapEntityIfNeeded = function (entity, snapshotAge, callback) { + if (snapshotAge > SNAPSHOT_MAX_AGE) { + this.logger.log("EntityInstantiator", "updating snapshot (" + snapshotAge + " versions) of entity \"" + entity.uid + "\""); + this._snapEntity(entity, callback); + } else + callback(); +}; + EntityInstantiator.prototype._snapEntity = function (entity, callback) { var self = this; diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index ff1c1e6..0f23bd6 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -268,7 +268,6 @@ class PostgresqlEventStore extends Base return @_handleError(err, client, done, callback) if err? done() @_instantiateEventsFromRows results.rows, (err, events) -> - console.log "-- _find", query, err, events callback err, events _count: (params, callback) -> @@ -315,7 +314,6 @@ class PostgresqlEventStore extends Base prefix + "'" + string + "'" saveEvent: (event, callback) => - console.log "&&&", "saveEvent", event p = new Profiler "PostgresqlEventStore#saveEvent (db request)", @logger p.start() @@ -374,8 +372,6 @@ class PostgresqlEventStore extends Base callback null, snapshot saveSnapshot: (snapshot, callback) -> - console.log "-- saveSnapshot", snapshot - p = new Profiler "PostgresqlEventStore#saveSnapshot (db request)", @logger p.start() @@ -389,10 +385,7 @@ class PostgresqlEventStore extends Base query = "WITH upsert AS (UPDATE %s SET version=%d, contents=%s WHERE entity_uid=%s RETURNING *) INSERT INTO %s (version, contents, entity_uid) SELECT %d, %s, %s WHERE NOT EXISTS (SELECT * FROM upsert);" query = format query, @snapshotTableName, version, @escapeString(contents), @escapeString(entityUid), @snapshotTableName, version, @escapeString(contents), @escapeString(entityUid) - console.log "--- query", query - clientReceiver = client.query query, (err, results) => - console.log "--- saveSnapshot err", err p.end() return @_handleError(err, client, done, callback) if err? done() @@ -415,8 +408,6 @@ class PostgresqlEventStore extends Base rowsQueue.push rows _instantiateEventFromRow: (row, callback) -> - console.log "--- instantiate event ---", typeof row.data, row.data - data = uid : row.id name : row.name From c83173f8056b06d8b290faf1225bd7bb4101a1c8 Mon Sep 17 00:00:00 2001 From: Thibault Poncelet Date: Mon, 16 Jun 2014 14:20:08 +0200 Subject: [PATCH 62/84] Fix fat arrow --- lib/event_store/mongodb.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index ec0f679..7d371ea 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -91,7 +91,7 @@ class MongoDbEventStore extends Base startUid = options.startUid if startUid? - @eventCollection.findOne uid: startUid, (err, event) -> + @eventCollection.findOne uid: startUid, (err, event) => return callback err if err? query = From cdafea67bc32c337ad12ec9da7af4a67d94f0778 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Wed, 18 Jun 2014 09:30:49 +0200 Subject: [PATCH 63/84] Defer recursive call --- lib/event_store/mongodb.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 7d371ea..2f1aa7b 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -122,7 +122,7 @@ class MongoDbEventStore extends Base return callback err if err? eventHandler event, (err) -> return callback err if err? - retrieve() + defer retrieve else p.end() callback null From 10d256adc67088a55a081217d8ac5397cb7faee7 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 4 Sep 2014 15:50:24 +0200 Subject: [PATCH 64/84] Align lines --- lib/event_store/mongodb.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index 2f1aa7b..def9288 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -265,12 +265,12 @@ class MongoDbEventStore extends Base rowsQueue.push rows _instantiateEventFromRow: (row, callback) -> - uid = row.uid - name = row.name + uid = row.uid + name = row.name entityUid = row.entityUid - data = row.data - timestamp = row.timestamp - version = row.version + data = row.data + timestamp = row.timestamp + version = row.version @_loadAttachmentsFromRow row, (err, attachments) -> return rowCallback err if err? From 87ea7e1e835d7106e5a113a201144b6931ea51ac Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 4 Sep 2014 17:17:03 +0200 Subject: [PATCH 65/84] Add warning to debug command stacking --- lib/command_bus.coffee | 4 ++++ lib/domain_repository.coffee | 33 ++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index 4d675bd..d61a8b7 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -58,12 +58,16 @@ class CommandBus p.end() done args... transaction.callback = callback + # TODO: remove + transaction.commandHandler = commandHandler else transaction = (done) -> p.start() commandHandler.run (args...) -> p.end() done args... + # TODO: remove + transaction.commandHandler = commandHandler domainRepository.queueTransaction transaction logger.log "CommandBus#executeCommand", "transaction for command '#{command.getName()}' queued" diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index c3a402d..ba38286 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -30,21 +30,24 @@ class DomainRepository @logger.log "transaction", "starting" p = new Profiler "DomainRepository(transactionQueue.operation)", @logger p.start() - transaction (err) => - p.end() - if err? - @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" - @_rollback => - @logger.log "transaction", "rolled back (#{@transactionQueue.length()} more transaction(s) in queue)" - @transacting = false - done() - else - @logger.log "transaction", "succeeded, comitting" - @_commit (err) => - throw err if err? - @logger.log "transaction", "committed (#{@transactionQueue.length()} more transaction(s) in queue)" - @transacting = false - done() + # TODO: remove + @logger.warning "transaction", "exec", transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler + defer => + transaction (err) => + p.end() + if err? + @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" + @_rollback => + @logger.log "transaction", "rolled back (#{@transactionQueue.length()} more transaction(s) in queue)" + @transacting = false + done() + else + @logger.log "transaction", "succeeded, comitting" + @_commit (err) => + throw err if err? + @logger.log "transaction", "committed (#{@transactionQueue.length()} more transaction(s) in queue)" + @transacting = false + done() , 1 queueTransaction: (transaction) -> From 354bc1a56a4cc02f27274a617510ca7422185971 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 4 Sep 2014 17:33:04 +0200 Subject: [PATCH 66/84] Add logs --- lib/domain_repository.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index ba38286..610ac67 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -34,6 +34,8 @@ class DomainRepository @logger.warning "transaction", "exec", transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler defer => transaction (err) => + # TODO: remove + @logger.warning "transaction", "ended", err, transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler p.end() if err? @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" From 5a9501be5fd962b73a5a5708d120f9b6cee51ade Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 4 Sep 2014 21:37:33 +0200 Subject: [PATCH 67/84] Add logger info for debug purpose --- lib/command_bus.coffee | 7 +++++++ lib/domain_repository.coffee | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index d61a8b7..def124b 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -22,6 +22,7 @@ class CommandBus createNewUid: (callback) -> @domainRepository.createNewUid callback + # TODO: remove logger.info executeCommand: (command, callback) -> domainRepository = @domainRepository logger = @logger @@ -45,9 +46,12 @@ class CommandBus transaction = (done) -> p.start() p1.start() + logger.info "executeCommand", "start transaction with validation" commandHandler.validate (err, result) -> p1.end() + logger.info "executeCommand", "validation done" if err? + logger.warning "executeCommand", "validation error", err callback err done err return @@ -56,6 +60,7 @@ class CommandBus commandHandler.run (args...) -> p2.end() p.end() + logger.info "executeCommand", "execution done" done args... transaction.callback = callback # TODO: remove @@ -63,8 +68,10 @@ class CommandBus else transaction = (done) -> p.start() + logger.info "executeCommand", "start transaction without validation" commandHandler.run (args...) -> p.end() + logger.info "executeCommand", "execution done (no validation)" done args... # TODO: remove transaction.commandHandler = commandHandler diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 610ac67..b39de67 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -31,11 +31,12 @@ class DomainRepository p = new Profiler "DomainRepository(transactionQueue.operation)", @logger p.start() # TODO: remove - @logger.warning "transaction", "exec", transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler + @logger.info "transaction", "exec", transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler + @logger.info "transaction code", transaction defer => transaction (err) => # TODO: remove - @logger.warning "transaction", "ended", err, transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler + @logger.info "transaction", "ended", err, transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler p.end() if err? @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" From f0d8f4bd2db16edbdf4ebe287a35d79ac241017e Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 4 Sep 2014 22:27:52 +0200 Subject: [PATCH 68/84] Remove logs used for debugging purpose --- lib/command_bus.coffee | 10 ---------- lib/command_bus_server.coffee | 6 +++++- lib/domain_repository.coffee | 5 ----- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/lib/command_bus.coffee b/lib/command_bus.coffee index def124b..9684dbd 100644 --- a/lib/command_bus.coffee +++ b/lib/command_bus.coffee @@ -22,7 +22,6 @@ class CommandBus createNewUid: (callback) -> @domainRepository.createNewUid callback - # TODO: remove logger.info executeCommand: (command, callback) -> domainRepository = @domainRepository logger = @logger @@ -46,10 +45,8 @@ class CommandBus transaction = (done) -> p.start() p1.start() - logger.info "executeCommand", "start transaction with validation" commandHandler.validate (err, result) -> p1.end() - logger.info "executeCommand", "validation done" if err? logger.warning "executeCommand", "validation error", err callback err @@ -60,21 +57,14 @@ class CommandBus commandHandler.run (args...) -> p2.end() p.end() - logger.info "executeCommand", "execution done" done args... transaction.callback = callback - # TODO: remove - transaction.commandHandler = commandHandler else transaction = (done) -> p.start() - logger.info "executeCommand", "start transaction without validation" commandHandler.run (args...) -> p.end() - logger.info "executeCommand", "execution done (no validation)" done args... - # TODO: remove - transaction.commandHandler = commandHandler domainRepository.queueTransaction transaction logger.log "CommandBus#executeCommand", "transaction for command '#{command.getName()}' queued" diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index 4f38c72..159de63 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -87,6 +87,10 @@ class CommandBusServer logger.log "CommandBusServer", "deserialize command \"#{commandName}\"" @commandBus.deserializeCommand commandName, payload, (err, command) => + if err? + logger.error "CommandBusServer", "error while deserializing command", err + djump res, 500, error: err + @commandBus.executeCommand command, (err) -> if err? logger.alert "CommandBusServer", "error while executing command (#{err})" @@ -103,4 +107,4 @@ djump = (res, code, obj) -> else res.end() -module.exports = CommandBusServer \ No newline at end of file +module.exports = CommandBusServer diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index b39de67..86d58f9 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -30,13 +30,8 @@ class DomainRepository @logger.log "transaction", "starting" p = new Profiler "DomainRepository(transactionQueue.operation)", @logger p.start() - # TODO: remove - @logger.info "transaction", "exec", transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler - @logger.info "transaction code", transaction defer => transaction (err) => - # TODO: remove - @logger.info "transaction", "ended", err, transaction.commandHandler.constructor.getCommandName(), "->", transaction.commandHandler p.end() if err? @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" From 55586c5117f3b4526b4c5519cacd1782e7aa0fce Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 5 Sep 2014 02:55:12 +0200 Subject: [PATCH 69/84] Add readOnly option in assembler to avoid having the commandBus server listening on a port --- lib/assembler.js | 21 ++++++++++++--------- lib/profiler.js | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 2558f5c..37fae1c 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -45,10 +45,12 @@ Assembler.prototype.loadCompleteApp = function (options, callback) { options.setupReportStores = false; if (typeof options.startReporters == "undefined") options.startReporters = true; + if (typeof options.readOnly == "undefined") + options.readOnly = false; async.series([ function (next) { - self.assembleApp(next); + self.assembleApp(options, next); }, function (next) { if (options.clearQueues) @@ -86,7 +88,7 @@ Assembler.prototype.loadCompleteApp = function (options, callback) { }; // Assemble an app only, no reporters -Assembler.prototype.assembleApp = function (callback) { +Assembler.prototype.assembleApp = function (options, callback) { var self = this; if (self.app) @@ -111,7 +113,7 @@ Assembler.prototype.assembleApp = function (callback) { }; if (self.configuration.domain) { - self._assembleDomainInfrastructure(function(err){ + self._assembleDomainInfrastructure(options, function(err){ if (err) return callback(err); appOptions.domainRepository = self.domainRepository; @@ -363,7 +365,7 @@ Assembler.prototype.unloadEventBusReceivers = function (callback) { queue.push(self.eventBusReceivers); }; -Assembler.prototype._assembleDomainInfrastructure = function (callback) { +Assembler.prototype._assembleDomainInfrastructure = function (options, callback) { var self = this; var EventStore = self.configuration.domain.eventStore.ctor; @@ -372,7 +374,7 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { logger: self.logger }); - function initializeDomainInfrastructure(callback) { + function initializeDomainInfrastructure(options, callback) { var EventBusEmitter = self.configuration.domain.eventBusEmitter.ctor; var eventBusEmitterOptions = { @@ -395,9 +397,10 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { logger: self.logger, replaying: self.replaying }); + var commandBus = new CommandBus({ domainRepository: domainRepository, - port: self.configuration.domain.commandBus && self.configuration.domain.commandBus.port, + port: !options.readOnly && self.configuration.domain.commandBus && self.configuration.domain.commandBus.port, logger: self.logger, replaying: self.replaying }); @@ -438,11 +441,11 @@ Assembler.prototype._assembleDomainInfrastructure = function (callback) { eventStore.initialize(function(err){ if (err) return callback(err); - initializeDomainInfrastructure(callback); + initializeDomainInfrastructure(options, callback); }); } else { - initializeDomainInfrastructure(callback); + initializeDomainInfrastructure(options, callback); } }; -module.exports = Assembler; \ No newline at end of file +module.exports = Assembler; diff --git a/lib/profiler.js b/lib/profiler.js index 642f3a0..f84a322 100644 --- a/lib/profiler.js +++ b/lib/profiler.js @@ -26,4 +26,4 @@ function _padLeft(str, length) { return str; }; -module.exports = Profiler; \ No newline at end of file +module.exports = Profiler; From a31d46b654cc17d727f7610d4afb5a8e7992823e Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 5 Sep 2014 10:07:21 +0200 Subject: [PATCH 70/84] Update async version --- lib/domain_repository.coffee | 31 +++++++++++++++---------------- package.json | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 86d58f9..c3a402d 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -30,22 +30,21 @@ class DomainRepository @logger.log "transaction", "starting" p = new Profiler "DomainRepository(transactionQueue.operation)", @logger p.start() - defer => - transaction (err) => - p.end() - if err? - @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" - @_rollback => - @logger.log "transaction", "rolled back (#{@transactionQueue.length()} more transaction(s) in queue)" - @transacting = false - done() - else - @logger.log "transaction", "succeeded, comitting" - @_commit (err) => - throw err if err? - @logger.log "transaction", "committed (#{@transactionQueue.length()} more transaction(s) in queue)" - @transacting = false - done() + transaction (err) => + p.end() + if err? + @logger.alert "transaction", "failed, rolling back (#{err.stack || util.inspect(err)})" + @_rollback => + @logger.log "transaction", "rolled back (#{@transactionQueue.length()} more transaction(s) in queue)" + @transacting = false + done() + else + @logger.log "transaction", "succeeded, comitting" + @_commit (err) => + throw err if err? + @logger.log "transaction", "committed (#{@transactionQueue.length()} more transaction(s) in queue)" + @transacting = false + done() , 1 queueTransaction: (transaction) -> diff --git a/package.json b/package.json index 845041a..d379de8 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ ], "dependencies": { "coffee-script": "1.7.1", - "async": "0.1.22", + "async": "0.9.0", "node-uuid": "1.3.3", "redis": "0.9.0", "hiredis": "0.1.16", From de7a63b16668e2b8e5be662aada10540e0f44e45 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Fri, 5 Sep 2014 15:11:57 +0200 Subject: [PATCH 71/84] Change postgres event store --- lib/event_store/postgresql.coffee | 36 ++++++++++--------------------- package.json | 2 +- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/lib/event_store/postgresql.coffee b/lib/event_store/postgresql.coffee index 0f23bd6..f406c32 100644 --- a/lib/event_store/postgresql.coffee +++ b/lib/event_store/postgresql.coffee @@ -158,28 +158,27 @@ class PostgresqlEventStore extends Base callback null setup: (callback) -> - console.log "Setup postgresql event store" async.series [ (next) => - console.log "Dropping event table" + @logger.info "Postgres event store", "Dropping event table" @_dropEventTable next (next) => - console.log "Creating event table" + @logger.info "Postgres event store", "Creating event table" @_createEventTable next (next) => - console.log "Creating index on event table" + @logger.info "Postgres event store", "Creating index on event table" @_createIndexOnEventTable next (next) => - console.log "Adding auto increment on event table" + @logger.info "Postgres event store", "Adding auto increment on event table" @_addAutoIncrementOnEventVersion next (next) => - console.log "Dropping snapshot table" + @logger.info "Postgres event store", "Dropping snapshot table" @_dropSnapshotTable next (next) => - console.log "Creating snapshot table" + @logger.info "Postgres event store", "Creating snapshot table" @_createSnapshotTable next (next) => - console.log "Creating index on snapshot table" + @logger.info "Postgres event store", "Creating index on snapshot table" @_createIndexOnSnapshotTable next ], callback @@ -196,10 +195,6 @@ class PostgresqlEventStore extends Base p = new Profiler "PostgresqlEventStore#_iterateOverEvents (db request)", @logger p.start() - treatedRows = 0 - affectedRows = undefined - returned = false - pg.connect @uri, (err, client, done) => return @_handleError(err, client, done, callback) if err? @@ -215,7 +210,8 @@ class PostgresqlEventStore extends Base clientReceiver = client.query query clientReceiver.on "error", (err) => - return @_handleError(err, client, done, callback) if err? + p.end() + return @_handleError(err, client, done, callback) clientReceiver.on "row", (row) => @_instantiateEventFromRow row, (err, event) -> @@ -224,20 +220,10 @@ class PostgresqlEventStore extends Base eventHandler event, (err) -> return @_handleError(err, client, done, callback) if err? - treatedRows++ - if treatedRows is affectedRows and not returned - p.end() - returned = true - return callback null - clientReceiver.on "end", (results) => - affectedRows = results.rowCount + p.end() done() - - if treatedRows is affectedRows and not returned - p.end() - returned = true - return callback null + callback() findAllEvents: (options, callback) -> params = true diff --git a/package.json b/package.json index d379de8..3afffb4 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "nano": "~3.3.6", "devnull": "0.0.10", "mongodb": "~1.2.12", - "pg": "~1.0.4", + "pg": "~3.4.2", "bson": "~0.1.7", "multipart": "0.1.5", "crypto": "0.0.3", From 6ebc026d2274208926ad890f293542463b100e40 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sat, 6 Sep 2014 12:56:04 +0200 Subject: [PATCH 72/84] Return in case of error --- lib/command_bus_server.coffee | 2 +- lib/entity.coffee | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/command_bus_server.coffee b/lib/command_bus_server.coffee index 159de63..62adad8 100644 --- a/lib/command_bus_server.coffee +++ b/lib/command_bus_server.coffee @@ -89,7 +89,7 @@ class CommandBusServer @commandBus.deserializeCommand commandName, payload, (err, command) => if err? logger.error "CommandBusServer", "error while deserializing command", err - djump res, 500, error: err + return djump res, 500, error: err @commandBus.executeCommand command, (err) -> if err? diff --git a/lib/entity.coffee b/lib/entity.coffee index 152597c..82984c0 100644 --- a/lib/entity.coffee +++ b/lib/entity.coffee @@ -1,4 +1,3 @@ -async = require "async" Q = require "q" DomainObject = require "./domain_object" inherit = require "./inherit" From 377627b50db8790d532ff5552fb0c79f6afea288 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sun, 7 Sep 2014 01:19:33 +0200 Subject: [PATCH 73/84] Improve publish to direct listener --- lib/domain_repository.coffee | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index c3a402d..74ceb6b 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -203,20 +203,22 @@ class DomainRepository callback err _publishEventToDirectListeners: (event, callback) -> - directListeners = @directListeners[event.name] + directListeners = @directListeners[event.name] + pending = 0 + errors = [] + queuedListeners = false - queue = async.queue (directListener, taskCallback) -> - directListener event, (err) -> - return callback err if err? - taskCallback() - , Infinity - - queue.drain = callback - - queuedListeners = false for _, directListener of directListeners queuedListeners = true unless queuedListeners - queue.push directListener + pending++ + directListener event, (err) -> + if err? + @logger.error "DomainRepository#_publishEventToDirectListeners", "a direct listener failed: #{err}" + errors.push err + pending-- + if pending is 0 + error = if errors.length > 0 then new Error "Some direct listener(s) failed" else null + callback error callback null unless queuedListeners From c04fa97eca04616e3c082d2c1b9533b7d6b8f794 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sun, 7 Sep 2014 13:49:18 +0200 Subject: [PATCH 74/84] Remove defer where useless --- lib/domain_repository.coffee | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 74ceb6b..e95f764 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -172,9 +172,10 @@ class DomainRepository return callback null unless savedEvents.length > 0 publicationQueue = async.queue (event, publicationCallback) => - @_publishEvent event, (err) -> - return callback err if err? - publicationCallback() + defer => + @_publishEvent event, (err) -> + return callback err if err? + publicationCallback() , Infinity publicationQueue.drain = callback @@ -190,17 +191,16 @@ class DomainRepository callback() _publishEvent: (event, callback) -> - defer => - @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" - @_publishEventToDirectListeners event, (err) => - return callback err if err? - - @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" - if @silent - callback() - else - @emitter.emit event, (err) => - callback err + @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" + @_publishEventToDirectListeners event, (err) => + return callback err if err? + + @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" + if @silent + callback() + else + @emitter.emit event, (err) => + callback err _publishEventToDirectListeners: (event, callback) -> directListeners = @directListeners[event.name] From 8623527b345085442c2bfc8ea038d85d6a2895de Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sun, 7 Sep 2014 13:51:36 +0200 Subject: [PATCH 75/84] Change version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3afffb4..aa7e80c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "author": "Julien Biezemans ", "name": "plutonium", - "version": "0.0.0-alpha", + "version": "0.0.1", "description": "Infrastructure framework based on DDD, Event Sourcing and CQRS principles", "main": "lib/main", "scripts": { From 397aeb0bb142be747633256ff0a681586c6b6fb9 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sun, 7 Sep 2014 20:59:54 +0200 Subject: [PATCH 76/84] Remove unused constants --- lib/domain_repository.coffee | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index e95f764..d244743 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -4,9 +4,6 @@ EntityInstantiator = require "./entity_instantiator" util = require "util" defer = require "./defer" -COUCHDB_STORE = "couchdb" -REDIS_STORE = "redis" - class DomainRepository constructor: ({@store, @emitter, @logger, @replaying}) -> From b42cbbc833c57e9cea3f03140122e74a109b0484 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Sun, 7 Sep 2014 21:02:37 +0200 Subject: [PATCH 77/84] Personal implementation of a queue system --- lib/queue.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 lib/queue.js diff --git a/lib/queue.js b/lib/queue.js new file mode 100644 index 0000000..379ac48 --- /dev/null +++ b/lib/queue.js @@ -0,0 +1,37 @@ +function Queue(worker, concurrency) { + var workers = 0; + var q = { + tasks : [], + concurrency : concurrency, + drain : null, + push : function(data) { + if (!(data instanceof Array)) + data = [data]; + if (data.length == 0 && q.drain) + setImmediate(q.drain); + for (i = 0; i < data.length; i++) { + q.tasks.push(data[i]); + setImmediate(q.process); + } + }, + process : function() { + if (workers < q.concurrency && q.tasks.length) { + var task = q.tasks.shift(); + workers++; + var next = function () { + workers--; + if (q.drain && q.tasks.length + workers === 0) + q.drain(); + setImmediate(q.process); + }; + worker(task, next); + } + }, + length: function () { + return q.tasks.length; + } + }; + return q; +} + +module.exports = Queue; From 83b42174fa6ecaa0d060c9dd010126588b07e834 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Tue, 9 Sep 2014 12:04:02 +0200 Subject: [PATCH 78/84] Use own queue implementation that's faster than async.queue --- lib/domain_repository.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index d244743..71536b6 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -1,8 +1,8 @@ -async = require "async" Profiler = require "./profiler" EntityInstantiator = require "./entity_instantiator" util = require "util" defer = require "./defer" +Queue = require "./queue" class DomainRepository @@ -17,7 +17,7 @@ class DomainRepository @transacting = false @halted = false @silent = false - @transactionQueue = async.queue (transaction, done) => + @transactionQueue = new Queue (transaction, done) => if @halted @logger.warning "transaction", "skipped (#{@transactionQueue.length()} more transaction(s) in queue)" transaction.callback() if transaction.callback? @@ -143,11 +143,11 @@ class DomainRepository savedEvents = []; - entityQueue = async.queue (entityAppliedEvents, entityTaskCallback) => + entityQueue = new Queue (entityAppliedEvents, entityTaskCallback) => firstEvent = entityAppliedEvents.shift() if firstEvent? - queue = async.queue (event, eventTaskCallback) => + queue = new Queue (event, eventTaskCallback) => nextEvent = entityAppliedEvents.shift() queue.push nextEvent if nextEvent? @@ -168,7 +168,7 @@ class DomainRepository entityQueue.drain = => return callback null unless savedEvents.length > 0 - publicationQueue = async.queue (event, publicationCallback) => + publicationQueue = new Queue (event, publicationCallback) => defer => @_publishEvent event, (err) -> return callback err if err? From 71ed00eb534fce9706fd46012d7a1d13b259f04a Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Mon, 29 Sep 2014 10:10:54 +0200 Subject: [PATCH 79/84] Refactor call to callback --- lib/assembler.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index 37fae1c..dbbe61a 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -124,9 +124,7 @@ Assembler.prototype.assembleApp = function (options, callback) { }); }); } else { - createApp(function() { - callback(); - }); + createApp(callback); } }; From 9e0d7dc7234eb4f8f5837a9f94e77955ab5bac55 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Mon, 29 Sep 2014 10:20:23 +0200 Subject: [PATCH 80/84] Do not use defer in Queue worker anymore --- lib/domain_repository.coffee | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 71536b6..396defb 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -1,7 +1,6 @@ Profiler = require "./profiler" EntityInstantiator = require "./entity_instantiator" util = require "util" -defer = require "./defer" Queue = require "./queue" class DomainRepository @@ -169,10 +168,9 @@ class DomainRepository return callback null unless savedEvents.length > 0 publicationQueue = new Queue (event, publicationCallback) => - defer => - @_publishEvent event, (err) -> - return callback err if err? - publicationCallback() + @_publishEvent event, (err) -> + return callback err if err? + publicationCallback() , Infinity publicationQueue.drain = callback From d86e832c10594d851b56209dc3a189480e0ec6c0 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 2 Oct 2014 11:33:55 +0200 Subject: [PATCH 81/84] Fix readOnly assembler option --- lib/assembler.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/assembler.js b/lib/assembler.js index dbbe61a..b355558 100644 --- a/lib/assembler.js +++ b/lib/assembler.js @@ -396,11 +396,15 @@ Assembler.prototype._assembleDomainInfrastructure = function (options, callback) replaying: self.replaying }); + var port; + if (!options.readOnly) + port = self.configuration.domain.commandBus && self.configuration.domain.commandBus.port; + var commandBus = new CommandBus({ - domainRepository: domainRepository, - port: !options.readOnly && self.configuration.domain.commandBus && self.configuration.domain.commandBus.port, - logger: self.logger, - replaying: self.replaying + domainRepository : domainRepository, + port : port, + logger : self.logger, + replaying : self.replaying }); // These initializations take place on the constructor, not instances. From ae264ad7156e709fb0c00c1483398d4e618c4c26 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 25 Sep 2014 15:55:24 +0200 Subject: [PATCH 82/84] Fix logger scope and use batchSize on cursor for replay --- lib/domain_repository.coffee | 7 ++++--- lib/event_bus/redis/emitter.js | 2 +- lib/event_store/mongodb.coffee | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 396defb..f6b5c7d 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -186,11 +186,9 @@ class DomainRepository callback() _publishEvent: (event, callback) -> - @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" @_publishEventToDirectListeners event, (err) => return callback err if err? - @logger.log "publishEvent", "publishing \"#{event.name}\" from entity #{event.entityUid} to event bus" if @silent callback() else @@ -202,13 +200,16 @@ class DomainRepository pending = 0 errors = [] queuedListeners = false + logger = @logger + + logger.log "DomainRepository#_publishEventToDirectListeners", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" for _, directListener of directListeners queuedListeners = true unless queuedListeners pending++ directListener event, (err) -> if err? - @logger.error "DomainRepository#_publishEventToDirectListeners", "a direct listener failed: #{err}" + logger.error "DomainRepository#_publishEventToDirectListeners", "a direct listener failed: #{err}" errors.push err pending-- if pending is 0 diff --git a/lib/event_bus/redis/emitter.js b/lib/event_bus/redis/emitter.js index 340199a..6df31ba 100644 --- a/lib/event_bus/redis/emitter.js +++ b/lib/event_bus/redis/emitter.js @@ -48,7 +48,7 @@ RedisEventBusEmitter.prototype.emit = function (event, callback) { self.queueManager.hget(subscribedKey, event.name, function (err, subscribed){ if (err) return callback(err); if (subscribed) { - self.logger.log("RedisEventBusEmitter", "pushing event \"" + event.name + "\" to key \"" + key + "\""); + self.logger.log("RedisEventBusEmitter", "pushing event \"" + event.name + "\" from entity " + event.entityUid + " to key \"" + key + "\""); transaction.lpush(key, value); self.lastEmittedEvents[queueName] = event; } diff --git a/lib/event_store/mongodb.coffee b/lib/event_store/mongodb.coffee index def9288..c84c4b6 100644 --- a/lib/event_store/mongodb.coffee +++ b/lib/event_store/mongodb.coffee @@ -113,7 +113,7 @@ class MongoDbEventStore extends Base _iterateOverEvents: (params, order, eventHandler, callback) -> p = new Profiler "MongoDbEventStore#_iterateOverEvents(db request)", @logger p.start() - cursor = @eventCollection.find(params).sort(order) + cursor = @eventCollection.find(params).batchSize(1000).sort(order) retrieve = => cursor.nextObject (err, item) => return callback err if err? From 8f238496e5b193b65c68f5852adb68e385149851 Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 25 Sep 2014 15:59:09 +0200 Subject: [PATCH 83/84] Only log the publishing to directlisteners information when an event is actually pushed --- lib/domain_repository.coffee | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index f6b5c7d..6cc0e27 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -202,10 +202,11 @@ class DomainRepository queuedListeners = false logger = @logger - logger.log "DomainRepository#_publishEventToDirectListeners", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" - for _, directListener of directListeners - queuedListeners = true unless queuedListeners + if !queuedListeners + queuedListeners = true + logger.log "DomainRepository#_publishEventToDirectListeners", "publishing \"#{event.name}\" from entity #{event.entityUid} to direct listeners" + pending++ directListener event, (err) -> if err? From 10e6ff40a39023926f8d035ed6cbc3098f98db1e Mon Sep 17 00:00:00 2001 From: David Jeusette Date: Thu, 25 Sep 2014 22:39:14 +0200 Subject: [PATCH 84/84] Refactor publish event method --- lib/domain_repository.coffee | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/domain_repository.coffee b/lib/domain_repository.coffee index 6cc0e27..31bbae4 100644 --- a/lib/domain_repository.coffee +++ b/lib/domain_repository.coffee @@ -188,12 +188,13 @@ class DomainRepository _publishEvent: (event, callback) -> @_publishEventToDirectListeners event, (err) => return callback err if err? + @_publishEventToEventBus event, callback - if @silent - callback() - else - @emitter.emit event, (err) => - callback err + _publishEventToEventBus: (event, callback) -> + if @silent + callback null + else + @emitter.emit event, callback _publishEventToDirectListeners: (event, callback) -> directListeners = @directListeners[event.name]