diff --git a/Gemfile b/Gemfile index fa75df1..426de69 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,6 @@ source 'https://rubygems.org' - gemspec + +gem 'pry' +gem 'pry-nav' +gem 'pry-stack_explorer' diff --git a/app/jobs/cangaroo/job.rb b/app/jobs/cangaroo/job.rb index d1bf58a..119f900 100644 --- a/app/jobs/cangaroo/job.rb +++ b/app/jobs/cangaroo/job.rb @@ -1,5 +1,6 @@ module Cangaroo class Job < ActiveJob::Base + include Cangaroo::Log include Cangaroo::ClassConfiguration queue_as :cangaroo @@ -10,6 +11,8 @@ class Job < ActiveJob::Base class_configuration :process_response, true def perform(*) + log.set_context(job: self) + restart_flow(connection_request) end @@ -21,11 +24,32 @@ def transform { type.singularize => payload } end - protected + def payload_state + other_translation = translation.related_translations.first + + if other_translation.present? + :updated + else + :new + end + end + + protected if !Rails.env.test? def connection_request - Cangaroo::Webhook::Client.new(destination_connection, path) - .post(transform, @job_id, parameters) + translation.save! + + log.info 'attempting translation', + destination_connection: destination_connection.name, + path: path, + translation: translation + + response = Cangaroo::Webhook::Client.new(destination_connection, path) + .post(transform, @job_id, parameters, translation) + + translation.update_column :response, (response.blank?) ? {} : response + + response end def restart_flow(response) @@ -36,6 +60,11 @@ def restart_flow(response) return end + if response.blank? + log.info 'blank response; not processing' + return + end + PerformFlow.call( source_connection: destination_connection, json_body: response.to_json, @@ -58,5 +87,23 @@ def payload def destination_connection @connection ||= Cangaroo::Connection.find_by!(name: connection) end + + def translation + # NOTE @job_id will remain consistent across retries + # TODO we should move this logic to the translation model + + @translation ||= Cangaroo::Translation.where(job_id: @job_id).first_or_initialize( + # TODO use job in place of destination connection + # TODO use source job is place of source connection + # ^ this will provide more detail to the user + + source_connection: source_connection, + destination_connection: destination_connection, + + object_type: self.type, + + request: self.payload + ) + end end end diff --git a/app/models/cangaroo/attempt.rb b/app/models/cangaroo/attempt.rb new file mode 100644 index 0000000..a2f6be9 --- /dev/null +++ b/app/models/cangaroo/attempt.rb @@ -0,0 +1,3 @@ +class Cangaroo::Attempt < ActiveRecord::Base + belongs_to :translation +end diff --git a/app/models/cangaroo/translation.rb b/app/models/cangaroo/translation.rb new file mode 100644 index 0000000..224180c --- /dev/null +++ b/app/models/cangaroo/translation.rb @@ -0,0 +1,63 @@ +module Cangaroo + class Translation < ActiveRecord::Base + include Cangaroo::Log + + belongs_to :destination_connection, class_name: 'Cangaroo::Connection' + belongs_to :source_connection, class_name: 'Cangaroo::Connection' + + has_many :attempts + + def request=(payload) + super + + self.object_id = nil + self.object_key = nil + + determine_payload_identifier + + self.request + end + + def successful? + !!self.response + end + + def related_translations + Cangaroo::Translation.where( + object_type: self.object_type, + object_key: self.object_key, + object_id: self.object_id, + ) + .where.not(job_id: self.job_id) + end + + def determine_object_key_from_payload + Rails.configuration.cangaroo.payload_keys.each do |payload_key| + if self.request[payload_key].present? + return payload_key + end + end + + nil + end + + def determine_payload_identifier + self.object_key = determine_object_key_from_payload + + if self.object_key + self.object_id = self.request[self.object_key] + else + log.info 'unable to find primary key', translation: self + end + end + + def retry + Cangaroo::PerformJobs.call( + # TODO this should be abstracted away into a interator that accepts json payloads + json_body: { self.object_type => [ self.request ] }.to_json, + source_connection: self.source_connection, + jobs: Rails.configuration.cangaroo.jobs + ) + end + end +end diff --git a/cangaroo.gemspec b/cangaroo.gemspec index 66eb502..69b8aa6 100644 --- a/cangaroo.gemspec +++ b/cangaroo.gemspec @@ -36,6 +36,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rubocop' s.add_development_dependency 'shoulda-matchers' s.add_development_dependency 'simplecov' - s.add_development_dependency 'sqlite3' + s.add_development_dependency 'pg' s.add_development_dependency 'webmock' + s.add_development_dependency 'faker' end diff --git a/db/migrate/20160405214735_create_translations.rb b/db/migrate/20160405214735_create_translations.rb new file mode 100644 index 0000000..33c8967 --- /dev/null +++ b/db/migrate/20160405214735_create_translations.rb @@ -0,0 +1,22 @@ +class CreateTranslations < ActiveRecord::Migration + def change + create_table :cangaroo_translations do |t| + t.references :source_connection, index: true + t.references :destination_connection, index: true + + t.string :job_id + + t.string :object_type + t.string :object_id + t.string :object_key + + t.jsonb :request + t.jsonb :response + + t.timestamps null: false + end + + add_foreign_key :cangaroo_translations, :cangaroo_connections, column: :source_connection_id + add_foreign_key :cangaroo_translations, :cangaroo_connections, column: :destination_connection_id + end +end diff --git a/db/migrate/20160406003211_create_cangaroo_attempts.rb b/db/migrate/20160406003211_create_cangaroo_attempts.rb new file mode 100644 index 0000000..418aa99 --- /dev/null +++ b/db/migrate/20160406003211_create_cangaroo_attempts.rb @@ -0,0 +1,13 @@ +class CreateCangarooAttempts < ActiveRecord::Migration + def change + create_table :cangaroo_attempts do |t| + t.references :translation, index: true + t.integer :response_code + t.jsonb :response + + t.timestamps null: false + end + + add_foreign_key :cangaroo_attempts, :cangaroo_translations, column: :translation_id + end +end diff --git a/lib/cangaroo/engine.rb b/lib/cangaroo/engine.rb index 82da8dd..79fda7f 100644 --- a/lib/cangaroo/engine.rb +++ b/lib/cangaroo/engine.rb @@ -13,7 +13,9 @@ class Engine < ::Rails::Engine Rails.configuration.cangaroo = ActiveSupport::OrderedOptions.new Rails.configuration.cangaroo.jobs = [] Rails.configuration.cangaroo.poll_jobs = [] + Rails.configuration.cangaroo.process_response = true Rails.configuration.cangaroo.basic_auth = false + Rails.configuration.cangaroo.payload_keys = ['id'] end end end diff --git a/lib/cangaroo/webhook/client.rb b/lib/cangaroo/webhook/client.rb index be03d3b..2fb74cc 100644 --- a/lib/cangaroo/webhook/client.rb +++ b/lib/cangaroo/webhook/client.rb @@ -12,7 +12,7 @@ def initialize(connection, path) @path = path end - def post(payload, request_id, parameters) + def post(payload, request_id, parameters, translation = nil) request_body = body(payload, request_id, parameters).to_json request_options = { @@ -33,6 +33,14 @@ def post(payload, request_id, parameters) sanitized_response = sanitize_response(req) + if translation.present? + Cangaroo::Attempt.create!( + translation: translation, + response_code: req.response.code, + response: sanitized_response + ) + end + if %w(200 201 202 204).include?(req.response.code) sanitized_response else diff --git a/spec/factories/cangaroo_connections.rb b/spec/factories/cangaroo_connections.rb index 2b40786..b632840 100644 --- a/spec/factories/cangaroo_connections.rb +++ b/spec/factories/cangaroo_connections.rb @@ -3,7 +3,7 @@ name :store url 'www.store.com' parameters { { first: 'first', second: 'second' } } - key '1e4e888ac66f8dd41e00c5a7ac36a32a9950d271' - token '8d49cddb4291562808bfca1bee8a9f7cf947a987' + key { SecureRandom.hex(13) } + token { SecureRandom.hex(13) } end end diff --git a/spec/factories/translation_factory.rb b/spec/factories/translation_factory.rb new file mode 100644 index 0000000..fa90a37 --- /dev/null +++ b/spec/factories/translation_factory.rb @@ -0,0 +1,17 @@ +FactoryGirl.define do + factory :translation, class: 'Cangaroo::Translation' do + source_connection { FactoryGirl.create(:cangaroo_connection, name: Faker::Company.name, url: Faker::Internet.domain_name) } + destination_connection { FactoryGirl.create(:cangaroo_connection, name: Faker::Company.name, url: Faker::Internet.domain_name) } + + job_id { SecureRandom.uuid } + object_type 'customers' + + request { + { + "id" => SecureRandom.random_number(1000), + "updated_at" => DateTime.now.iso8601, + "created_at" => DateTime.now.iso8601 + } + } + end +end diff --git a/spec/interactors/cangaroo/perform_jobs_spec.rb b/spec/interactors/cangaroo/perform_jobs_spec.rb index f4c7e43..b8ee467 100644 --- a/spec/interactors/cangaroo/perform_jobs_spec.rb +++ b/spec/interactors/cangaroo/perform_jobs_spec.rb @@ -18,8 +18,8 @@ class JobB < Cangaroo::Job; end end describe '.call' do - let(:job_a) { double('job_a', perform?: true, enqueue: nil) } - let(:job_b) { double('job_b', perform?: false, enqueue: nil) } + let(:job_a) { double('job_a', perform?: true, enqueue: nil, payload_state: :new) } + let(:job_b) { double('job_b', perform?: false, enqueue: nil, payload_state: :new) } context 'payload with objects' do let(:json_body) { load_fixture('json_payload_ok.json') } diff --git a/spec/jobs/cangaroo/job_spec.rb b/spec/jobs/cangaroo/job_spec.rb index 991bfdf..95a42e7 100644 --- a/spec/jobs/cangaroo/job_spec.rb +++ b/spec/jobs/cangaroo/job_spec.rb @@ -11,7 +11,7 @@ class FakeJob < Cangaroo::Job let(:job_class) { FakeJob } let(:destination_connection) { create(:cangaroo_connection) } let(:type) { 'orders' } - let(:payload) { { id: 'O123' } } + let(:payload) { { "id" => 'O123' } } let(:connection_response) { parse_fixture('json_payload_connection_response.json') } let(:options) do @@ -43,11 +43,12 @@ class FakeJob < Cangaroo::Job it 'calls post on client' do job.perform expect(client).to have_received(:post) - .with(job.transform, job.job_id, email: 'info@nebulab.it') + .with(job.transform, job.job_id, { email: 'info@nebulab.it' }, an_instance_of(Cangaroo::Translation)) end it 'restart the flow' do job.perform + expect(Cangaroo::PerformFlow).to have_received(:call) .once .with(source_connection: destination_connection, @@ -63,6 +64,31 @@ class FakeJob < Cangaroo::Job expect(Cangaroo::PerformFlow).to_not have_received(:call) end + it 'creates a translation record and stores the response' do + job.perform + + translation = Cangaroo::Translation.find_by(job_id: job.job_id) + + expect(translation).to_not be_nil + expect(translation.successful?).to eq(true) + expect(translation.destination_connection).to eq(destination_connection) + expect(translation.object_type).to eq('orders') + expect(translation.object_key).to eq('id') + expect(translation.object_id).to eq('O123') + end + + context 'endpoint communication fails' do + it 'creates a unsuccessful translation model' do + client.stub(:post).and_raise('an error') + + expect { job.perform }.to raise_error + + translation = Cangaroo::Translation.find_by(job_id: job.job_id) + expect(translation).to_not be_nil + expect(translation.successful?).to eq(false) + end + end + context 'endpoint provides a empty response' do it 'should not restart the flow' do allow(client).to receive(:post).and_return('') @@ -81,5 +107,17 @@ class FakeJob < Cangaroo::Job describe '#transform' do it { expect(job_class.new(options).transform).to eq('order' => payload) } end + + describe '#payload_state' do + let(:job) { job_class.new(options) } + + it { expect(job.payload_state).to eq(:new) } + + it 'returns updated when a previous matching payload exists' do + allow_any_instance_of(Cangaroo::Translation).to receive(:related_translations).and_return([OpenStruct.new()]) + + expect(job.payload_state).to eq(:updated) + end + end end end diff --git a/spec/models/cangaroo/translation_spec.rb b/spec/models/cangaroo/translation_spec.rb new file mode 100644 index 0000000..7c76618 --- /dev/null +++ b/spec/models/cangaroo/translation_spec.rb @@ -0,0 +1,51 @@ +require 'rails_helper' + +RSpec.describe Cangaroo::Translation, type: :model do + let(:translation) { FactoryGirl.build(:translation) } + + before do + Rails.configuration.cangaroo.payload_keys = ['id', 'order_id'] + end + + describe '#related_translations' do + it 'returns other translation of this same object that have been sent through the pipeline' do + translation.save! + + previous_translation = FactoryGirl.build(:translation) + + previous_translation.object_type = translation.object_type + # copying the request will also copy the ID + key type on save + previous_translation.request = translation.request.deep_dup + previous_translation.request['another_field'] = Faker::Lorem.word + previous_translation.save! + + expect(translation.related_translations).to_not be_empty + expect(translation.related_translations.first.id).to eq(previous_translation.id) + end + + it 'returns an empty collection when on previous objects exist' do + expect(translation.related_translations).to be_empty + end + end + + describe "#object_key" do + it "chooses the first primary key available" do + translation.request = { + "id" => nil, + "order_id" => 123 + } + + expect(translation.object_key).to eq("order_id") + expect(translation.object_id).to eq("123") + end + + it 'returns nil when no key is available' do + translation.request = { + "order_id" => nil + } + + expect(translation.object_key).to be_nil + expect(translation.object_id).to be_nil + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index dc773c1..67bf980 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -1,6 +1,7 @@ require 'simplecov' require 'codeclimate-test-reporter' require 'pry-byebug' +require 'faker' SimpleCov.start 'rails' do add_group 'Commands', 'app/commands' @@ -39,6 +40,7 @@ # reset config before each spec config.before(:each) do Rails.configuration.cangaroo.basic_auth = false + Rails.configuration.cangaroo.process_response = true Rails.configuration.cangaroo.jobs = [] Rails.configuration.cangaroo.poll_job = [] end diff --git a/spec/support/rails_app.rb b/spec/support/rails_app.rb index d4e3be1..5139317 100644 --- a/spec/support/rails_app.rb +++ b/spec/support/rails_app.rb @@ -6,8 +6,7 @@ require 'cangaroo' -database_path = File.expand_path('../../../tmp/cangaroo_test.sqlite3', __FILE__) -ENV['DATABASE_URL'] = "sqlite3://#{database_path}" +ENV['DATABASE_URL'] = "postgres:///cangaroo_test" # Initialize our test app