diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 9a226b5..5e805c2 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -36,7 +36,7 @@ Metrics/MethodLength: # Offense count: 1 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: - Max: 124 + Max: 163 # Offense count: 1 # Configuration parameters: ExpectMatchingDefinition, CheckDefinitionPathHierarchy, CheckDefinitionPathHierarchyRoots, Regex, IgnoreExecutableScripts, AllowedAcronyms. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d34b13..8116595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `morph download` command that downloads a scraper's sqlite database from + morph.io to `data.sqlite`, working out the scraper name from the git remote + when it isn't given explicitly - SimpleCov coverage reporting and a much expanded test suite (CLI and HTTP behaviour tested with WebMock, no network access in tests) diff --git a/README.md b/README.md index 5f747c4..995dd3b 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,16 @@ you'd rather not wait for the upload, leave it out with The first time you run it, it will ask for your morph.io API key, which it saves in `~/.morph`. +To download the sqlite database of a scraper from morph.io to `data.sqlite` +in your current directory + + morph download + +That works out which scraper you mean from the git remote of the current +directory. You can also name the scraper explicitly + + morph download openaustralia/example_scraper + For help morph help @@ -44,8 +54,6 @@ For help It uploads your code every time. So if it's big it might take a little while. Scrapers are not usually so I'm hoping this won't really be an issue. -It doesn't yet return you the resulting sqlite database. - ## Development After checking out the repo, run `bundle install` to install dependencies. diff --git a/lib/morph-cli.rb b/lib/morph-cli.rb index 6a22d81..e148bd7 100644 --- a/lib/morph-cli.rb +++ b/lib/morph-cli.rb @@ -2,6 +2,7 @@ require 'yaml' require 'find' require 'json' +require 'open3' require 'pathname' require 'tempfile' require 'fileutils' @@ -21,17 +22,15 @@ def self.execute(directory, _development, env_config, skip_data: false) end database_path = MorphCLI.database_path(directory) - if skip_data - all_paths.delete(database_path) - database_path = nil - end + all_paths.delete(database_path) + database_path = nil if skip_data - size = MorphCLI.get_dir_size(directory, all_paths) + size = MorphCLI.get_dir_size(directory, all_paths + [database_path].compact) puts "Uploading #{size}#{" (including #{database_path})" if database_path}..." file = MorphCLI.create_tar(directory, all_paths) - scraper_output = run(file, env_config) + scraper_output = run(file, env_config, directory, database_path) puts "Scraper didn't output anything, but it ran successfully." unless scraper_output end @@ -39,7 +38,7 @@ def self.execute(directory, _development, env_config, skip_data: false) # Uploads the code to the server, streams the run output to the local # stdout/stderr and returns whether the scraper itself wrote anything # to stdout or stderr - def self.run(file, env_config) + def self.run(file, env_config, directory, database_path) connection = Faraday.new(url: env_config[:base_url]) do |f| f.request :multipart f.response :raise_error @@ -49,10 +48,18 @@ def self.run(file, env_config) buffer = +"" scraper_output = false connection.post("/run") do |req| - req.body = { + body = { api_key: env_config[:api_key], code: Faraday::Multipart::FilePart.new(file, "application/gzip") } + if database_path + body[:database] = Faraday::Multipart::FilePart.new( + File.join(directory, database_path), + "application/octet-stream", + database_path + ) + end + req.body = body # 10 minutes should be "enough for everyone", right? # Setting :timeout to nil in the config will disable the timeout # entirely. The Faraday default is 60 seconds. @@ -72,6 +79,46 @@ def self.run(file, env_config) scraper_output end + def self.download(directory, env_config, scraper) + connection = Faraday.new(url: env_config[:base_url]) do |f| + f.response :raise_error + f.adapter Faraday.default_adapter + end + + # Download to a tempfile in the same directory first so a failed download + # doesn't clobber an existing database + tempfile = Tempfile.new(["morph", ".sqlite"], directory) + tempfile.binmode + + begin + connection.get("/#{scraper}/data.sqlite") do |req| + req.params[:key] = env_config[:api_key] + req.options.timeout = env_config.fetch(:timeout, 600) + req.options.on_data = proc do |chunk, _overall_received_bytes, env| + tempfile.write(chunk) if env.status == 200 + end + end + + tempfile.close + File.rename(tempfile.path, File.join(directory, "data.sqlite")) + ensure + tempfile.close unless tempfile.closed? + FileUtils.rm_f(tempfile.path) + end + + size = Filesize.from("#{File.size(File.join(directory, 'data.sqlite'))} B").pretty + puts "Saved #{size} to data.sqlite" + end + + # The name of the scraper on morph (owner/name), worked out from the git + # remote of the given directory. Returns nil if it can't be worked out. + def self.scraper_name(directory) + url, _stderr, status = Open3.capture3("git", "-C", directory, "config", "--get", "remote.origin.url") + return nil unless status.success? + + url.strip[%r{([^/:]+/[^/:]+?)(?:\.git)?\z}, 1] + end + # Writes the line to the local stdout/stderr and returns the name of the # stream it came from def self.log(line) diff --git a/lib/morph-cli/cli.rb b/lib/morph-cli/cli.rb index 73070ce..9f5660e 100644 --- a/lib/morph-cli/cli.rb +++ b/lib/morph-cli/cli.rb @@ -18,33 +18,30 @@ def self.exit_on_failure? desc: "Don't upload the local data.sqlite database with the scraper" def execute - config = MorphCLI.load_config - env_config = if options[:dev] - config[:development] - else - config[:production] - end + env_config = load_env_config - config = ask_and_save_api_key(env_config, config) if env_config[:api_key].nil? + with_working_api_key(env_config) do + MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data]) + end + end + + desc 'download [SCRAPER]', 'download the sqlite database of a scraper from morph' + option :directory, default: Dir.getwd + + def download(scraper = nil) + env_config = load_env_config + + scraper ||= MorphCLI.scraper_name(options[:directory]) + if scraper.nil? + warn "Can't work out the scraper name from the git remote. Give it explicitly with: morph download OWNER/SCRAPER" + exit(1) + end - api_key_is_valid = false - until api_key_is_valid - begin - MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data]) - api_key_is_valid = true - rescue Faraday::UnauthorizedError - puts "Your key isn't working. Let's try again." - config = ask_and_save_api_key(env_config, config) - rescue Faraday::ConnectionFailed => e - warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::ServerError => e - warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::Error => e - warn "Request to #{env_config[:base_url]} failed (#{e})" - exit(1) - end + with_working_api_key(env_config) do + MorphCLI.download(options[:directory], env_config, scraper) + rescue Faraday::ResourceNotFound + warn "Can't find a database for #{scraper} on #{env_config[:base_url]}. Has the scraper run successfully?" + exit(1) end end @@ -55,10 +52,41 @@ def version end no_commands do - def ask_and_save_api_key(env_config, config) + def load_env_config + @config = MorphCLI.load_config + env_config = if options[:dev] + @config[:development] + else + @config[:production] + end + + ask_and_save_api_key(env_config) if env_config[:api_key].nil? + env_config + end + + # Runs the block, prompting for a new API key and retrying if the server + # rejects the current one, and turning other request failures into + # friendly errors + def with_working_api_key(env_config) + yield + rescue Faraday::UnauthorizedError + puts "Your key isn't working. Let's try again." + ask_and_save_api_key(env_config) + retry + rescue Faraday::ConnectionFailed => e + warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::ServerError => e + warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::Error => e + warn "Request to #{env_config[:base_url]} failed (#{e})" + exit(1) + end + + def ask_and_save_api_key(env_config) env_config[:api_key] = ask("What is your key? (Go to #{env_config[:base_url]}/settings)") - MorphCLI.save_config(config) - config + MorphCLI.save_config(@config) end end end diff --git a/spec/morph_cli/cli_spec.rb b/spec/morph_cli/cli_spec.rb index bc7dad5..a183c20 100644 --- a/spec/morph_cli/cli_spec.rb +++ b/spec/morph_cli/cli_spec.rb @@ -112,4 +112,91 @@ end.to output(%r{Request to https://morph\.io failed}).to_stderr end end + + describe 'download' do + let(:config) do + { + development: { base_url: 'http://127.0.0.1:3000', api_key: 'dev-key' }, + production: { base_url: 'https://morph.io', api_key: 'prod-key' } + } + end + + before do + allow(MorphCLI).to receive(:load_config).and_return(config) + allow(MorphCLI).to receive(:save_config) + end + + it 'downloads the database of the given scraper with the production config' do + allow(MorphCLI).to receive(:download) + + described_class.start(['download', 'openaustralia/scraper', '--directory', '/somewhere']) + + expect(MorphCLI).to have_received(:download) + .with('/somewhere', config[:production], 'openaustralia/scraper') + end + + it 'works out the scraper name from the git remote when none is given' do + allow(MorphCLI).to receive(:download) + allow(MorphCLI).to receive(:scraper_name).with('/somewhere').and_return('openaustralia/scraper') + + described_class.start(['download', '--directory', '/somewhere']) + + expect(MorphCLI).to have_received(:download) + .with('/somewhere', config[:production], 'openaustralia/scraper') + end + + it 'exits with an error when the scraper name cannot be worked out' do + allow(MorphCLI).to receive(:scraper_name).and_return(nil) + + expect do + expect { described_class.start(['download']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(/Can't work out the scraper name/).to_stderr + end + + it 'uses the development config when --dev is given' do + allow(MorphCLI).to receive(:download) + + described_class.start(['download', 'openaustralia/scraper', '--dev']) + + expect(MorphCLI).to have_received(:download) + .with(anything, config[:development], 'openaustralia/scraper') + end + + it 'asks for a new API key and retries when the server rejects it' do + attempts = 0 + allow(MorphCLI).to receive(:download) do + attempts += 1 + raise Faraday::UnauthorizedError, '401' if attempts == 1 + end + allow(Thor::LineEditor).to receive(:readline).and_return('fresh-key') + + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to output(/Your key isn't working\. Let's try again\./).to_stdout + + expect(attempts).to eq(2) + expect(config[:production][:api_key]).to eq('fresh-key') + expect(MorphCLI).to have_received(:save_config).with(config) + end + + it 'exits with an error when the scraper has no downloadable database' do + allow(MorphCLI).to receive(:download) + .and_raise(Faraday::ResourceNotFound, '404') + + expect do + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Can't find a database for openaustralia/scraper on https://morph\.io}).to_stderr + end + + it 'exits with an error when morph is not reachable' do + allow(MorphCLI).to receive(:download) + .and_raise(Faraday::ConnectionFailed, 'connection refused') + + expect do + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Morph doesn't look to be running at https://morph\.io}).to_stderr + end + end end diff --git a/spec/morph_cli_spec.rb b/spec/morph_cli_spec.rb index 93b7e26..da1006b 100644 --- a/spec/morph_cli_spec.rb +++ b/spec/morph_cli_spec.rb @@ -107,7 +107,9 @@ def tar_entry_names(body) end expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req| - tar_entry_names(req.body).include?("data.sqlite") + # Sent as its own multipart field rather than packed into the tar + req.body.include?("data.sqlite") && + !tar_entry_names(req.body).include?("data.sqlite") end) end @@ -122,7 +124,8 @@ def tar_entry_names(body) end expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req| - !tar_entry_names(req.body).include?("data.sqlite") + !req.body.include?("data.sqlite") && + !tar_entry_names(req.body).include?("data.sqlite") end) end @@ -146,6 +149,96 @@ def tar_entry_names(body) end end + describe ".download" do + let(:env_config) { { base_url: "https://morph.io", api_key: "secret-key" } } + + it "saves the scraper's database as data.sqlite and reports the size" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 200, body: "sqlite bytes") + + Dir.mktmpdir do |dir| + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to output("Saved 12.00 B to data.sqlite\n").to_stdout + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("sqlite bytes") + end + end + + it "overwrites an existing database on a successful download" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 200, body: "new data") + + Dir.mktmpdir do |dir| + File.write(File.join(dir, "data.sqlite"), "old data") + + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to output(/Saved/).to_stdout + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("new data") + end + end + + it "leaves an existing database and no tempfile behind when the download fails" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 404, body: "") + + Dir.mktmpdir do |dir| + File.write(File.join(dir, "data.sqlite"), "old data") + + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to raise_error(Faraday::ResourceNotFound) + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("old data") + expect(Dir.children(dir)).to contain_exactly("data.sqlite") + end + end + end + + describe ".scraper_name" do + def with_git_remote(url) + Dir.mktmpdir do |dir| + system("git", "init", "--quiet", dir, exception: true) + system("git", "-C", dir, "remote", "add", "origin", url, exception: true) + yield dir + end + end + + it "derives owner/name from an https git remote" do + with_git_remote("https://github.com/openaustralia/morph-cli.git") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "derives owner/name from an ssh git remote" do + with_git_remote("git@github.com:openaustralia/morph-cli.git") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "derives owner/name from a remote without a .git suffix" do + with_git_remote("https://github.com/openaustralia/morph-cli") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "returns nil when the directory is not a git repository" do + Dir.mktmpdir do |dir| + expect(described_class.scraper_name(dir)).to be_nil + end + end + + it "returns nil when the repository has no origin remote" do + Dir.mktmpdir do |dir| + system("git", "init", "--quiet", dir, exception: true) + + expect(described_class.scraper_name(dir)).to be_nil + end + end + end + describe ".log" do it "writes stdout stream lines to stdout" do expect { described_class.log(%({"stream":"stdout","text":"out"})) }