Add morph download for a scraper's sqlite database

.rubocop_todo.yml

@@ -36,7 +36,7 @@ Metrics/MethodLength:
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ModuleLength:
-  Max: 124
+  Max: 163
 
 # Offense count: 1
 # Configuration parameters: ExpectMatchingDefinition, CheckDefinitionPathHierarchy, CheckDefinitionPathHierarchyRoots, Regex, IgnoreExecutableScripts, AllowedAcronyms.

CHANGELOG.md

@@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `morph download` command that downloads a scraper's sqlite database from
+  morph.io to `data.sqlite`, working out the scraper name from the git remote
+  when it isn't given explicitly
 - SimpleCov coverage reporting and a much expanded test suite (CLI and HTTP behaviour
   tested with WebMock, no network access in tests)
 

README.md

@@ -35,6 +35,16 @@ you'd rather not wait for the upload, leave it out with
 The first time you run it, it will ask for your morph.io API key, which it
 saves in `~/.morph`.
 
+To download the sqlite database of a scraper from morph.io to `data.sqlite`
+in your current directory
+
+    morph download
+
+That works out which scraper you mean from the git remote of the current
+directory. You can also name the scraper explicitly
+
+    morph download openaustralia/example_scraper
+
 For help
 
     morph help
@@ -44,8 +54,6 @@ For help
 It uploads your code every time. So if it's big it might take a little while.
 Scrapers are not usually so I'm hoping this won't really be an issue.
 
-It doesn't yet return you the resulting sqlite database.
-
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies.

lib/morph-cli.rb

@@ -2,6 +2,7 @@
 require 'yaml'
 require 'find'
 require 'json'
+require 'open3'
 require 'pathname'
 require 'tempfile'
 require 'fileutils'
@@ -21,25 +22,23 @@ def self.execute(directory, _development, env_config, skip_data: false)
     end
 
     database_path = MorphCLI.database_path(directory)
-    if skip_data
-      all_paths.delete(database_path)
-      database_path = nil
-    end
+    all_paths.delete(database_path)
+    database_path = nil if skip_data
 
-    size = MorphCLI.get_dir_size(directory, all_paths)
+    size = MorphCLI.get_dir_size(directory, all_paths + [database_path].compact)
     puts "Uploading #{size}#{" (including #{database_path})" if database_path}..."
 
     file = MorphCLI.create_tar(directory, all_paths)
 
-    scraper_output = run(file, env_config)
+    scraper_output = run(file, env_config, directory, database_path)
 
     puts "Scraper didn't output anything, but it ran successfully." unless scraper_output
   end
 
   # Uploads the code to the server, streams the run output to the local
   # stdout/stderr and returns whether the scraper itself wrote anything
   # to stdout or stderr
-  def self.run(file, env_config)
+  def self.run(file, env_config, directory, database_path)
     connection = Faraday.new(url: env_config[:base_url]) do |f|
       f.request :multipart
       f.response :raise_error
@@ -49,10 +48,18 @@ def self.run(file, env_config)
     buffer = +""
     scraper_output = false
     connection.post("/run") do |req|
-      req.body = {
+      body = {
         api_key: env_config[:api_key],
         code: Faraday::Multipart::FilePart.new(file, "application/gzip")
       }
+      if database_path
+        body[:database] = Faraday::Multipart::FilePart.new(
+          File.join(directory, database_path),
+          "application/octet-stream",
+          database_path
+        )
+      end
+      req.body = body
       # 10 minutes should be "enough for everyone", right?
       # Setting :timeout to nil in the config will disable the timeout
       # entirely. The Faraday default is 60 seconds.
@@ -72,6 +79,46 @@ def self.run(file, env_config)
     scraper_output
   end
 
+  def self.download(directory, env_config, scraper)
+    connection = Faraday.new(url: env_config[:base_url]) do |f|
+      f.response :raise_error
+      f.adapter Faraday.default_adapter
+    end
+
+    # Download to a tempfile in the same directory first so a failed download
+    # doesn't clobber an existing database
+    tempfile = Tempfile.new(["morph", ".sqlite"], directory)
+    tempfile.binmode
+
+    begin
+      connection.get("/#{scraper}/data.sqlite") do |req|
+        req.params[:key] = env_config[:api_key]
+        req.options.timeout = env_config.fetch(:timeout, 600)
+        req.options.on_data = proc do |chunk, _overall_received_bytes, env|
+          tempfile.write(chunk) if env.status == 200
+        end
+      end
+
+      tempfile.close
+      File.rename(tempfile.path, File.join(directory, "data.sqlite"))
+    ensure
+      tempfile.close unless tempfile.closed?
+      FileUtils.rm_f(tempfile.path)
+    end
+
+    size = Filesize.from("#{File.size(File.join(directory, 'data.sqlite'))} B").pretty
+    puts "Saved #{size} to data.sqlite"
+  end
+
+  # The name of the scraper on morph (owner/name), worked out from the git
+  # remote of the given directory. Returns nil if it can't be worked out.
+  def self.scraper_name(directory)
+    url, _stderr, status = Open3.capture3("git", "-C", directory, "config", "--get", "remote.origin.url")
+    return nil unless status.success?
+
+    url.strip[%r{([^/:]+/[^/:]+?)(?:\.git)?\z}, 1]
+  end
+
   # Writes the line to the local stdout/stderr and returns the name of the
   # stream it came from
   def self.log(line)

lib/morph-cli/cli.rb

@@ -18,33 +18,30 @@ def self.exit_on_failure?
                        desc: "Don't upload the local data.sqlite database with the scraper"
 
     def execute
-      config = MorphCLI.load_config
-      env_config = if options[:dev]
-                     config[:development]
-                   else
-                     config[:production]
-                   end
+      env_config = load_env_config
 
-      config = ask_and_save_api_key(env_config, config) if env_config[:api_key].nil?
+      with_working_api_key(env_config) do
+        MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data])
+      end
+    end
+
+    desc 'download [SCRAPER]', 'download the sqlite database of a scraper from morph'
+    option :directory, default: Dir.getwd
+
+    def download(scraper = nil)
+      env_config = load_env_config
+
+      scraper ||= MorphCLI.scraper_name(options[:directory])
+      if scraper.nil?
+        warn "Can't work out the scraper name from the git remote. Give it explicitly with: morph download OWNER/SCRAPER"
+        exit(1)
+      end
 
-      api_key_is_valid = false
-      until api_key_is_valid
-        begin
-          MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data])
-          api_key_is_valid = true
-        rescue Faraday::UnauthorizedError
-          puts "Your key isn't working. Let's try again."
-          config = ask_and_save_api_key(env_config, config)
-        rescue Faraday::ConnectionFailed => e
-          warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})"
-          exit(1)
-        rescue Faraday::ServerError => e
-          warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})"
-          exit(1)
-        rescue Faraday::Error => e
-          warn "Request to #{env_config[:base_url]} failed (#{e})"
-          exit(1)
-        end
+      with_working_api_key(env_config) do
+        MorphCLI.download(options[:directory], env_config, scraper)
+      rescue Faraday::ResourceNotFound
+        warn "Can't find a database for #{scraper} on #{env_config[:base_url]}. Has the scraper run successfully?"
+        exit(1)
       end
     end
 
@@ -55,10 +52,41 @@ def version
     end
 
     no_commands do
-      def ask_and_save_api_key(env_config, config)
+      def load_env_config
+        @config = MorphCLI.load_config
+        env_config = if options[:dev]
+                       @config[:development]
+                     else
+                       @config[:production]
+                     end
+
+        ask_and_save_api_key(env_config) if env_config[:api_key].nil?
+        env_config
+      end
+
+      # Runs the block, prompting for a new API key and retrying if the server
+      # rejects the current one, and turning other request failures into
+      # friendly errors
+      def with_working_api_key(env_config)
+        yield
+      rescue Faraday::UnauthorizedError
+        puts "Your key isn't working. Let's try again."
+        ask_and_save_api_key(env_config)
+        retry
+      rescue Faraday::ConnectionFailed => e
+        warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})"
+        exit(1)
+      rescue Faraday::ServerError => e
+        warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})"
+        exit(1)
+      rescue Faraday::Error => e
+        warn "Request to #{env_config[:base_url]} failed (#{e})"
+        exit(1)
+      end
+
+      def ask_and_save_api_key(env_config)
         env_config[:api_key] = ask("What is your key? (Go to #{env_config[:base_url]}/settings)")
-        MorphCLI.save_config(config)
-        config
+        MorphCLI.save_config(@config)
       end
     end
   end

spec/morph_cli/cli_spec.rb

@@ -112,4 +112,91 @@
       end.to output(%r{Request to https://morph\.io failed}).to_stderr
     end
   end
+
+  describe 'download' do
+    let(:config) do
+      {
+        development: { base_url: 'http://127.0.0.1:3000', api_key: 'dev-key' },
+        production: { base_url: 'https://morph.io', api_key: 'prod-key' }
+      }
+    end
+
+    before do
+      allow(MorphCLI).to receive(:load_config).and_return(config)
+      allow(MorphCLI).to receive(:save_config)
+    end
+
+    it 'downloads the database of the given scraper with the production config' do
+      allow(MorphCLI).to receive(:download)
+
+      described_class.start(['download', 'openaustralia/scraper', '--directory', '/somewhere'])
+
+      expect(MorphCLI).to have_received(:download)
+        .with('/somewhere', config[:production], 'openaustralia/scraper')
+    end
+
+    it 'works out the scraper name from the git remote when none is given' do
+      allow(MorphCLI).to receive(:download)
+      allow(MorphCLI).to receive(:scraper_name).with('/somewhere').and_return('openaustralia/scraper')
+
+      described_class.start(['download', '--directory', '/somewhere'])
+
+      expect(MorphCLI).to have_received(:download)
+        .with('/somewhere', config[:production], 'openaustralia/scraper')
+    end
+
+    it 'exits with an error when the scraper name cannot be worked out' do
+      allow(MorphCLI).to receive(:scraper_name).and_return(nil)
+
+      expect do
+        expect { described_class.start(['download']) }
+          .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) }
+      end.to output(/Can't work out the scraper name/).to_stderr
+    end
+
+    it 'uses the development config when --dev is given' do
+      allow(MorphCLI).to receive(:download)
+
+      described_class.start(['download', 'openaustralia/scraper', '--dev'])
+
+      expect(MorphCLI).to have_received(:download)
+        .with(anything, config[:development], 'openaustralia/scraper')
+    end
+
+    it 'asks for a new API key and retries when the server rejects it' do
+      attempts = 0
+      allow(MorphCLI).to receive(:download) do
+        attempts += 1
+        raise Faraday::UnauthorizedError, '401' if attempts == 1
+      end
+      allow(Thor::LineEditor).to receive(:readline).and_return('fresh-key')
+
+      expect { described_class.start(['download', 'openaustralia/scraper']) }
+        .to output(/Your key isn't working\. Let's try again\./).to_stdout
+
+      expect(attempts).to eq(2)
+      expect(config[:production][:api_key]).to eq('fresh-key')
+      expect(MorphCLI).to have_received(:save_config).with(config)
+    end
+
+    it 'exits with an error when the scraper has no downloadable database' do
+      allow(MorphCLI).to receive(:download)
+        .and_raise(Faraday::ResourceNotFound, '404')
+
+      expect do
+        expect { described_class.start(['download', 'openaustralia/scraper']) }
+          .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) }
+      end.to output(%r{Can't find a database for openaustralia/scraper on https://morph\.io}).to_stderr
+    end
+
+    it 'exits with an error when morph is not reachable' do
+      allow(MorphCLI).to receive(:download)
+        .and_raise(Faraday::ConnectionFailed, 'connection refused')
+
+      expect do
+        expect { described_class.start(['download', 'openaustralia/scraper']) }
+          .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) }
+      end.to output(%r{Morph doesn't look to be running at https://morph\.io}).to_stderr
+    end
+  end
 end