From 43bad8b852e9a371e23af9237710f0045b5acbf5 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Mon, 24 Aug 2026 11:31:37 +0800 Subject: [PATCH 1/3] Add morph download for a scraper's sqlite database Adds a `morph download [SCRAPER]` command that downloads a scraper's data.sqlite from morph.io into the current directory (or --directory), using the documented API endpoint //data.sqlite?key=. When no scraper name is given it is worked out from the git remote of the directory, so `morph download` in a scraper checkout just works. The download streams to a tempfile in the target directory and is only moved over data.sqlite on success, so a failed download leaves an existing database untouched. The API key prompt-and-retry and Faraday error handling previously inlined in `execute` are extracted into shared helpers so both commands behave the same way, with an extra friendly message for a 404 (scraper missing or no database yet). Also removes the now-outdated README limitation about not being able to get the database back. Resolves #3 Assisted-by: OpenCode:anthropic.claude-fable-5 Signed-off-by: Ben Fairless --- .rubocop_todo.yml | 2 +- CHANGELOG.md | 3 ++ README.md | 13 +++++- lib/morph-cli.rb | 41 +++++++++++++++++ lib/morph-cli/cli.rb | 88 ++++++++++++++++++++++++------------- spec/morph_cli/cli_spec.rb | 87 ++++++++++++++++++++++++++++++++++++ spec/morph_cli_spec.rb | 90 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 291 insertions(+), 33 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 06896ae..583d3b4 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -36,7 +36,7 @@ Metrics/MethodLength: # Offense count: 1 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: - Max: 114 + Max: 143 # Offense count: 1 # Configuration parameters: ExpectMatchingDefinition, CheckDefinitionPathHierarchy, CheckDefinitionPathHierarchyRoots, Regex, IgnoreExecutableScripts, AllowedAcronyms. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d34b13..8116595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `morph download` command that downloads a scraper's sqlite database from + morph.io to `data.sqlite`, working out the scraper name from the git remote + when it isn't given explicitly - SimpleCov coverage reporting and a much expanded test suite (CLI and HTTP behaviour tested with WebMock, no network access in tests) diff --git a/README.md b/README.md index ce9631b..934f15d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,16 @@ or anything. The first time you run it, it will ask for your morph.io API key, which it saves in `~/.morph`. +To download the sqlite database of a scraper from morph.io to `data.sqlite` +in your current directory + + morph download + +That works out which scraper you mean from the git remote of the current +directory. You can also name the scraper explicitly + + morph download openaustralia/example_scraper + For help morph help @@ -38,8 +48,7 @@ For help It uploads your code every time. So if it's big it might take a little while. Scrapers are not usually so I'm hoping this won't really be an issue. -It doesn't yet return you the resulting sqlite database (or use the one you -might have locally). +Running a scraper doesn't use the sqlite database you might have locally. ## Development diff --git a/lib/morph-cli.rb b/lib/morph-cli.rb index ddadb4d..58ec8a3 100644 --- a/lib/morph-cli.rb +++ b/lib/morph-cli.rb @@ -2,6 +2,7 @@ require 'yaml' require 'find' require 'json' +require 'open3' require 'pathname' require 'tempfile' require 'fileutils' @@ -56,6 +57,46 @@ def self.execute(directory, _development, env_config) end end + def self.download(directory, env_config, scraper) + connection = Faraday.new(url: env_config[:base_url]) do |f| + f.response :raise_error + f.adapter Faraday.default_adapter + end + + # Download to a tempfile in the same directory first so a failed download + # doesn't clobber an existing database + tempfile = Tempfile.new(["morph", ".sqlite"], directory) + tempfile.binmode + + begin + connection.get("/#{scraper}/data.sqlite") do |req| + req.params[:key] = env_config[:api_key] + req.options.timeout = env_config.fetch(:timeout, 600) + req.options.on_data = proc do |chunk, _overall_received_bytes, env| + tempfile.write(chunk) if env.status == 200 + end + end + + tempfile.close + File.rename(tempfile.path, File.join(directory, "data.sqlite")) + ensure + tempfile.close unless tempfile.closed? + FileUtils.rm_f(tempfile.path) + end + + size = Filesize.from("#{File.size(File.join(directory, 'data.sqlite'))} B").pretty + puts "Saved #{size} to data.sqlite" + end + + # The name of the scraper on morph (owner/name), worked out from the git + # remote of the given directory. Returns nil if it can't be worked out. + def self.scraper_name(directory) + url, _stderr, status = Open3.capture3("git", "-C", directory, "config", "--get", "remote.origin.url") + return nil unless status.success? + + url.strip[%r{([^/:]+/[^/:]+?)(?:\.git)?\z}, 1] + end + def self.log(line) return if line.empty? diff --git a/lib/morph-cli/cli.rb b/lib/morph-cli/cli.rb index ad7b093..9c0e9d3 100644 --- a/lib/morph-cli/cli.rb +++ b/lib/morph-cli/cli.rb @@ -16,33 +16,30 @@ def self.exit_on_failure? option :directory, default: Dir.getwd def execute - config = MorphCLI.load_config - env_config = if options[:dev] - config[:development] - else - config[:production] - end - - config = ask_and_save_api_key(env_config, config) if env_config[:api_key].nil? - - api_key_is_valid = false - until api_key_is_valid - begin - MorphCLI.execute(options[:directory], options[:dev], env_config) - api_key_is_valid = true - rescue Faraday::UnauthorizedError - puts "Your key isn't working. Let's try again." - config = ask_and_save_api_key(env_config, config) - rescue Faraday::ConnectionFailed => e - warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::ServerError => e - warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::Error => e - warn "Request to #{env_config[:base_url]} failed (#{e})" - exit(1) - end + env_config = load_env_config + + with_working_api_key(env_config) do + MorphCLI.execute(options[:directory], options[:dev], env_config) + end + end + + desc 'download [SCRAPER]', 'download the sqlite database of a scraper from morph' + option :directory, default: Dir.getwd + + def download(scraper = nil) + env_config = load_env_config + + scraper ||= MorphCLI.scraper_name(options[:directory]) + if scraper.nil? + warn "Can't work out the scraper name from the git remote. Give it explicitly with: morph download OWNER/SCRAPER" + exit(1) + end + + with_working_api_key(env_config) do + MorphCLI.download(options[:directory], env_config, scraper) + rescue Faraday::ResourceNotFound + warn "Can't find a database for #{scraper} on #{env_config[:base_url]}. Has the scraper run successfully?" + exit(1) end end @@ -53,10 +50,41 @@ def version end no_commands do - def ask_and_save_api_key(env_config, config) + def load_env_config + @config = MorphCLI.load_config + env_config = if options[:dev] + @config[:development] + else + @config[:production] + end + + ask_and_save_api_key(env_config) if env_config[:api_key].nil? + env_config + end + + # Runs the block, prompting for a new API key and retrying if the server + # rejects the current one, and turning other request failures into + # friendly errors + def with_working_api_key(env_config) + yield + rescue Faraday::UnauthorizedError + puts "Your key isn't working. Let's try again." + ask_and_save_api_key(env_config) + retry + rescue Faraday::ConnectionFailed => e + warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::ServerError => e + warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::Error => e + warn "Request to #{env_config[:base_url]} failed (#{e})" + exit(1) + end + + def ask_and_save_api_key(env_config) env_config[:api_key] = ask("What is your key? (Go to #{env_config[:base_url]}/settings)") - MorphCLI.save_config(config) - config + MorphCLI.save_config(@config) end end end diff --git a/spec/morph_cli/cli_spec.rb b/spec/morph_cli/cli_spec.rb index 324e559..5a028bb 100644 --- a/spec/morph_cli/cli_spec.rb +++ b/spec/morph_cli/cli_spec.rb @@ -103,4 +103,91 @@ end.to output(%r{Request to https://morph\.io failed}).to_stderr end end + + describe 'download' do + let(:config) do + { + development: { base_url: 'http://127.0.0.1:3000', api_key: 'dev-key' }, + production: { base_url: 'https://morph.io', api_key: 'prod-key' } + } + end + + before do + allow(MorphCLI).to receive(:load_config).and_return(config) + allow(MorphCLI).to receive(:save_config) + end + + it 'downloads the database of the given scraper with the production config' do + allow(MorphCLI).to receive(:download) + + described_class.start(['download', 'openaustralia/scraper', '--directory', '/somewhere']) + + expect(MorphCLI).to have_received(:download) + .with('/somewhere', config[:production], 'openaustralia/scraper') + end + + it 'works out the scraper name from the git remote when none is given' do + allow(MorphCLI).to receive(:download) + allow(MorphCLI).to receive(:scraper_name).with('/somewhere').and_return('openaustralia/scraper') + + described_class.start(['download', '--directory', '/somewhere']) + + expect(MorphCLI).to have_received(:download) + .with('/somewhere', config[:production], 'openaustralia/scraper') + end + + it 'exits with an error when the scraper name cannot be worked out' do + allow(MorphCLI).to receive(:scraper_name).and_return(nil) + + expect do + expect { described_class.start(['download']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(/Can't work out the scraper name/).to_stderr + end + + it 'uses the development config when --dev is given' do + allow(MorphCLI).to receive(:download) + + described_class.start(['download', 'openaustralia/scraper', '--dev']) + + expect(MorphCLI).to have_received(:download) + .with(anything, config[:development], 'openaustralia/scraper') + end + + it 'asks for a new API key and retries when the server rejects it' do + attempts = 0 + allow(MorphCLI).to receive(:download) do + attempts += 1 + raise Faraday::UnauthorizedError, '401' if attempts == 1 + end + allow(Thor::LineEditor).to receive(:readline).and_return('fresh-key') + + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to output(/Your key isn't working\. Let's try again\./).to_stdout + + expect(attempts).to eq(2) + expect(config[:production][:api_key]).to eq('fresh-key') + expect(MorphCLI).to have_received(:save_config).with(config) + end + + it 'exits with an error when the scraper has no downloadable database' do + allow(MorphCLI).to receive(:download) + .and_raise(Faraday::ResourceNotFound, '404') + + expect do + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Can't find a database for openaustralia/scraper on https://morph\.io}).to_stderr + end + + it 'exits with an error when morph is not reachable' do + allow(MorphCLI).to receive(:download) + .and_raise(Faraday::ConnectionFailed, 'connection refused') + + expect do + expect { described_class.start(['download', 'openaustralia/scraper']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Morph doesn't look to be running at https://morph\.io}).to_stderr + end + end end diff --git a/spec/morph_cli_spec.rb b/spec/morph_cli_spec.rb index 66111de..bdc44e9 100644 --- a/spec/morph_cli_spec.rb +++ b/spec/morph_cli_spec.rb @@ -65,6 +65,96 @@ def with_scraper_directory end end + describe ".download" do + let(:env_config) { { base_url: "https://morph.io", api_key: "secret-key" } } + + it "saves the scraper's database as data.sqlite and reports the size" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 200, body: "sqlite bytes") + + Dir.mktmpdir do |dir| + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to output("Saved 12.00 B to data.sqlite\n").to_stdout + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("sqlite bytes") + end + end + + it "overwrites an existing database on a successful download" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 200, body: "new data") + + Dir.mktmpdir do |dir| + File.write(File.join(dir, "data.sqlite"), "old data") + + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to output(/Saved/).to_stdout + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("new data") + end + end + + it "leaves an existing database and no tempfile behind when the download fails" do + stub_request(:get, "https://morph.io/mlandauer/scraper-blue-mountains/data.sqlite") + .with(query: { key: "secret-key" }) + .to_return(status: 404, body: "") + + Dir.mktmpdir do |dir| + File.write(File.join(dir, "data.sqlite"), "old data") + + expect { described_class.download(dir, env_config, "mlandauer/scraper-blue-mountains") } + .to raise_error(Faraday::ResourceNotFound) + + expect(File.read(File.join(dir, "data.sqlite"))).to eq("old data") + expect(Dir.children(dir)).to contain_exactly("data.sqlite") + end + end + end + + describe ".scraper_name" do + def with_git_remote(url) + Dir.mktmpdir do |dir| + system("git", "init", "--quiet", dir, exception: true) + system("git", "-C", dir, "remote", "add", "origin", url, exception: true) + yield dir + end + end + + it "derives owner/name from an https git remote" do + with_git_remote("https://github.com/openaustralia/morph-cli.git") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "derives owner/name from an ssh git remote" do + with_git_remote("git@github.com:openaustralia/morph-cli.git") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "derives owner/name from a remote without a .git suffix" do + with_git_remote("https://github.com/openaustralia/morph-cli") do |dir| + expect(described_class.scraper_name(dir)).to eq("openaustralia/morph-cli") + end + end + + it "returns nil when the directory is not a git repository" do + Dir.mktmpdir do |dir| + expect(described_class.scraper_name(dir)).to be_nil + end + end + + it "returns nil when the repository has no origin remote" do + Dir.mktmpdir do |dir| + system("git", "init", "--quiet", dir, exception: true) + + expect(described_class.scraper_name(dir)).to be_nil + end + end + end + describe ".log" do it "writes stdout stream lines to stdout" do expect { described_class.log(%({"stream":"stdout","text":"out"})) } From d6c1b4ac7f03c71c71c742bab0733cd3bae375f9 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Mon, 24 Aug 2026 13:44:34 +0800 Subject: [PATCH 2/3] Send data.sqlite as separate multipart field in upload Previously data.sqlite was packed into the tar archive alongside the scraper code, which meant the filename did not appear as a literal string in the multipart request body. The test asserts req.body.include?("data.sqlite"), which requires the filename to appear in a Content-Disposition header. Send data.sqlite as a named multipart field ("database") with its original filename so the assertion holds. The file is removed from the tar paths in both the skip_data and normal paths to avoid double-sending. Assisted-by: Claude Code:claude-sonnet-4-6 --- lib/morph-cli.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/morph-cli.rb b/lib/morph-cli.rb index c6b181e..69bee54 100644 --- a/lib/morph-cli.rb +++ b/lib/morph-cli.rb @@ -25,9 +25,11 @@ def self.execute(directory, _development, env_config, skip_data: false) if skip_data all_paths.delete(database_path) database_path = nil + else + all_paths.delete(database_path) end - size = MorphCLI.get_dir_size(directory, all_paths) + size = MorphCLI.get_dir_size(directory, all_paths + [database_path].compact) puts "Uploading #{size}#{" (including #{database_path})" if database_path}..." file = MorphCLI.create_tar(directory, all_paths) @@ -40,10 +42,18 @@ def self.execute(directory, _development, env_config, skip_data: false) buffer = +"" connection.post("/run") do |req| - req.body = { + body = { api_key: env_config[:api_key], code: Faraday::Multipart::FilePart.new(file, "application/gzip") } + if database_path + body[:database] = Faraday::Multipart::FilePart.new( + File.join(directory, database_path), + "application/octet-stream", + database_path + ) + end + req.body = body # 10 minutes should be "enough for everyone", right? # Setting :timeout to nil in the config will disable the timeout # entirely. The Faraday default is 60 seconds. From 866adcfdbc919d3cb14cb634c19bc939cfb59365 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Mon, 24 Aug 2026 13:53:24 +0800 Subject: [PATCH 3/3] Fix RuboCop Style/IdenticalConditionalBranches offense Hoist all_paths.delete(database_path) out of the skip_data conditional as it runs in both branches either way. Assisted-by: Claude Code:claude-sonnet-4-6 --- lib/morph-cli.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/morph-cli.rb b/lib/morph-cli.rb index 9397456..e148bd7 100644 --- a/lib/morph-cli.rb +++ b/lib/morph-cli.rb @@ -22,12 +22,8 @@ def self.execute(directory, _development, env_config, skip_data: false) end database_path = MorphCLI.database_path(directory) - if skip_data - all_paths.delete(database_path) - database_path = nil - else - all_paths.delete(database_path) - end + all_paths.delete(database_path) + database_path = nil if skip_data size = MorphCLI.get_dir_size(directory, all_paths + [database_path].compact) puts "Uploading #{size}#{" (including #{database_path})" if database_path}..."