added remove indexes before bulk insert and add indexes after

Gemfile

@@ -10,7 +10,12 @@ if defined?(JRUBY_VERSION)
 else
   gem 'sqlite3'
   gem 'mysql2'
-  gem 'pg'
+  if RUBY_VERSION == '1.8.7'
+    gem 'pg', '0.17.1'
+    gem 'i18n', '0.6.11'
+  else
+    gem 'pg'
+  end
   if rails_version =~ /(^|[^.\d])(2|3\.0)\.\d+/
     gem 'activerecord-mysql2-adapter'
   end

README.markdown

@@ -145,6 +145,8 @@ reset database:
 
     rake dump:restore MIGRATE_DOWN=reset
 
+`REBUILD_INDEXES` — remove indexes for each table before restore, and create them after restore if you pass "1", "true" or "yes". `REBUILD_INDEXES` is useful to speed up restoring for large tables (see https://github.com/toy/dump/pull/12#issuecomment-69462275), but may affect index structure because of database adapters implementations.
+
 `RESTORE_SCHEMA` — don't read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored)
 
 don't restore schema:

lib/dump/env.rb

@@ -16,6 +16,7 @@ module Env
       :backup => %w[BACKUP AUTOBACKUP AUTO_BACKUP],
       :transfer_via => %w[TRANSFER_VIA],
       :migrate_down => %w[MIGRATE_DOWN],
+      :rebuild_indexes => %w[REBUILD_INDEXES],
       :restore_schema => %w[RESTORE_SCHEMA],
       :restore_tables => %w[RESTORE_TABLES],
       :restore_assets => %w[RESTORE_ASSETS],
@@ -33,6 +34,7 @@ module Env
       :backup => 'no autobackup if you pass "0", "no" or "false"',
       :transfer_via => 'transfer method (rsync, sftp or scp)',
       :migrate_down => 'don\'t run down for migrations not present in dump if you pass "0", "no" or "false"; pass "reset" to recreate (drop and create) db',
+      :rebuild_indexes => 'remove indexes before restore, and add them after if you pass "1", "true" or "yes"; speed up restoring for large tables, but may affect indexes structure (see README for details)',
       :restore_schema => 'don\'t read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored)',
       :restore_tables => 'works as TABLES, but for restoring',
       :restore_assets => 'works as ASSETS, but for restoring',

lib/dump/reader.rb

@@ -5,6 +5,8 @@
 require 'rake'
 require 'zlib'
 require 'tempfile'
+require 'dump/reader/summary'
+require 'dump/reader/assets'
 
 module Dump
   # Reading dump
@@ -19,35 +21,11 @@ def self.restore(path)
           dump.read_schema
 
           dump.read_tables
-          dump.read_assets
+          Assets.new(dump).read
         end
       end
     end
 
-    # Helper class for building summary of dump
-    class Summary
-      attr_reader :text
-      alias_method :to_s, :text
-      def initialize
-        @text = ''
-      end
-
-      def header(header)
-        @text << "  #{header}:\n"
-      end
-
-      def data(entries)
-        entries.each do |entry|
-          @text << "    #{entry}\n"
-        end
-      end
-
-      # from ActionView::Helpers::TextHelper
-      def self.pluralize(count, singular)
-        "#{count} #{count == 1 ? singular : singular.pluralize}"
-      end
-    end
-
     def self.summary(path, options = {})
       new(path).open do |dump|
         dump.read_config
@@ -98,7 +76,8 @@ def open
     def find_entry(matcher)
       stream.each do |entry|
         if entry.full_name.match(matcher)
-          # we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start
+          # we can not return entry - after exiting stream.each
+          # the entry will be invalid and will read from tar start
           return yield(entry)
         end
       end
@@ -186,109 +165,42 @@ def read_tables
       end
     end
 
+    def rebuild_indexes?
+      Dump::Env.yes?(:rebuild_indexes)
+    end
+
     def read_table(table, rows_count)
       find_entry("#{table}.dump") do |entry|
         table_sql = quote_table_name(table)
         clear_table(table_sql)
 
-        columns = Marshal.load(entry)
-        columns_sql = columns_insert_sql(columns)
-        Progress.start(table, rows_count) do
-          until entry.eof?
-            rows_sql = []
-            1000.times do
-              rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof?
-            end
-
-            begin
-              insert_into_table(table_sql, columns_sql, rows_sql)
-              Progress.step(rows_sql.length)
-            rescue
-              rows_sql.each do |row_sql|
-                insert_into_table(table_sql, columns_sql, row_sql)
-                Progress.step
-              end
-            end
+        columns_sql = columns_insert_sql(Marshal.load(entry))
+        if rebuild_indexes?
+          with_disabled_indexes table do
+            bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql)
           end
+        else
+          bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql)
         end
         fix_sequence!(table)
       end
     end
 
-    def read_assets
-      return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty?
-      return if config[:assets].blank?
-
-      assets = config[:assets]
-      if assets.is_a?(Hash)
-        assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value }
-        assets_paths = assets.keys
-      else
-        assets_count, assets_paths = nil, assets
-      end
-
-      if Dump::Env[:restore_assets]
-        assets_paths.each do |asset|
-          Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child|
-            next unless read_asset?(child, Dump.rails_root)
-            case
-            when File.file?(child)
-              File.unlink(child)
-            when File.directory?(child)
-              begin
-                Dir.unlink(child)
-              rescue Errno::ENOTEMPTY
-                nil
-              end
-            end
-          end
-        end
-      else
-        Dump::Env.with_env(:assets => assets_paths.join(':')) do
-          Rake::Task['assets:delete'].invoke
-        end
-      end
-
-      read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix|
-        if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix)
-          stream.extract_entry(root, entry)
-        end
-      end
-    end
-
-    def read_asset?(path, prefix)
-      Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value|
-        File.fnmatch(File.join(prefix, value), path) ||
-          File.fnmatch(File.join(prefix, value, '**'), path)
-      end
-    end
-
-    def read_assets_entries(_assets_paths, assets_count)
-      Progress.start('Assets', assets_count || 1) do
-        found_assets = false
-        # old style - in separate tar
-        find_entry('assets.tar') do |assets_tar|
-          def assets_tar.rewind
-            # rewind will fail - it must go to center of gzip
-            # also we don't need it - this is last step in dump restore
+    def bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql)
+      Progress.start(table, rows_count) do
+        until entry.eof?
+          rows_sql = []
+          1000.times do
+            rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof?
           end
-          Archive::Tar::Minitar.open(assets_tar) do |inp|
-            inp.each do |entry|
-              yield inp, Dump.rails_root, entry, nil
-              Progress.step if assets_count
-            end
-          end
-          found_assets = true
-        end
 
-        unless found_assets
-          # new style - in same tar
-          assets_root_link do |tmpdir, prefix|
-            stream.each do |entry|
-              if entry.full_name.starts_with?("#{prefix}/")
-                yield stream, tmpdir, entry, prefix
-                Progress.step if assets_count
-              end
+          begin
+            insert_into_table(table_sql, columns_sql, rows_sql)
+            Progress.step(rows_sql.length)
+          rescue
+            rows_sql.each do |row_sql|
+              insert_into_table(table_sql, columns_sql, row_sql)
+              Progress.step
             end
           end
         end

lib/dump/reader/assets.rb

@@ -0,0 +1,93 @@
+module Dump
+  class Reader < Snapshot
+    # Helper class for reading assets
+    class Assets
+      attr_reader :dump
+      delegate :config, :to => :dump
+
+      def initialize(dump)
+        @dump = dump
+      end
+
+      def read
+        return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty?
+        return if config[:assets].blank?
+
+        assets = config[:assets]
+        if assets.is_a?(Hash)
+          assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value }
+          assets_paths = assets.keys
+        else
+          assets_count, assets_paths = nil, assets
+        end
+
+        if Dump::Env[:restore_assets]
+          assets_paths.each do |asset|
+            Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child|
+              next unless read_asset?(child, Dump.rails_root)
+              case
+              when File.file?(child)
+                File.unlink(child)
+              when File.directory?(child)
+                begin
+                  Dir.unlink(child)
+                rescue Errno::ENOTEMPTY
+                  nil
+                end
+              end
+            end
+          end
+        else
+          Dump::Env.with_env(:assets => assets_paths.join(':')) do
+            Rake::Task['assets:delete'].invoke
+          end
+        end
+
+        read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix|
+          if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix)
+            stream.extract_entry(root, entry)
+          end
+        end
+      end
+
+      def read_assets_entries(_assets_paths, assets_count)
+        Progress.start('Assets', assets_count || 1) do
+          found_assets = false
+          # old style - in separate tar
+          dump.find_entry('assets.tar') do |assets_tar|
+            def assets_tar.rewind
+              # rewind will fail - it must go to center of gzip
+              # also we don't need it - this is last step in dump restore
+            end
+            Archive::Tar::Minitar.open(assets_tar) do |inp|
+              inp.each do |entry|
+                yield inp, Dump.rails_root, entry, nil
+                Progress.step if assets_count
+              end
+            end
+            found_assets = true
+          end
+
+          unless found_assets
+            # new style - in same tar
+            dump.send :assets_root_link do |tmpdir, prefix|
+              dump.stream.each do |entry|
+                if entry.full_name.starts_with?("#{prefix}/")
+                  yield dump.stream, tmpdir, entry, prefix
+                  Progress.step if assets_count
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def read_asset?(path, prefix)
+        Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value|
+          File.fnmatch(File.join(prefix, value), path) ||
+            File.fnmatch(File.join(prefix, value, '**'), path)
+        end
+      end
+    end
+  end
+end

lib/dump/reader/summary.rb

@@ -0,0 +1,27 @@
+module Dump
+  class Reader < Snapshot
+    # Helper class for building summary of dump
+    class Summary
+      attr_reader :text
+      alias_method :to_s, :text
+      def initialize
+        @text = ''
+      end
+
+      def header(header)
+        @text << "  #{header}:\n"
+      end
+
+      def data(entries)
+        entries.each do |entry|
+          @text << "    #{entry}\n"
+        end
+      end
+
+      # from ActionView::Helpers::TextHelper
+      def self.pluralize(count, singular)
+        "#{count} #{count == 1 ? singular : singular.pluralize}"
+      end
+    end
+  end
+end

lib/dump/table_manipulation.rb

@@ -29,6 +29,35 @@ def clear_table(table_sql)
       connection.delete("DELETE FROM #{table_sql}", 'Clearing table')
     end
 
+    def with_disabled_indexes(table, &block)
+      table_indexes = ActiveRecord::Base.connection.indexes(table)
+      remove_indexes(table_indexes)
+      block.call
+      add_indexes(table_indexes)
+    end
+
+    def remove_indexes(indexes)
+      indexes.each do |index|
+        ActiveRecord::Base.connection.remove_index index.table, :name => index.name
+      end
+    end
+
+    VALID_INDEX_OPTIONS = [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type].freeze
+
+    def index_options(index)
+      options = VALID_INDEX_OPTIONS.map{ |field| [field, index.members.include?(field) ? index.send(field) : nil] }
+      non_empty_options = options.select{ |pair| !pair[1].nil? }
+      non_empty_options << [:length, index.lengths] if index.try(:lengths).present?
+
+      Hash[*non_empty_options.flatten]
+    end
+
+    def add_indexes(indexes)
+      indexes.each do |index|
+        ActiveRecord::Base.connection.add_index index.table, index.columns, index_options(index)
+      end
+    end
+
     def insert_into_table(table_sql, columns_sql, values_sql)
       values_sql = values_sql.join(',') if values_sql.is_a?(Array)
       sql = "INSERT INTO #{table_sql} #{columns_sql} VALUES #{values_sql}"