Fix record history export

migration/v1_to_v2/data/export_data.rb

@@ -18,7 +18,7 @@ def exporters
 timestamp = DateTime.now.strftime('%Y%m%d%H%M%S')
 export_dir = "record-data-files-#{timestamp}"
 exporters.each do |exporter|
-  data_exporter = Object.const_get(exporter).new(export_dir: export_dir, batch_size: 250)
+  data_exporter = Object.const_get(exporter).new(export_dir: export_dir)
   data_exporter.export
 end
 

migration/v1_to_v2/data/exporters/attachment_exporter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative('data_exporter.rb')
+require 'erb'
 
 # rubocop:disable Metrics/ClassLength
 # Class that get record's attachments and generate files to be inserted
@@ -18,7 +19,7 @@ class AttachmentExporter < DataExporter
     'other_documents' => 'other_documents'
   }.freeze
 
-  def initialize(options = {})
+  def initialize(options = { batch_size: 50 })
     super(options)
     @indent = 0
     @json_to_export = {}
@@ -94,9 +95,14 @@ def build_data_to_attach(files, folder, type, form)
       name = value.is_a?(String) ? value : value.file_name
 
       @json_to_export[type][@record_id][form] << {
-        record_type: @record_type.to_s, record_id: @record_id, field_name: form,
-        file_name: name, date: value.try(:date), comments: value.try(:comments),
-        is_current: value.try(:is_current) || false, description: value.try(:document_description),
+        record_type: @record_type.to_s,
+        record_id: @record_id,
+        field_name: form,
+        file_name: name,
+        date: value.try(:date),
+        comments: value.try(:comments),
+        is_current: value.try(:is_current) || false,
+        description: value.try(:document_description),
         path: "/#{@record_id}/#{form}/#{name}"
       }
     end
@@ -152,34 +158,53 @@ def add_date_to_file_for_attachment(date)
     @output.puts "attachement.date = \"#{date.strftime('%Y-%m-%d')}\""
   end
 
-  def write_script_for_attachment(form_name, files)
-    files.each do |data|
-      @output.puts "puts 'Inserting \"#{get_attachment_type(data[:path])}\" to #{data[:record_id]}'"
-      @output.puts "attachement = Attachment.new(#{data.except(:path, :field_name, :date)})"
-      add_date_to_file_for_attachment(data[:date])
-      @output.puts "attachement.record_type = #{data[:record_type]}"
-      add_attachment_type_to_file_for_attachment(data[:path])
-      @output.puts "attachement.field_name = '#{get_field_name(form_name)}'"
-      @output.puts "attachement.file.attach(io: File.open(\"\#{File.dirname(__FILE__)}#{data[:path]}\"), filename: '#{data[:file_name]}')"
-      @output.puts "begin"
-      @output.puts "  attachement.save!"
-      @output.puts "rescue StandardError => e"
-      @output.puts "  puts \"Cannot attach #{data[:file_name]}. Error \#{e.message}\""
-      @output.puts "end\n\n\n"
+  ATTACHMENT_IMPORTER_TEMPLATE = ERB.new(<<~RUBY)
+    puts "Inserting <%= get_attachment_type(data[:path]) %> to <%= data[:record_id] >"
+    attachement = Attachment.new(<%= data.except(:path, :field_name, :date) %>)
+    <% if date.present? %>attachement.date = "<%= date.strftime('%Y-%m-%d') %>"<% end %>
+    attachement.record_type = <%= data[:record_type] %>
+    attachement.attachment_type = "<%= get_attachment_type(path) %>"
+    attachement.field_name = "<%= get_field_name(form_name) %>"
+    attachement.file.attach(io: File.open("\#{File.dirname(__FILE__)}<%= data[:path] %>"), filename: <%= data[:file_name].inspect %>)
+    begin
+      attachement.save!
+    rescue StandardError => e
+      puts "Cannot attach <%= data[:file_name].inspect %>. Error \#{e.message}"
     end
-  end
 
-  def build_file(folder_to_save, type, sufix)
-    initialize_script_for_attachment(folder_to_save, type, sufix)
-    data_object_names.each do |object_name|
-      next if @json_to_export[object_name].blank?
+  RUBY
 
-      @json_to_export[object_name].values.each do |value|
-        value.each do |form_name, files|
-          write_script_for_attachment(form_name, files)
-        end
-      end
-    end
+  def render_attachment_importer(form_name, data)
+    @output.puts ATTACHMENT_IMPORTER_TEMPLATE.result(binding)
   end
+
+   def build_file(folder_to_save, type, sufix)
+     initialize_script_for_attachment(folder_to_save, type, sufix)
+
+     attachments = []
+     data_object_names.each do |type|
+       records = @json_to_export[type]
+
+       next if records.blank?
+
+       records.values.each do |form|
+         form.each do |form_name, files|
+           next if files.nil?
+
+           attachments.push(*files.map { |file| [form_name, file] })
+         end
+       end
+     end
+
+
+     attachments.each_slice(10) do |slice|
+       slice.each do |form, file|
+         render_attachment_importer(form, file)
+       end
+
+       @output.puts "# force ruby to gargabe collect here as attachmented files can build up in memory!"
+       @output.puts "GC.start"
+     end
+   end
 end
 # rubocop:enable Metrics/ClassLength

migration/v1_to_v2/data/exporters/data_exporter.rb

@@ -143,14 +143,8 @@ def value_to_ruby_string(value, include_blank = false)
   # rubocop:enable Metrics/MethodLength
   # rubocop:enable Metrics/AbcSize
 
-  def valid_key?(key)
-    return false if key.is_a?(Integer) || key.include?('-') || key.include?(' ') || !key.match?('^[a-zA-Z]')
-
-    true
-  end
-
   def key_to_ruby(key)
-    valid_key?(key) ? key : "'#{key}'"
+    "'#{ key.to_s.gsub(/'/, "\\'") }'"
   end
 
   def parse_date_and_location_fields(object, data_hash)