Merge branch 'tag-1.4'

Author: weiweishi

Gemfile

@@ -69,6 +69,9 @@ gem "pdf-reader"
 # to generate sitemap for google scholar et al
 gem 'sitemap', github: 'ualbertalib/rails-sitemap'
 
+# to fetch noid from fedora for reindex job
+gem 'rest-client'
+
 gem 'noid', '~> 0.8'
 
 group :test do
@@ -96,5 +99,4 @@ group :development, :test do
   gem "better_errors"
   gem "binding_of_caller"
   gem 'ruby-prof'
-  gem 'rest-client'
 end

README.md

@@ -138,7 +138,8 @@ Audit Fix
 A set of rake tasks is also added for index jobs:
 * ```rake hydranorth:solr:index[id]```         Index a single object with ID
 * ```rake hydranorth:solr:index_pairtree[input]```   Index with a pairtree structure
-* ```rake hydranorth:solr:update_generic_file_index```  Index all Generic File objects
+* ```rake hydranorth:solr:batch_index[directory|file]``` Index from a list of noids, usually from a solr csv output that just contains noids. 
+* ```rake hydranorth:solr:reindex_all```	Complete reindex of the repository
 
 A shell script will update namespace uris
 * ```/bin/fix/fix.rb``` is to update all the namespace uris. Requires user to replace @server with the Fedora server location before using.

lib/tasks/hydranorth-solr.rake

@@ -6,6 +6,29 @@ require 'rdf/turtle'
 namespace :hydranorth do
   namespace :solr do
 
+    desc "Index objects by batch files"
+    task :batch_index, [:batch_dir] => :environment do |t, args|
+      batch_dir = args[:batch_dir]
+      raise "Please provide a directory where the batch files are located" if batch_dir.nil?
+      RakeLogger.info "Run through all the files in #{batch_dir}"
+      Dir.glob(batch_dir+"*").each do |f|
+        RakeLogger.info "Currently working on file #{f}"
+        start = Time.now
+        File.open(f, 'r+').each_line do |l|
+          noid = l.strip
+          RakeLogger.info "Currently working on #{noid}"
+          begin
+            ActiveFedora::Base.find(noid).update_index
+          rescue Exception => e
+            RakeLogger.error "ERROR: #{noid} with #{e.message}"
+          end
+        end
+        finish = Time.now
+        used_time = finish - start
+        RakeLogger.info "This file #{f} used #{used_time}"
+      end
+    end
+
     desc "Index a single object in solr"
     task :index, [:id] => :environment do |t, args|
       id = args[:id]
@@ -17,37 +40,64 @@ namespace :hydranorth do
       RakeLogger.info "reindexed #{id} used #{used_time}"
     end
 
-    desc "update the index on all GenericFiles"
-    task update_generic_file_index: :environment do
-      GenericFile.all.each(&:update_index)
-    end
-
     desc "Index with a pairtree"
     task "index_pairtree", [:input] => :environment do |t, args|
       input = args[:input]
       RakeLogger.info "***********START index_pairtree***************"
       read_config
+      RakeLogger.info "reindex #{input}"
+      index_pairtree(input)
+      RakeLogger.info "***********FINISH index_pairtree**************"
+    end
+
+    desc "Complete Reindex"
+    task "reindex_all" => :environment do |t, args|
+      RakeLogger.info "***********START reindex *********************"
+      read_config
       start = Time.now
-      objects = find_objects(input)
-      objects.each do |o|
-        index_single(o)
-      end
+      count = index_all_objects
       finish = Time.now
-      used_time = finish - start
-      RakeLogger.info "Indexed #{objects.size} objects, used #{used_time}"
+      used_time = finish-start
+      RakeLogger.info "A Complete Reindex of #{count} objects, used #{used_time}"
       RakeLogger.info "***********FINISH index_pairtree**************"
     end
 
     def read_config
       rails_env = Rails.env
- 
+
       config = YAML.load_file("config/fedora.yml")
       @user = config[rails_env]['user']
       @password = config[rails_env]['password']
       @rest = config[rails_env]['url']
       @base_path = config[rails_env]['base_path']
     end
 
+    def index_all_objects
+       count = 0
+       [(0..9),('a'..'z')].map {|i| i.to_a}.flatten.each do |a|
+          [(0..9),('a'..'z')].map {|i| i.to_a}.flatten.each do |b|
+            pairtree = a.to_s + b.to_s
+            number_reindexed = index_pairtree(pairtree)
+            count = count + number_reindexed
+         end
+       end
+       return count
+    end
+
+    def index_pairtree(pairtree)
+      RakeLogger.info "Start to reindex all objects starting with #{pairtree}"
+      start = Time.now
+      objects = find_objects(pairtree)
+      RakeLogger.info "Reindex #{objects.size} objects"
+      objects.each do |o|
+        index_single(o)
+      end
+      finish = Time.now
+      used_time = finish-start
+      RakeLogger.info "Indexed #{objects.size} objects that starts with #{pairtree}, used #{used_time} seconds"
+      return objects.size
+    end
+
     def find_objects(input)
       require 'rest-client'
       objects=[]
@@ -84,6 +134,7 @@ namespace :hydranorth do
     end
 
     def index_single(id)
+      RakeLogger.info "start reindexing #{id}"
       start = Time.now
         ActiveFedora::Base.find(id).update_index
       finish = Time.now
@@ -92,3 +143,5 @@ namespace :hydranorth do
     end
   end
 end
+
+

lib/tasks/hydranorth.rake

@@ -79,11 +79,6 @@ namespace :hydranorth do
   end
 
 
-  desc "Re-solrize all objects"
-  task resolrize: :environment do
-    Sufia.queue.push(ResolrizeJob.new)
-  end
-
   namespace :export do
     desc "Dump metadata as RDF/XML for e.g. Summon integration"
     task rdfxml: :environment do