Ruby 3.2.6 & compact_index support (#91)

* Compact Index support for the discerning bundler of Gemfiles
* Increased caching during gemirro index for faster operation
* Better error catching for command line (no change to syntax)
* New config option: update_thread_count , to control how many threads are used with disk-intensive operations.
* Better support for manually uploaded (file copy, sftp, etc.) gems in gemirro index --update
* Now requires Ruby 3.0 due to compact_index 0.15.0 dependency.

.rubocop.yml

@@ -1,6 +1,7 @@
 AllCops:
+  SuggestExtensions: false
   NewCops: enable
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.0
   Include:
     - '**/Gemfile'
     - lib/**/*.rb
@@ -15,9 +16,9 @@ AllCops:
 Naming/FileName:
   Exclude:
     - Rakefile
-Layout/MethodLength:
+Metrics/MethodLength:
   Enabled: false
-Layout/ClassLength:
+Metrics/ClassLength:
   Enabled: false
 Metrics/CyclomaticComplexity:
   Enabled: false
@@ -35,3 +36,5 @@ Style/OptionalBooleanParameter:
   Enabled: false
 Lint/MissingSuper:
   Enabled: false
+Style/TrailingUnderscoreVariable:
+  Enabled: false

Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    gemirro (1.5.0)
+    gemirro (1.6.0)
       addressable (~> 2.8)
       builder (~> 3.2)
+      compact_index (~> 0.15)
       confstruct (~> 1.1)
       erubis (~> 2.7)
       httpclient (~> 2.8)
@@ -21,6 +22,7 @@ GEM
     ast (2.4.2)
     base64 (0.2.0)
     builder (3.3.0)
+    compact_index (0.15.0)
     confstruct (1.1.0)
       hashie (>= 3.3, < 5)
     daemons (1.4.1)
@@ -36,7 +38,7 @@ GEM
     mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     parallel (1.26.3)
-    parser (3.3.6.0)
+    parser (3.3.7.0)
       ast (~> 2.4.1)
       racc
     public_suffix (6.0.1)
@@ -112,4 +114,4 @@ DEPENDENCIES
   simplecov (~> 0.21)
 
 BUNDLED WITH
-   2.1.4
+   2.4.19

MANIFEST

@@ -8,7 +8,6 @@ Rakefile
 bin/gemirro
 gemirro.gemspec
 lib/gemirro.rb
-lib/gemirro/cache.rb
 lib/gemirro/cli.rb
 lib/gemirro/cli/index.rb
 lib/gemirro/cli/init.rb
@@ -32,7 +31,6 @@ lib/gemirro/versions_fetcher.rb
 lib/gemirro/versions_file.rb
 spec/fixtures/gems/gemirro-0.0.1.gem
 spec/fixtures/quick/gemirro-0.0.1.gemspec.rz
-spec/gemirro/cache_spec.rb
 spec/gemirro/cli_spec.rb
 spec/gemirro/configuration_spec.rb
 spec/gemirro/gem_spec.rb

README.md

@@ -6,7 +6,7 @@ More, to mirroring a source, you only need to start the server, and gems will au
 
 ## Requirements
 
-* Ruby 2.2 or newer
+* Ruby 3.0 or newer
 * Enough space to store Gems
 * A recent version of Rubygems (`gem update --system`)
 

bin/gemirro

@@ -4,4 +4,10 @@
 require File.expand_path('../../lib/gemirro', __FILE__)
 
 options = Gemirro::CLI.options
-puts options if options.parse.empty?
+
+begin
+  puts options if options.parse.empty?
+rescue Slop::InvalidOptionError => e
+  puts e.message
+  puts options
+end

gemirro.gemspec

@@ -16,10 +16,11 @@ Gem::Specification.new do |s|
 
   s.files = File.read(File.expand_path('../MANIFEST', __FILE__)).split("\n")
 
-  s.required_ruby_version = '>= 2.5'
+  s.required_ruby_version = '>= 3.0'
 
   s.add_dependency 'addressable', '~>2.8'
   s.add_dependency 'builder', '~>3.2'
+  s.add_dependency 'compact_index', '~> 0.15'
   s.add_dependency 'confstruct', '~>1.1'
   s.add_dependency 'erubis', '~>2.7'
   s.add_dependency 'httpclient', '~>2.8'

lib/gemirro.rb

@@ -20,7 +20,6 @@
 
 require 'gemirro/version'
 require 'gemirro/configuration'
-require 'gemirro/cache'
 require 'gemirro/utils'
 require 'gemirro/gem'
 require 'gemirro/gem_version'

lib/gemirro/cli/index.rb

@@ -22,8 +22,17 @@
     indexer    = Gemirro::Indexer.new(config.destination)
     indexer.ui = Gem::SilentUI.new
 
-    config.logger.info('Generating indexes')
-    indexer.generate_index if opts[:u].nil?
-    indexer.update_index unless opts[:u].nil?
+    if File.exist?(File.join(config.destination, "specs.#{Gem.marshal_version}.gz"))
+      if opts[:u]
+        config.logger.info('Generating index updates')
+        indexer.update_index
+      else
+        config.logger.info('Generating indexes')
+        indexer.generate_index
+      end
+    else
+      config.logger.error("/public/specs.#{Gem.marshal_version}.gz file is missing.")
+      config.logger.error('Run "gemirro update" before running index.')
+    end
   end
 end

lib/gemirro/cli/init.rb

@@ -26,6 +26,11 @@
       end
     end
 
+    # make sure index updates blank local specs
+    ['specs.4.8', 'latest_specs.4.8', 'prerelease_specs.4.8'].each do |s|
+      File.utime(Time.at(0), Time.at(0), File.join(directory, 'public', s))
+    end
+
     puts "Initialized empty mirror in #{directory}"
   end
 end

lib/gemirro/cli/update.rb

@@ -17,5 +17,11 @@
     gems     = Gemirro::GemsFetcher.new(source, versions)
 
     gems.fetch
+
+    source.gems.each do |gem|
+      gem.gemspec = true
+    end
+
+    gems.fetch
   end
 end

lib/gemirro/configuration.rb

@@ -12,7 +12,7 @@ def self.configuration
         error_log: '/tmp/gemirro.access.log',
         daemonize: true
       },
-
+      update_thread_count: begin; Etc.nprocessors - 1; rescue StandardError; 4; end,
       update_on_fetch: true,
       fetch_gem: true
     }

lib/gemirro/gems_fetcher.rb

@@ -31,10 +31,10 @@ def fetch
           gem.platform = versions[1] if versions
           version = versions[0] if versions
           if gem.gemspec?
-            gemfile = fetch_gemspec(gem, version)
-            if gemfile
+            gemspec = fetch_gemspec(gem, version)
+            if gemspec
               Utils.configuration.mirror_gemspecs_directory
-                   .add_file(gem.gemspec_filename(version), gemfile)
+                   .add_file(gem.gemspec_filename(version), gemspec)
             end
           else
             gemfile = fetch_gem(gem, version)
@@ -100,15 +100,10 @@ def fetch_gemspec(gem, version)
     #
     def fetch_gem(gem, version)
       filename = gem.filename(version)
-      satisfied = if gem.only_latest?
-                    true
-                  else
-                    gem.requirement.satisfied_by?(version)
-                  end
+      satisfied = gem.only_latest? || gem.requirement.satisfied_by?(version)
       name = gem.name
 
-      if gem_exists?(filename) || ignore_gem?(name, version, gem.platform) ||
-         !satisfied
+      if gem_exists?(filename) || ignore_gem?(name, version, gem.platform) || !satisfied
         Utils.logger.debug("Skipping #{filename}")
         return
       end