From 245d369404f0c26a017e06b068e57669e87d13c3 Mon Sep 17 00:00:00 2001 From: Sudrien Date: Fri, 31 Jan 2025 21:35:28 -0500 Subject: [PATCH] Ruby 3.2.6 & compact_index support (#91) * Compact Index support for the discerning bundler of Gemfiles * Increased caching during gemirro index for faster operation * Better error catching for command line (no change to syntax) * New config option: update_thread_count , to control how many threads are used with disk-intensive operations. * Better support for manually uploaded (file copy, sftp, etc.) gems in gemirro index --update * Now requires Ruby 3.0 due to compact_index 0.15.0 dependency. --- .rubocop.yml | 9 +- Gemfile.lock | 8 +- MANIFEST | 2 - README.md | 2 +- bin/gemirro | 8 +- gemirro.gemspec | 3 +- lib/gemirro.rb | 1 - lib/gemirro/cache.rb | 115 ------- lib/gemirro/cli/index.rb | 15 +- lib/gemirro/cli/init.rb | 5 + lib/gemirro/cli/update.rb | 6 + lib/gemirro/configuration.rb | 2 +- lib/gemirro/gems_fetcher.rb | 15 +- lib/gemirro/indexer.rb | 481 ++++++++++++++++++++++----- lib/gemirro/mirror_file.rb | 1 + lib/gemirro/server.rb | 239 ++++--------- lib/gemirro/utils.rb | 210 ++++++++---- lib/gemirro/version.rb | 2 +- lib/gemirro/versions_fetcher.rb | 8 +- spec/gemirro/cache_spec.rb | 32 -- spec/gemirro/server_spec.rb | 124 ++++--- template/config.rb | 6 + template/public/latest_specs.4.8 | Bin 0 -> 4 bytes template/public/prerelease_specs.4.8 | Bin 0 -> 4 bytes template/public/specs.4.8 | Bin 0 -> 4 bytes views/gem.erb | 14 +- views/index.erb | 10 +- 27 files changed, 762 insertions(+), 556 deletions(-) delete mode 100644 lib/gemirro/cache.rb delete mode 100644 spec/gemirro/cache_spec.rb create mode 100644 template/public/latest_specs.4.8 create mode 100644 template/public/prerelease_specs.4.8 create mode 100644 template/public/specs.4.8 diff --git a/.rubocop.yml b/.rubocop.yml index 8b57299..7dece12 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,7 @@ AllCops: + SuggestExtensions: false NewCops: enable - TargetRubyVersion: 2.5 + TargetRubyVersion: 3.0 Include: - '**/Gemfile' - lib/**/*.rb @@ -15,9 +16,9 @@ AllCops: Naming/FileName: Exclude: - Rakefile -Layout/MethodLength: +Metrics/MethodLength: Enabled: false -Layout/ClassLength: +Metrics/ClassLength: Enabled: false Metrics/CyclomaticComplexity: Enabled: false @@ -35,3 +36,5 @@ Style/OptionalBooleanParameter: Enabled: false Lint/MissingSuper: Enabled: false +Style/TrailingUnderscoreVariable: + Enabled: false diff --git a/Gemfile.lock b/Gemfile.lock index e04eee8..c8365fd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,10 @@ PATH remote: . specs: - gemirro (1.5.0) + gemirro (1.6.0) addressable (~> 2.8) builder (~> 3.2) + compact_index (~> 0.15) confstruct (~> 1.1) erubis (~> 2.7) httpclient (~> 2.8) @@ -21,6 +22,7 @@ GEM ast (2.4.2) base64 (0.2.0) builder (3.3.0) + compact_index (0.15.0) confstruct (1.1.0) hashie (>= 3.3, < 5) daemons (1.4.1) @@ -36,7 +38,7 @@ GEM mustermann (3.0.3) ruby2_keywords (~> 0.0.1) parallel (1.26.3) - parser (3.3.6.0) + parser (3.3.7.0) ast (~> 2.4.1) racc public_suffix (6.0.1) @@ -112,4 +114,4 @@ DEPENDENCIES simplecov (~> 0.21) BUNDLED WITH - 2.1.4 + 2.4.19 diff --git a/MANIFEST b/MANIFEST index 58b106b..6bdbf51 100644 --- a/MANIFEST +++ b/MANIFEST @@ -8,7 +8,6 @@ Rakefile bin/gemirro gemirro.gemspec lib/gemirro.rb -lib/gemirro/cache.rb lib/gemirro/cli.rb lib/gemirro/cli/index.rb lib/gemirro/cli/init.rb @@ -32,7 +31,6 @@ lib/gemirro/versions_fetcher.rb lib/gemirro/versions_file.rb spec/fixtures/gems/gemirro-0.0.1.gem spec/fixtures/quick/gemirro-0.0.1.gemspec.rz -spec/gemirro/cache_spec.rb spec/gemirro/cli_spec.rb spec/gemirro/configuration_spec.rb spec/gemirro/gem_spec.rb diff --git a/README.md b/README.md index 50a7708..c43e2eb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ More, to mirroring a source, you only need to start the server, and gems will au ## Requirements -* Ruby 2.2 or newer +* Ruby 3.0 or newer * Enough space to store Gems * A recent version of Rubygems (`gem update --system`) diff --git a/bin/gemirro b/bin/gemirro index bac3b97..1e9f055 100755 --- a/bin/gemirro +++ b/bin/gemirro @@ -4,4 +4,10 @@ require File.expand_path('../../lib/gemirro', __FILE__) options = Gemirro::CLI.options -puts options if options.parse.empty? + +begin + puts options if options.parse.empty? +rescue Slop::InvalidOptionError => e + puts e.message + puts options +end diff --git a/gemirro.gemspec b/gemirro.gemspec index 88f73ef..2aec3c3 100644 --- a/gemirro.gemspec +++ b/gemirro.gemspec @@ -16,10 +16,11 @@ Gem::Specification.new do |s| s.files = File.read(File.expand_path('../MANIFEST', __FILE__)).split("\n") - s.required_ruby_version = '>= 2.5' + s.required_ruby_version = '>= 3.0' s.add_dependency 'addressable', '~>2.8' s.add_dependency 'builder', '~>3.2' + s.add_dependency 'compact_index', '~> 0.15' s.add_dependency 'confstruct', '~>1.1' s.add_dependency 'erubis', '~>2.7' s.add_dependency 'httpclient', '~>2.8' diff --git a/lib/gemirro.rb b/lib/gemirro.rb index 039e217..389c4aa 100644 --- a/lib/gemirro.rb +++ b/lib/gemirro.rb @@ -20,7 +20,6 @@ require 'gemirro/version' require 'gemirro/configuration' -require 'gemirro/cache' require 'gemirro/utils' require 'gemirro/gem' require 'gemirro/gem_version' diff --git a/lib/gemirro/cache.rb b/lib/gemirro/cache.rb deleted file mode 100644 index 1ef496d..0000000 --- a/lib/gemirro/cache.rb +++ /dev/null @@ -1,115 +0,0 @@ -# frozen_string_literal: true - -module Gemirro - ## - # The Cache class contains all method to store marshal informations - # into files. - # - # @!attribute [r] root_path - # @return [String] - # - class Cache - attr_reader :root_path - - ## - # Initialize cache root path - # - # @param [String] path - # - def initialize(path) - @root_path = path - create_root_path - end - - ## - # Create root path - # - def create_root_path - FileUtils.mkdir_p(@root_path) - end - - ## - # Flush cache directory - # - def flush - FileUtils.rm_rf(@root_path) - create_root_path - end - - ## - # Flush key - # - # @param [String] key - # - def flush_key(key) - path = key_path(key2hash(key)) - FileUtils.rm_f(path) - end - - ## - # Cache data - # - # @param [String] key - # - # @return [Mixed] - # - def cache(key) - key_hash = key2hash(key) - read(key_hash) || (write(key_hash, yield) if block_given?) - end - - private - - ## - # Convert key to hash - # - # @param [String] key - # - # @return [String] - # - def key2hash(key) - Digest::MD5.hexdigest(key) - end - - ## - # Path from key hash - # - # @param [String] key_hash - # - # @return [String] - # - def key_path(key_hash) - File.join(@root_path, key_hash) - end - - ## - # Read cache - # - # @param [String] key_hash - # - # @return [Mixed] - # - def read(key_hash) - path = key_path(key_hash) - Marshal.load(File.open(path)) if File.exist?(path) - end - - ## - # write cache - # - # @param [String] key_hash - # @param [Mixed] value - # - # @return [Mixed] - # - def write(key_hash, value) - return value if value.nil? || value.empty? - - File.open(key_path(key_hash), 'wb') do |f| - Marshal.dump(value, f) - end - - value - end - end -end diff --git a/lib/gemirro/cli/index.rb b/lib/gemirro/cli/index.rb index 15dfeee..67d31e6 100644 --- a/lib/gemirro/cli/index.rb +++ b/lib/gemirro/cli/index.rb @@ -22,8 +22,17 @@ indexer = Gemirro::Indexer.new(config.destination) indexer.ui = Gem::SilentUI.new - config.logger.info('Generating indexes') - indexer.generate_index if opts[:u].nil? - indexer.update_index unless opts[:u].nil? + if File.exist?(File.join(config.destination, "specs.#{Gem.marshal_version}.gz")) + if opts[:u] + config.logger.info('Generating index updates') + indexer.update_index + else + config.logger.info('Generating indexes') + indexer.generate_index + end + else + config.logger.error("/public/specs.#{Gem.marshal_version}.gz file is missing.") + config.logger.error('Run "gemirro update" before running index.') + end end end diff --git a/lib/gemirro/cli/init.rb b/lib/gemirro/cli/init.rb index d1d0f48..bc93264 100644 --- a/lib/gemirro/cli/init.rb +++ b/lib/gemirro/cli/init.rb @@ -26,6 +26,11 @@ end end + # make sure index updates blank local specs + ['specs.4.8', 'latest_specs.4.8', 'prerelease_specs.4.8'].each do |s| + File.utime(Time.at(0), Time.at(0), File.join(directory, 'public', s)) + end + puts "Initialized empty mirror in #{directory}" end end diff --git a/lib/gemirro/cli/update.rb b/lib/gemirro/cli/update.rb index 2361237..d50b59d 100644 --- a/lib/gemirro/cli/update.rb +++ b/lib/gemirro/cli/update.rb @@ -17,5 +17,11 @@ gems = Gemirro::GemsFetcher.new(source, versions) gems.fetch + + source.gems.each do |gem| + gem.gemspec = true + end + + gems.fetch end end diff --git a/lib/gemirro/configuration.rb b/lib/gemirro/configuration.rb index 37280cc..a4ae1e4 100644 --- a/lib/gemirro/configuration.rb +++ b/lib/gemirro/configuration.rb @@ -12,7 +12,7 @@ def self.configuration error_log: '/tmp/gemirro.access.log', daemonize: true }, - + update_thread_count: begin; Etc.nprocessors - 1; rescue StandardError; 4; end, update_on_fetch: true, fetch_gem: true } diff --git a/lib/gemirro/gems_fetcher.rb b/lib/gemirro/gems_fetcher.rb index cd6557e..7ecfc55 100644 --- a/lib/gemirro/gems_fetcher.rb +++ b/lib/gemirro/gems_fetcher.rb @@ -31,10 +31,10 @@ def fetch gem.platform = versions[1] if versions version = versions[0] if versions if gem.gemspec? - gemfile = fetch_gemspec(gem, version) - if gemfile + gemspec = fetch_gemspec(gem, version) + if gemspec Utils.configuration.mirror_gemspecs_directory - .add_file(gem.gemspec_filename(version), gemfile) + .add_file(gem.gemspec_filename(version), gemspec) end else gemfile = fetch_gem(gem, version) @@ -100,15 +100,10 @@ def fetch_gemspec(gem, version) # def fetch_gem(gem, version) filename = gem.filename(version) - satisfied = if gem.only_latest? - true - else - gem.requirement.satisfied_by?(version) - end + satisfied = gem.only_latest? || gem.requirement.satisfied_by?(version) name = gem.name - if gem_exists?(filename) || ignore_gem?(name, version, gem.platform) || - !satisfied + if gem_exists?(filename) || ignore_gem?(name, version, gem.platform) || !satisfied Utils.logger.debug("Skipping #{filename}") return end diff --git a/lib/gemirro/indexer.rb b/lib/gemirro/indexer.rb index cef22cd..bcff5b4 100644 --- a/lib/gemirro/indexer.rb +++ b/lib/gemirro/indexer.rb @@ -19,12 +19,14 @@ module Gemirro # @return [Array] # class Indexer < ::Gem::Indexer - attr_accessor(:files, - :quick_marshal_dir, - :directory, - :dest_directory, - :only_origin, - :updated_gems) + attr_accessor( + :files, + :quick_marshal_dir, + :directory, + :dest_directory, + :only_origin, + :updated_gems + ) ## # Create an indexer that will index the gems in +directory+. @@ -37,34 +39,38 @@ def initialize(directory, options = {}) require 'fileutils' require 'tmpdir' require 'zlib' + require 'builder/xchar' + require 'compact_index' - unless defined?(Builder::XChar) - raise 'Gem::Indexer requires that the XML Builder ' \ - 'library be installed:' \ - "\n\tgem install builder" - end - - options = { build_modern: true }.merge options + options.merge!({ build_modern: true, build_compact: true }) @build_modern = options[:build_modern] + @build_compact = options[:build_compact] @dest_directory = directory - @directory = File.join(Dir.tmpdir, - "gem_generate_index_#{rand(1_000_000_000)}") + @directory = + File.join(Dir.tmpdir, "gem_generate_index_#{rand(1_000_000_000)}") marshal_name = "Marshal.#{::Gem.marshal_version}" - @master_index = File.join @directory, 'yaml' - @marshal_index = File.join @directory, marshal_name + @master_index = + File.join(@directory, 'yaml') + @marshal_index = + File.join(@directory, marshal_name) - @quick_dir = File.join @directory, 'quick' - @quick_marshal_dir = File.join @quick_dir, marshal_name - @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH + @quick_dir = File.join(@directory, 'quick') + @quick_marshal_dir = + File.join(@quick_dir, marshal_name) + @quick_marshal_dir_base = + File.join(@dest_directory, 'quick', marshal_name) # FIX: UGH - @quick_index = File.join @quick_dir, 'index' - @latest_index = File.join @quick_dir, 'latest_index' + @quick_index = + File.join(@quick_dir, 'index') + @latest_index = + File.join(@quick_dir, 'latest_index') - @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}" + @specs_index = + File.join(@directory, "specs.#{::Gem.marshal_version}") @latest_specs_index = File.join(@directory, "latest_specs.#{::Gem.marshal_version}") @prerelease_specs_index = @@ -75,6 +81,10 @@ def initialize(directory, options = {}) File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}") @dest_prerelease_specs_index = File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}") + @infos_dir = + File.join(@dest_directory, 'info') + @api_v1_dependencies_dir = + File.join(@dest_directory, 'api', 'v1', 'dependencies') @files = [] end @@ -100,28 +110,24 @@ def install_indicies if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir) files.delete @quick_marshal_dir - dst_name = File.join(@dest_directory, @quick_marshal_dir_base) - FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose) - FileUtils.rm_rf(dst_name, verbose: verbose) - FileUtils.mv(@quick_marshal_dir, dst_name, - verbose: verbose, force: true) + FileUtils.mkdir_p(File.dirname(@quick_marshal_dir_base), verbose: verbose) + FileUtils.rm_rf(@quick_marshal_dir_base, verbose: verbose) + FileUtils.mv(@quick_marshal_dir, @quick_marshal_dir_base, verbose: verbose, force: true) end files.each do |path| file = path.sub(%r{^#{Regexp.escape @directory}/?}, '') - src_name = File.join(@directory, file) - dst_name = File.join(@dest_directory, file) if ["#{@specs_index}.gz", "#{@latest_specs_index}.gz", "#{@prerelease_specs_index}.gz"].include?(path) - res = build_zlib_file(file, src_name, dst_name, true) + res = build_zlib_file(file, File.join(@directory, file), File.join(@dest_directory, file), true) next unless res else source_content = download_from_source(file) next if source_content.nil? - MirrorFile.new(dst_name).write(source_content) + MirrorFile.new(File.join(@dest_directory, file)).write(source_content) end FileUtils.rm_rf(path) @@ -129,7 +135,7 @@ def install_indicies end ## - # Download file from source + # Download file from source (example: rubygems.org) # # @param [String] file File path # @return [String] @@ -164,14 +170,250 @@ def build_indicies ::Gem::Specification.all = specs if ::Gem::VERSION >= '2.5.0' - build_marshal_gemspecs specs - build_modern_indices specs if @build_modern + build_marshal_gemspecs(specs) + build_modern_indices(specs) if @build_modern compress_indices else build_marshal_gemspecs build_modern_indicies if @build_modern compress_indicies end + + build_api_v1_dependencies(specs) + + return unless @build_compact + + build_compact_index_names + build_compact_index_infos(specs) + build_compact_index_versions(specs) + end + + ## + # Cache Modern Index endpoints /api/v1/dependencies?gems= and /api/v1/dependencies.json?gems= + # This single request may include many fragments. server.rb determines which are required per request. + # + # @return nil + # + def build_api_v1_dependencies(specs, partial = false) + FileUtils.mkdir_p(@api_v1_dependencies_dir) + + if partial + specs.collect(&:name).uniq do |name| + FileUtils.rm_rf(Dir.glob(File.join(@api_v1_dependencies_dir, "#{name}.*.*.list"))) + end + else + FileUtils.rm_rf(Dir.glob(File.join(@api_v1_dependencies_dir, '*.list'))) + end + + grouped_specs = specs.sort_by(&:name).group_by(&:name) + grouped_specs.each_with_index do |(name, gem_versions), index| + Utils.logger.info("[#{index + 1}/#{grouped_specs.size}]: Caching /api/v1/dependencies/#{name}") + + gem_versions = + gem_versions.sort do |a, b| + a.version <=> b.version + end + + cg = [] + Parallel.each_with_index( + gem_versions, + in_threads: Utils.configuration.update_thread_count + ) do |spec, index2| + next if spec.nil? + + dependencies = spec.dependencies.select do |d| + d.type == :runtime + end + + dependencies = dependencies.collect do |d| + [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s] + end + + cg[index2] = + { + name: spec.name, + number: spec.version.to_s, + platform: spec.platform, + dependencies: dependencies + } + end + + Tempfile.create("api_v1_dependencies_#{name}.list") do |f| + f.write Marshal.dump(cg) + f.rewind + + File.rename( + f.path, + File.join( + @api_v1_dependencies_dir, + "#{name}.#{Digest::MD5.file(f.path).hexdigest}.#{Digest::SHA256.file(f.path).hexdigest}.list" + ) + ) + end + end + end + + ## + # Cache compact_index endpoint /names + # Report all gems with versions available. Does not require opening spec files. + # + # @return nil + # + def build_compact_index_names + Utils.logger.info('[1/1]: Caching /names') + FileUtils.rm_rf(Dir.glob(File.join(@dest_directory, 'names*.list'))) + + gem_name_list = Dir.glob('*.gem', base: File.join(@dest_directory, 'gems')).collect do |x| + x.sub(/-\d+(\.\d+)*(\.[a-zA-Z\d]+)*([-_a-zA-Z\d]+)?\.gem/, '') + end.uniq.sort! + + Tempfile.create('names.list') do |f| + f.write CompactIndex.names(gem_name_list).to_s + f.rewind + File.rename( + f.path, + File.join(@dest_directory, + "names.#{Digest::MD5.file(f.path).hexdigest}.#{Digest::SHA256.file(f.path).hexdigest}.list") + ) + end + + nil + end + + ## + # Cache compact_index endpoint /versions + # + # @param [Array] specs Gems list + # @param [Boolean] partial Is gem list an update or a full index + # @return nil + # + def build_compact_index_versions(specs, partial = false) + Utils.logger.info('[1/1]: Caching /versions') + + cg = + specs + .sort_by(&:name) + .group_by(&:name) + .collect do |name, gem_versions| + gem_versions = + gem_versions.sort do |a, b| + a.version <=> b.version + end + + info_file = Dir.glob(File.join(@infos_dir, "#{name}.*.*.list")).last + + throw "Info file for #{name} not found" unless info_file + + info_file_checksum = info_file.split('.', -4)[-3] + + CompactIndex::Gem.new( + name, + gem_versions.collect do |y| + CompactIndex::GemVersion.new( + y.version.to_s, + y.platform, + nil, + info_file_checksum + ) + end + ) + end + + Tempfile.create('versions.list') do |f| + previous_versions_file = Dir.glob(File.join(@dest_directory, 'versions*.list')).last + + if partial && previous_versions_file + versions_file = CompactIndex::VersionsFile.new(previous_versions_file) + else + versions_file = CompactIndex::VersionsFile.new(f.path) + f.write format('created_at: %s', Time.now.utc.iso8601) + f.write "\n---\n" + end + + f.write CompactIndex.versions(versions_file, cg) + f.rewind + + FileUtils.rm_rf(Dir.glob(File.join(@dest_directory, 'versions*.list'))) + + File.rename( + f.path, + File.join( + @dest_directory, + "versions.#{Digest::MD5.file(f.path).hexdigest}.#{Digest::SHA256.file(f.path).hexdigest}.list" + ) + ) + end + + nil + end + + ## + # Cache compact_index endpoint /info/[gemname] + # + # @param [Array] specs Gems list + # @param [Boolean] partial Is gem list an update or a full index + # @return nil + # + def build_compact_index_infos(specs, partial = false) + FileUtils.mkdir_p(@infos_dir) + + if partial + specs.collect(&:name).uniq do |name| + FileUtils.rm_rf(Dir.glob(File.join(@infos_dir, "#{name}.*.*.list"))) + end + else + FileUtils.rm_rf(Dir.glob(File.join(@infos_dir, '*.list'))) + end + + grouped_specs = specs.sort_by(&:name).group_by(&:name) + grouped_specs.each_with_index do |(name, gem_versions), index| + Utils.logger.info("[#{index + 1}/#{grouped_specs.size}]: Caching /info/#{name}") + + gem_versions = + gem_versions.sort do |a, b| + a.version <=> b.version + end + + versions = + Parallel.map(gem_versions, in_threads: Utils.configuration.update_thread_count) do |spec| + deps = + spec + .dependencies + .select { |d| d.type == :runtime } + .sort_by(&:name) + .collect do |dependency| + CompactIndex::Dependency.new( + dependency.name, + dependency.requirement.to_s + ) + end + + CompactIndex::GemVersion.new( + spec.version, + spec.platform, + Digest::SHA256.file(spec.loaded_from).hexdigest, + nil, + deps, + spec.required_ruby_version.to_s, + spec.required_rubygems_version.to_s + ) + end + + Tempfile.create("info_#{name}.list") do |f| + f.write CompactIndex.info(versions).to_s + f.rewind + + File.rename( + f.path, + File.join( + @infos_dir, + "#{name}.#{Digest::MD5.file(f.path).hexdigest}.#{Digest::SHA256.file(f.path).hexdigest}.list" + ) + ) + end + end + + nil end ## @@ -181,7 +423,9 @@ def build_indicies # @return [Array] # def map_gems_to_specs(gems) - gems.map.with_index do |gemfile, index| + results = [] + + Parallel.each_with_index(gems, in_threads: Utils.configuration.update_thread_count) do |gemfile, index| Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}") if File.empty?(gemfile) Utils.logger.warn("Skipping zero-length gem: #{gemfile}") @@ -190,12 +434,12 @@ def map_gems_to_specs(gems) begin begin - spec = if ::Gem::Package.respond_to? :open - ::Gem::Package - .open(File.open(gemfile, 'rb'), 'r', &:metadata) - else - ::Gem::Package.new(gemfile).spec - end + spec = + if ::Gem::Package.respond_to? :open + ::Gem::Package.open(File.open(gemfile, 'rb'), 'r', &:metadata) + else + ::Gem::Package.new(gemfile).spec + end rescue NotImplementedError next end @@ -213,7 +457,7 @@ def map_gems_to_specs(gems) end version = spec.version.version - unless version =~ /^\d+\.\d+\.\d+.*/ + unless version =~ /^\d+(\.\d+)?(\.\d+)?.*/ msg = "Skipping gem #{spec.full_name} - invalid version #{version}" Utils.logger.warn(msg) next @@ -223,8 +467,8 @@ def map_gems_to_specs(gems) spec.abbreviate spec.sanitize else - abbreviate spec - sanitize spec + abbreviate(spec) + sanitize(spec) end spec @@ -238,20 +482,48 @@ def map_gems_to_specs(gems) "\t#{e.backtrace.join "\n\t"}"].join("\n") Utils.logger.debug(msg) end - end.compact + + results[index] = spec + end + + # nils can result from insert by index + results.compact end + ## + # Handle `index --update`, detecting changed files and file lists. + # + # @return nil + # def update_index make_temp_directories - specs_mtime = File.stat(@dest_specs_index).mtime + present_gemfiles = Dir.glob('*.gem', base: File.join(@dest_directory, 'gems')) + indexed_gemfiles = Dir.glob('*.gemspec.rz', base: @quick_marshal_dir_base).collect { |x| x.gsub(/spec.rz$/, '') } + + @updated_gems = [] + # detect files manually added to public/gems + @updated_gems += (present_gemfiles - indexed_gemfiles).collect { |x| File.join(@dest_directory, 'gems', x) } + # detect files manually deleted from public/gems + @updated_gems += (indexed_gemfiles - present_gemfiles).collect { |x| File.join(@dest_directory, 'gems', x) } + + specs_mtime = + begin + File.stat(@dest_specs_index).mtime + rescue StandardError + Time.at(0) + end newest_mtime = Time.at(0) - @updated_gems = gem_file_list.select do |gem| - gem_mtime = File.stat(gem).mtime - newest_mtime = gem_mtime if gem_mtime > newest_mtime - gem_mtime > specs_mtime - end + # files that have been replaced + @updated_gems += + gem_file_list.select do |gem| + gem_mtime = File.stat(gem).mtime + newest_mtime = gem_mtime if gem_mtime > newest_mtime + gem_mtime > specs_mtime + end + + @updated_gems.uniq! if @updated_gems.empty? Utils.logger.info('No new gems') @@ -259,24 +531,46 @@ def update_index end specs = map_gems_to_specs(@updated_gems) + + # specs only includes latest discovered files. + # /info/[gemname] and /api/v1/dependencies can not be rebuilt + # incrementally, so retrive specs for all versions of these gems. + gem_name_updates = specs.collect(&:name).uniq + u2 = + Dir.glob(File.join(File.join(@dest_directory, 'gems'), '*.gem')).select do |possibility| + gem_name_updates.any? { |updated| File.basename(possibility) =~ /^#{updated}-\d/ } + end + + Utils.logger.info('Reloading for /info/[gemname]') + version_specs = map_gems_to_specs(u2) + prerelease, released = specs.partition { |s| s.version.prerelease? } ::Gem::Specification.dirs = [] ::Gem::Specification.all = *specs - files = if ::Gem::VERSION >= '2.5.0' - build_marshal_gemspecs specs - else - build_marshal_gemspecs - end + files = + if ::Gem::VERSION >= '2.5.0' + build_marshal_gemspecs specs + else + build_marshal_gemspecs + end ::Gem.time('Updated indexes') do - update_specs_index(released, @dest_specs_index, @specs_index) - update_specs_index(released, - @dest_latest_specs_index, - @latest_specs_index) - update_specs_index(prerelease, - @dest_prerelease_specs_index, - @prerelease_specs_index) + update_specs_index( + released, + @dest_specs_index, + @specs_index + ) + update_specs_index( + released, + @dest_latest_specs_index, + @latest_specs_index + ) + update_specs_index( + prerelease, + @dest_prerelease_specs_index, + @prerelease_specs_index + ) end if ::Gem::VERSION >= '2.5.0' @@ -285,6 +579,14 @@ def update_index compress_indicies end + build_api_v1_dependencies(version_specs, true) + + if @build_compact + build_compact_index_names + build_compact_index_infos(version_specs, true) + build_compact_index_versions(specs, true) + end + Utils.logger.info("Updating production dir #{@dest_directory}") if verbose files << @specs_index files << "#{@specs_index}.gz" @@ -295,36 +597,36 @@ def update_index files.each do |path| file = path.sub(%r{^#{Regexp.escape @directory}/?}, '') - src_name = File.join(@directory, file) - dst_name = File.join(@dest_directory, file) - if ["#{@specs_index}.gz", - "#{@latest_specs_index}.gz", - "#{@prerelease_specs_index}.gz"].include?(path) - res = build_zlib_file(file, src_name, dst_name) + if ["#{@specs_index}.gz", "#{@latest_specs_index}.gz", "#{@prerelease_specs_index}.gz"].include?(path) + res = build_zlib_file(file, File.join(@directory, file), File.join(@dest_directory, file)) next unless res else - FileUtils.mv(src_name, - dst_name, - verbose: verbose, - force: true) + FileUtils.mv( + File.join(@directory, file), + File.join(@dest_directory, file), + verbose: verbose, + force: true + ) end - File.utime(newest_mtime, newest_mtime, dst_name) + File.utime(newest_mtime, newest_mtime, File.join(@dest_directory, file)) end end def build_zlib_file(file, src_name, dst_name, from_source = false) content = Marshal.load(Zlib::GzipReader.open(src_name).read) - create_zlib_file("#{dst_name}.orig", content) + create_zlib_file("#{dst_name}.local", content) return false if @only_origin if from_source source_content = download_from_source(file) - source_content = Marshal.load(Zlib::GzipReader - .new(StringIO - .new(source_content)).read) + source_content = Marshal.load( + Zlib::GzipReader.new( + StringIO.new(source_content) + ).read + ) else source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read) end @@ -336,17 +638,18 @@ def build_zlib_file(file, src_name, dst_name, from_source = false) end def create_zlib_file(dst_name, content) - temp_file = Tempfile.new('gemirro') - - Zlib::GzipWriter.open(temp_file.path) do |io| - io.write(Marshal.dump(content)) + Tempfile.create(File.basename(dst_name)) do |f| + gzf = Zlib::GzipWriter.new(f) + gzf.write(Marshal.dump(content)) + gzf.close + + FileUtils.mv( + f.path, + dst_name, + verbose: verbose, + force: true + ) end - - FileUtils.mv(temp_file.path, - dst_name, - verbose: verbose, - force: true) - Utils.cache.flush_key(File.basename(dst_name)) end def verbose diff --git a/lib/gemirro/mirror_file.rb b/lib/gemirro/mirror_file.rb index 90e8c05..a99afe8 100644 --- a/lib/gemirro/mirror_file.rb +++ b/lib/gemirro/mirror_file.rb @@ -26,6 +26,7 @@ def initialize(path) # @param [String] content # def write(content) + FileUtils.mkdir_p(File.dirname(@path)) handle = File.open(@path, 'w') handle.write(content) diff --git a/lib/gemirro/server.rb b/lib/gemirro/server.rb index c4fde56..7a54490 100644 --- a/lib/gemirro/server.rb +++ b/lib/gemirro/server.rb @@ -10,12 +10,6 @@ module Gemirro # Launch Sinatra server to easily download gems. # class Server < Sinatra::Base - # rubocop:disable Layout/LineLength - URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/.freeze - # rubocop:enable Layout/LineLength - GEMSPEC_TYPE = 'gemspec.rz' - GEM_TYPE = 'gem' - access_logger = Logger.new(Utils.configuration.server.access_log).tap do |logger| ::Logger.class_eval { alias_method :write, :<< } logger.level = ::Logger::INFO @@ -62,7 +56,7 @@ class Server < Sinatra::Base end ## - # Display information about one gem + # Display information about one gem, human readable # # @return [nil] # @@ -76,7 +70,7 @@ class Server < Sinatra::Base ## # Display home page containing the list of gems already - # downloaded on the server + # downloaded on the server, human readable # # @return [nil] # @@ -85,13 +79,17 @@ class Server < Sinatra::Base end ## - # Return gem dependencies as binary + # Return gem dependencies as marshaled binary # # @return [nil] # get '/api/v1/dependencies' do content_type 'application/octet-stream' - query_gems.any? ? Marshal.dump(query_gems_list) : 200 + if params[:gems].to_s != '' && params[:gems].to_s.split(',').any? + Marshal.dump(dependencies_loader(params[:gems].to_s.split(','))) + else + 200 + end end ## @@ -101,196 +99,105 @@ class Server < Sinatra::Base # get '/api/v1/dependencies.json' do content_type 'application/json' - query_gems.any? ? JSON.dump(query_gems_list) : {} + + return '[]' unless params[:gems] + + gem_names = params[:gems].to_s + .split(',') + .map(&:strip) + .reject(&:empty?) + return '[]' if gem_names.empty? + + JSON.dump(dependencies_loader(gem_names)) end ## - # Try to get all request and download files - # if files aren't found. + # compact_index, Return list of available gem names # # @return [nil] # - get('*') do |path| - resource = "#{settings.public_folder}#{path}" + get '/names' do + content_type 'text/plain' - # Try to download gem - fetch_gem(resource) unless File.exist?(resource) - # If not found again, return a 404 - return not_found unless File.exist?(resource) + content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last + _, etag, repr_digest, _ = content_path.split('.', -4) - send_file(resource) + headers 'etag' => etag + headers 'repr-digest' => %(sha-256="#{repr_digest}") + send_file content_path end ## - # Try to fetch gem and download its if it's possible, and - # build and install indicies. + # compact_index, Return list of gem, including versions # - # @param [String] resource - # @return [Indexer] + # @return [nil] # - def fetch_gem(resource) - return unless Utils.configuration.fetch_gem - - name = File.basename(resource) - result = name.match(URI_REGEXP) - return unless result - - gem_name, gem_version, gem_platform, gem_type = result.captures - return unless gem_name && gem_version + get '/versions' do + content_type 'text/plain' - begin - gem = Utils.stored_gem(gem_name, gem_version, gem_platform) - gem.gemspec = true if gem_type == GEMSPEC_TYPE + content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last + _, etag, repr_digest, _ = content_path.split('.', -4) - return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE - return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE - - Utils.logger - .info("Try to download #{gem_name} with version #{gem_version}") - Utils.gems_fetcher.source.gems.clear - Utils.gems_fetcher.source.gems.push(gem) - Utils.gems_fetcher.fetch - - update_indexes if Utils.configuration.update_on_fetch - rescue StandardError => e - Utils.logger.error(e) - end + headers 'etag' => etag + headers 'repr-digest' => %(sha-256="#{repr_digest}") + send_file content_path end - ## - # Update indexes files + # compact_index, Return gem dependencies for all versions of a gem # - # @return [Indexer] + # @return [nil] # - def update_indexes - indexer = Gemirro::Indexer.new(Utils.configuration.destination) - indexer.only_origin = true - indexer.ui = ::Gem::SilentUI.new + get('/info/:gemname') do + gems = Utils.gems_collection + gem = gems.find_by_name(params[:gemname]) + return not_found if gem.nil? - Utils.logger.info('Generating indexes') - indexer.update_index - indexer.updated_gems.each do |gem| - Utils.cache.flush_key(File.basename(gem)) - end - rescue SystemExit => e - Utils.logger.info(e.message) - end + content_type 'text/plain' - ## - # Return all gems pass to query - # - # @return [Array] - # - def query_gems - params[:gems].to_s.split(',') + content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last + _, etag, repr_digest, _ = content_path.split('.', -4) + + headers 'etag' => etag + headers 'repr-digest' => %(sha-256="#{repr_digest}") + send_file content_path end ## - # Return gems list from query params + # Try to get all request and download files + # if files aren't found. # - # @return [Array] + # @return [nil] # - def query_gems_list - Utils.gems_collection(false) # load collection - gems = Parallel.map(query_gems, in_threads: 4) do |query_gem| - gem_dependencies(query_gem) - end + get('*') do |path| + resource = "#{settings.public_folder}#{path}" - gems.flatten! - gems.reject!(&:empty?) - gems + # Try to download gem + Gemirro::Utils.fetch_gem(resource) unless File.exist?(resource) + # If not found again, return a 404 + return not_found unless File.exist?(resource) + + send_file(resource) end ## - # List of versions and dependencies of each version - # from a gem name. + # Compile fragments for /api/v1/dependencies # - # @return [Array] + # @return [nil] # - def gem_dependencies(gem_name) - Utils.cache.cache(gem_name) do - gems = Utils.gems_collection(false) - gem_collection = gems.find_by_name(gem_name) - - return '' if gem_collection.nil? - - gem_collection = Parallel.map(gem_collection, in_threads: 4) do |gem| - [gem, spec_for(gem.name, gem.number, gem.platform)] - end - gem_collection.compact! - - Parallel.map(gem_collection, in_threads: 4) do |gem, spec| - next if spec.nil? - - dependencies = spec.dependencies.select do |d| - d.type == :runtime - end - - dependencies = Parallel.map(dependencies, in_threads: 4) do |d| - [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s] - end - - { - name: gem.name, - number: gem.number, - platform: gem.platform, - dependencies: dependencies - } - end - end - end - - helpers do - ## - # Return gem specification from gemname and version - # - # @param [String] gemname - # @param [String] version - # @return [::Gem::Specification] - # - def spec_for(gemname, version, platform = 'ruby') - gem = Utils.stored_gem(gemname, version.to_s, platform) - gemspec_path = File.join('quick', - Gemirro::Configuration.marshal_identifier, - gem.gemspec_filename) - spec_file = File.join(settings.public_folder, - gemspec_path) - fetch_gem(gemspec_path) unless File.exist?(spec_file) - - return unless File.exist?(spec_file) - - File.open(spec_file, 'r') do |uz_file| - uz_file.binmode - inflater = Zlib::Inflate.new - begin - inflate_data = inflater.inflate(uz_file.read) - ensure - inflater.finish - inflater.close - end - Marshal.load(inflate_data) + def dependencies_loader(names) + names.collect do |name| + f = File.join(settings.public_folder, 'api', 'v1', 'dependencies', "#{name}.*.*.list") + Marshal.load(File.read(Dir.glob(f).last)) + rescue StandardError => e + env['rack.errors'].write "Cound not open #{f}\n" + env['rack.errors'].write "#{e.message}\n" + e.backtrace.each do |err| + env['rack.errors'].write "#{err}\n" end + nil end - - ## - # Escape string - # - # @param [String] string - # @return [String] - # - def escape(string) - Rack::Utils.escape_html(string) - end - - ## - # Homepage link - # - # @param [Gem] spec - # @return [String] - # - def homepage(spec) - URI.parse(Addressable::URI.escape(spec.homepage)) - end + .flatten + .compact end end end diff --git a/lib/gemirro/utils.rb b/lib/gemirro/utils.rb index 51c75bd..2a7ea35 100644 --- a/lib/gemirro/utils.rb +++ b/lib/gemirro/utils.rb @@ -13,21 +13,16 @@ module Gemirro # @return [Gemirro::GemsFetcher] # class Utils - attr_reader(:cache, - :versions_fetcher, - :gems_fetcher, - :gems_collection, - :stored_gems) + attr_reader( + :versions_fetcher, + :gems_fetcher, + :gems_collection, + :stored_gems + ) - ## - # Cache class to store marshal and data into files - # - # @return [Gemirro::Cache] - # - def self.cache - @cache ||= Gemirro::Cache - .new(File.join(configuration.destination, '.cache')) - end + URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/ + GEMSPEC_TYPE = 'gemspec.rz' + GEM_TYPE = 'gem' ## # Generate Gems collection from Marshal dump @@ -35,62 +30,44 @@ def self.cache # @param [TrueClass|FalseClass] orig Fetch orig files # @return [Gemirro::GemVersionCollection] # - def self.gems_collection(orig = true) - @gems_collection = {} if @gems_collection.nil? - - is_orig = orig ? 1 : 0 - data = @gems_collection[is_orig] - data = { files: {}, values: nil } if data.nil? - - file_paths = specs_files_paths(orig) - has_file_changed = false - Parallel.map(file_paths, in_threads: 4) do |file_path| - next if data[:files].key?(file_path) && - data[:files][file_path] == File.mtime(file_path) - - has_file_changed = true - end + def self.gems_collection(local = true) + @gems_collection ||= {} + @gems_collection[local ? :local : :remote] ||= { files: {}, values: nil } + + file_paths = + %i[specs prerelease_specs].collect do |specs_file_type| + File.join( + configuration.destination, + if local + "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz.local" + else + "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz" + end + ) + end + + has_file_changed = + !file_paths.all? do |file_path| + @gems_collection[local ? :local : :remote][:files].key?(file_path) && + @gems_collection[local ? :local : :remote][:files][file_path] == File.mtime(file_path) + end # Return result if no file changed - return data[:values] if !has_file_changed && !data[:values].nil? + if !has_file_changed && !@gems_collection[local ? :local : :remote][:values].nil? + return @gems_collection[local ? :local : :remote][:values] + end gems = [] - Parallel.map(file_paths, in_threads: 4) do |file_path| + + # parallel is not for mtime, it's for the Marshal. + Parallel.map(file_paths, in_threads: Utils.configuration.update_thread_count) do |file_path| next unless File.exist?(file_path) gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read)) - data[:files][file_path] = File.mtime(file_path) + @gems_collection[local ? :local : :remote][:files][file_path] = File.mtime(file_path) end - collection = GemVersionCollection.new(gems) - data[:values] = collection - - collection - end - - ## - # Return specs fils paths - # - # @param [TrueClass|FalseClass] orig Fetch orig files - # @return [Array] - # - def self.specs_files_paths(orig = true) - marshal_version = Gemirro::Configuration.marshal_version - Parallel.map(specs_file_types, in_threads: 4) do |specs_file_type| - File.join(configuration.destination, - [specs_file_type, - marshal_version, - "gz#{orig ? '.orig' : ''}"].join('.')) - end - end - - ## - # Return specs fils types - # - # @return [Array] - # - def self.specs_file_types - %i[specs prerelease_specs] + @gems_collection[local ? :local : :remote][:values] = GemVersionCollection.new(gems) end ## @@ -112,8 +89,7 @@ def self.configuration # @see Gemirro::VersionsFetcher.fetch # def self.versions_fetcher - @versions_fetcher ||= Gemirro::VersionsFetcher - .new(configuration.source).fetch + @versions_fetcher ||= Gemirro::VersionsFetcher.new(configuration.source).fetch end ## @@ -133,13 +109,115 @@ def self.gems_fetcher def self.stored_gem(gem_name, gem_version, platform = 'ruby') platform = 'ruby' if platform.nil? @stored_gems ||= {} - # rubocop:disable Metrics/LineLength @stored_gems[gem_name] = {} unless @stored_gems.key?(gem_name) @stored_gems[gem_name][gem_version] = {} unless @stored_gems[gem_name].key?(gem_version) - @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform) unless @stored_gems[gem_name][gem_version].key?(platform) - # rubocop:enable Metrics/LineLength + unless @stored_gems[gem_name][gem_version].key?(platform) + @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform) + end @stored_gems[gem_name][gem_version][platform] end + + ## + # Return gem specification from gemname and version + # + # @param [String] gemname + # @param [String] version + # @return [::Gem::Specification] + # + def self.spec_for(gemname, version, platform = 'ruby') + gem = Utils.stored_gem(gemname, version.to_s, platform) + + spec_file = + File.join( + 'quick', + Gemirro::Configuration.marshal_identifier, + gem.gemspec_filename + ) + + fetch_gem(spec_file) unless File.exist?(spec_file) + + return unless File.exist?(spec_file) + + File.open(spec_file, 'r') do |uz_file| + uz_file.binmode + inflater = Zlib::Inflate.new + begin + inflate_data = inflater.inflate(uz_file.read) + ensure + inflater.finish + inflater.close + end + Marshal.load(inflate_data) + end + end + + ## + # Try to fetch gem and download its if it's possible, and + # build and install indicies. + # + # @param [String] resource + # @return [Indexer] + # + def self.fetch_gem(resource) + return unless Utils.configuration.fetch_gem + + name = File.basename(resource) + result = name.match(URI_REGEXP) + return unless result + + gem_name, gem_version, gem_platform, gem_type = result.captures + return unless gem_name && gem_version + + begin + gem = Utils.stored_gem(gem_name, gem_version, gem_platform) + gem.gemspec = true if gem_type == GEMSPEC_TYPE + + return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE + return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE + + Utils.logger.info("Try to download #{gem_name} with version #{gem_version}") + Utils.gems_fetcher.source.gems.clear + Utils.gems_fetcher.source.gems.push(gem) + Utils.gems_fetcher.fetch + + update_indexes if Utils.configuration.update_on_fetch + rescue StandardError => e + Utils.logger.error(e) + end + end + + ## + # Return gems list from query params + # + # @return [Array] + # + def self.query_gems_list(query_gems) + Utils.gems_collection(false) # load collection + gems = Parallel.map(query_gems, in_threads: Utils.configuration.update_thread_count) do |query_gem| + gem_dependencies(query_gem) + end + + gems.flatten.compact.reject(&:empty?) + end + + ## + # Update indexes files + # + # @return [Indexer] + # + def self.update_indexes + indexer = Gemirro::Indexer.new(Utils.configuration.destination) + indexer.only_origin = true + indexer.ui = ::Gem::SilentUI.new + + Utils.logger.info('Generating indexes') + indexer.update_index + # indexer.updated_gems.each do |gem| + # Utils.cache.flush_key(File.basename(gem)) + # end + rescue SystemExit => e + Utils.logger.info(e.message) + end end end diff --git a/lib/gemirro/version.rb b/lib/gemirro/version.rb index c7be402..baed64e 100644 --- a/lib/gemirro/version.rb +++ b/lib/gemirro/version.rb @@ -2,5 +2,5 @@ # Gemirro Version module Gemirro - VERSION = '1.5.0' + VERSION = '1.6.0' end diff --git a/lib/gemirro/versions_fetcher.rb b/lib/gemirro/versions_fetcher.rb index ab0f908..89bd9cb 100644 --- a/lib/gemirro/versions_fetcher.rb +++ b/lib/gemirro/versions_fetcher.rb @@ -22,8 +22,10 @@ def initialize(source) # @return [Gemirro::VersionsFile] # def fetch - VersionsFile.load(read_file(Configuration.versions_file), - read_file(Configuration.prerelease_versions_file, true)) + VersionsFile.load( + read_file(Configuration.versions_file), + read_file(Configuration.prerelease_versions_file, true) + ) end ## @@ -36,6 +38,8 @@ def read_file(file, prerelease = false) destination = Gemirro.configuration.destination file_dst = File.join(destination, file) unless File.exist?(file_dst) + throw 'No source defined' unless @source + File.write(file_dst, @source.fetch_versions) unless prerelease File.write(file_dst, @source.fetch_prerelease_versions) if prerelease end diff --git a/spec/gemirro/cache_spec.rb b/spec/gemirro/cache_spec.rb deleted file mode 100644 index 096c23e..0000000 --- a/spec/gemirro/cache_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -require 'spec_helper' -require 'gemirro/mirror_directory' -require 'gemirro/cache' - -# Gem tests -module Gemirro - describe 'Cache' do - include FakeFS::SpecHelpers - before(:each) do - MirrorDirectory.new('/tmp') - @cache = Cache.new('/tmp') - end - - it 'should play with flush key' do - @cache.cache('foo') do - 'something' - end - expect(@cache.cache('foo')).to eq('something') - @cache.flush_key('foo') - expect(@cache.cache('foo')).to be_nil - end - - it 'should play with flush' do - @cache.cache('foo') do - 'something' - end - expect(@cache.cache('foo')).to eq('something') - @cache.flush - expect(@cache.cache('foo')).to be_nil - end - end -end diff --git a/spec/gemirro/server_spec.rb b/spec/gemirro/server_spec.rb index 018569f..1360c24 100644 --- a/spec/gemirro/server_spec.rb +++ b/spec/gemirro/server_spec.rb @@ -1,7 +1,6 @@ require 'rack/test' require 'json' require 'parallel' -require 'gemirro/cache' require 'gemirro/utils' require 'gemirro/mirror_directory' require 'gemirro/mirror_file' @@ -37,11 +36,17 @@ module Gemirro Utils.instance_eval('@gems_orig_collection = nil') Utils.instance_eval('@gems_source_collection = nil') FakeFS::FileSystem.clone(Gemirro::Configuration.views_directory) + allow_any_instance_of(Indexer).to receive(:compress_indices) + allow_any_instance_of(Indexer).to receive(:compress_indicies) + allow_any_instance_of(Indexer).to receive(:rand).and_return('0') + + source = Source.new('Rubygems', 'https://rubygems.org') + allow(Gemirro.configuration).to receive(:source).and_return(source) end context 'HTML render' do it 'should display index page' do - allow(Logger).to receive(:new).twice.and_return(@fake_logger) + allow(Logger).to receive(:new).exactly(3).times.and_return(@fake_logger) allow(@fake_logger).to receive(:tap) .and_return(nil) .and_yield(@fake_logger) @@ -67,54 +72,63 @@ module Gemirro ::Gem::Version.create('0.1.0'), 'ruby']]) - MirrorFile.new('/var/www/gemirro/specs.4.8.gz.orig').write(marshal_dump) - Struct.new('SuccessGzipReader', :read) - gzip_reader = Struct::SuccessGzipReader.new(marshal_dump) - MirrorDirectory.new('/var/www/gemirro') - .add_directory('quick/Marshal.4.8') - # rubocop:disable Metrics/LineLength - MirrorFile.new('/var/www/gemirro/quick/Marshal.4.8/' \ - 'volay-0.1.0.gemspec.rz') - .write("x\x9C\x8D\x94]\x8F\xD2@\x14\x86\x89Y\xBB\xB4|\xEC\x12\xD7h" \ - "\xD4h\xD3K\x13J\x01\x97\xC84n\x9A\xA8\xBBi\xE2\xC5\x06\xBB" \ - "{\xC3\x85)\xE5\x00\x13f:u:E\xD1\xC4\xDF\xE6\xB5\xBF\xCAiK" \ - "\x11\xE3GK\xEF\x98\xF7\xBC\xCFy\xCF\xC9\xCCQ=A\x0F\xAE\x80" \ - "\"\xF4>\x82\x00/p\xE0\v\xCC\xC2;\xC1\xDD\xA3\xFA\xF4\xA1k4" \ - "\x06\xA6e\xF6_(Hy\xEBa\xD55\xB4\r#\xFEV\xB1k\xDE\r\xEAdu" \ - "\xB7\xC0cY1U\xE4\xA1\x95\x8A\xD3C7A\xAA\x87)\xB4\x9C\x1FO" \ - "\xBE\xD7\xE4OA\xEA\x17\x16\x82k\xD4o\xBC\xD7\x99\xC2x\xEC" \ - "\xAD@\xBFe$\xA1\xA0\xC7\xDBX\x00\xD5\x05/\xBC\xEFg\xDE\x13" \ - "\xF8\x98`\x0E\x14B1U\xE4w\xEC\x1A\xC7\x17\xAF2\x85\xADd\xC4" \ - "\xBE96\x87\xF9\x1F\xEA\xDF%\x8A\x95\xE3T\x9E\xCC2\xF3i\x9B" \ - "\xA1\xB3\xCC\xFE\rD\x10\xCE!\f\xB6\x1A\xD2\x9C\xD0\xA7\xB2" \ - "\xBF\x13\x8A?\x13<\xEB\x06\x04\xA7b\xD4q\xF8\xAF&\x0E!\xDF" \ - ".~\xEF\xE3\xDC\xCC@\xD2Hl\#@M\x9E\x84BN\x00\x9D:\x11\a\x0E" \ - "\x04\xFC\x18.\xD1#g\x93\xCF\xEB\xC3\x81m\\\xC1\x97\xD9" \ - "\x9Af7\\\xE3l\xD7_\xBC\x02BX\"\xD23\xBB\xF9o\x83A\xB1\x12" \ - "\xBBe\xB7\xED\x93K\xFB\xB4\x82\xB6\x80\xA9K\xB1\x1E\x96" \ - "\x10\xEA\x03sP\xCD\xBFP\x16\xEE\x8D\x85\xBF\x86E\\\x96" \ - "\xC02G\xF9\b\xEC\x16:\x9D\xC3\x06\b\x8B\xD2\xA9\x95\x84" \ - "\xD9\x97\xED\xC3p\x89+\x81\xA9}\xAB`\xD9\x9D\xFF\x03\xF6" \ - "\xD2\xC2\xBF\xCD\xFD`\xDD\x15\x10\x97\xED\xA4.[\xAB\xC6(" \ - "\x94\x05B\xE3\xB1\xBC\xA5e\xF6\xC3\xAA\x11\n\xE5>A\x8CiD " \ - "`\x9B\xF2\x04\xE3\xCA\t\xC6\x87\by-f,`Q\xD9\x1E,sp^q\x0F" \ - "\x85\xD4r\x8Dg\x11\x06\xCE\xC1\xE4>\x9D\xF9\xC9\xFC\xE5" \ - "\xC8YR\x1F\x133`4\xBB\xF9R~\xEF:\x93\xE8\x93\\\x92\xBF\r" \ - "\xA3\t\xF8\x84l\xF5<\xBF\xBE\xF9\xE3Q\xD2?q,\x04\x84:\x0E" \ - "\xF5\xF4\x1D1\xF3\xBA\xE7+!\"\xD4\xEB-\xB1X%\xB3\x14\xD3" \ - "\xCB\xEDw\xEE\xBD\xFDk\xE99OSz\xF3\xEA\xFA]w7\xF5\xAF\xB5" \ - "\x9F+\xFEG\x96") - # rubocop:enable Metrics/LineLength + MirrorFile.new('/var/www/gemirro/specs.4.8.gz.local').write(Marshal.dump({})) + - allow(Zlib::GzipReader).to receive(:open) - .once - .with('/var/www/gemirro/specs.4.8.gz.orig') - .and_return(gzip_reader) + allow(Zlib::GzipReader).to receive(:open).and_return(double(read: marshal_dump)) get '/gem/volay' + + expect(last_response.status).to eq(200) + expect(last_response).to be_ok + end + + it 'responds to compact_index /names' do + MirrorFile.new('/var/www/gemirro/names.md5.sha256.list').write('---\n- volay\n') + + get '/names' + expect(last_response.status).to eq(200) + expect(last_response).to be_ok + expect(last_response.body).to eq('---\n- volay\n') + expect(last_response.headers['etag']).to eq("md5") + expect(last_response.headers['repr-digest']).to eq('sha-256="sha256"') + end + + it 'responds to compact_index /info/[gemname]' do + marshal_dump = Marshal.dump([['volay', + ::Gem::Version.create('0.1.0'), + 'ruby']]) + + MirrorFile.new('/var/www/gemirro/specs.4.8.gz.local').write(Marshal.dump({})) + + allow(Zlib::GzipReader).to receive(:open).and_return(double(read: marshal_dump)) + + + MirrorDirectory.new('/var/www/gemirro/info') + MirrorFile.new('/var/www/gemirro/info/volay.md5.sha256.list').write('---\n 0.1.0 |checksum:sha256\n') + + + get '/info/volay' expect(last_response.status).to eq(200) expect(last_response).to be_ok + expect(last_response.body).to eq('---\n 0.1.0 |checksum:sha256\n') + expect(last_response.headers['etag']).to eq("md5") + expect(last_response.headers['repr-digest']).to eq('sha-256="sha256"') end + + + it 'responds to compact_index /versions' do + MirrorFile.new('/var/www/gemirro/versions.md5.sha256.list').write('created_at: 2025-01-01T00:00:00Z\m---\nvolay 0.1.0\n') + + get '/versions' + expect(last_response.status).to eq(200) + expect(last_response).to be_ok + expect(last_response.body).to eq('created_at: 2025-01-01T00:00:00Z\m---\nvolay 0.1.0\n') + expect(last_response.headers['etag']).to eq("md5") + expect(last_response.headers['repr-digest']).to eq('sha-256="sha256"') + end + + end context 'Download' do @@ -150,8 +164,8 @@ module Gemirro allow(::Gem::SilentUI).to receive(:new).once.and_return(true) allow(Gemirro.configuration).to receive(:logger) - .exactly(3).and_return(@fake_logger) - allow(@fake_logger).to receive(:info).exactly(3) + .exactly(4).and_return(@fake_logger) + allow(@fake_logger).to receive(:info).exactly(4) get '/gems/gemirro-0.0.1.gem' expect(last_response).to_not be_ok @@ -212,7 +226,8 @@ module Gemirro get '/api/v1/dependencies.json' expect(last_response.headers['Content-Type']) .to eq('application/json') - expect(last_response.body).to eq('') + puts last_response.body + expect(last_response.body).to eq('[]') expect(last_response).to be_ok end @@ -261,6 +276,21 @@ module Gemirro "\x9F+\xFEG\x96") # rubocop:enable Metrics/LineLength + MirrorFile.new('/var/www/gemirro/api/v1/dependencies/volay.md5.sha.list') + .write(Marshal.dump([ + { + name: 'volay', + number: "0.1.0", + platform: 'ruby', + dependencies: [ + { + name: 'json', + requirement: '~> 2.1' + } + ] + } + ])) + gem = Gemirro::GemVersion.new('volay', '0.1.0', 'ruby') collection = Gemirro::GemVersionCollection.new([gem]) allow(Utils).to receive(:gems_collection) diff --git a/template/config.rb b/template/config.rb index e997a43..f6f4b65 100644 --- a/template/config.rb +++ b/template/config.rb @@ -23,6 +23,12 @@ server.access_log File.expand_path('../logs/access.log', __FILE__) server.error_log File.expand_path('../logs/error.log', __FILE__) + # Number of parallel processes while indexing. Too many will kill + # your indexing process prematurely. + # + # update_threads Etc.nprocessors - 1 + # update_threads 4 + # If you don't want to generate indexes after each fetched gem. # # update_on_fetch false diff --git a/template/public/latest_specs.4.8 b/template/public/latest_specs.4.8 new file mode 100644 index 0000000000000000000000000000000000000000..0ba94359df472e358cfe3fa92192fc82832db5b6 GIT binary patch literal 4 LcmZSKh-Lr)0O9~> literal 0 HcmV?d00001 diff --git a/template/public/prerelease_specs.4.8 b/template/public/prerelease_specs.4.8 new file mode 100644 index 0000000000000000000000000000000000000000..0ba94359df472e358cfe3fa92192fc82832db5b6 GIT binary patch literal 4 LcmZSKh-Lr)0O9~> literal 0 HcmV?d00001 diff --git a/template/public/specs.4.8 b/template/public/specs.4.8 new file mode 100644 index 0000000000000000000000000000000000000000..0ba94359df472e358cfe3fa92192fc82832db5b6 GIT binary patch literal 4 LcmZSKh-Lr)0O9~> literal 0 HcmV?d00001 diff --git a/views/gem.erb b/views/gem.erb index bce197a..fde6155 100644 --- a/views/gem.erb +++ b/views/gem.erb @@ -10,19 +10,19 @@
"> -

<%= escape(name) %> <%= escape(versions.newest.number) %>

+

<%= Rack::Utils.escape_html(name) %> <%= Rack::Utils.escape_html(versions.newest.number) %>
<% newest_gem = versions.newest %> - <% if spec = spec_for(name, newest_gem.number, newest_gem.platform) %> -

<%= escape(spec.description) %>

+ <% if spec = Gemirro::Utils.spec_for(name, newest_gem.number, newest_gem.platform) %> +

<%= Rack::Utils.escape_html(spec.description) %>

Dependencies

@@ -31,7 +31,7 @@ @@ -42,9 +42,9 @@ <% versions.each.reverse_each do |version| %>

  • - gem install <%= escape(version.name) %> -v "<%= escape(version.number) %>" + gem install <%= Rack::Utils.escape_html(version.name) %> -v "<%= Rack::Utils.escape_html(version.number) %>" <% unless version.platform =~ /^ruby/i %> - <%= escape(version.platform) %> + <%= Rack::Utils.escape_html(version.platform) %> <% end %>

    diff --git a/views/index.erb b/views/index.erb index f60796f..cdecd67 100644 --- a/views/index.erb +++ b/views/index.erb @@ -12,21 +12,21 @@
    "> -

    <%= escape(name) %> <%= escape(versions.newest.number) %>

    +

    <%= Rack::Utils.escape_html(name) %> <%= Rack::Utils.escape_html(versions.newest.number) %>
    - <% spec = spec_for(name, versions.newest.number) %> + <% spec = Gemirro::Utils.spec_for(name, versions.newest.number) %> <% if spec.is_a?(::Gem::Specification) %> - <%= escape(spec.description) %> + <%= Rack::Utils.escape_html(spec.description) %> <% end %> <% versions.reverse_each.first(5).each do |version| %>

    - gem install <%= escape(version.name) %> <%= "--prerelease" if version.number.to_s.match(/[a-z]/i) %> -v "<%= escape(version.number) %>" + gem install <%= Rack::Utils.escape_html(version.name) %> <%= "--prerelease" if version.number.to_s.match(/[a-z]/i) %> -v "<%= Rack::Utils.escape_html(version.number) %>" <% unless version.platform =~ /^ruby/i %> - <%= escape(version.platform) %> + <%= Rack::Utils.escape_html(version.platform) %> <% end %>

    <% end %>