Skip to content

Commit

Permalink
Bribe rubocop
Browse files Browse the repository at this point in the history
  • Loading branch information
navarone-feekery committed May 31, 2024
1 parent 5049818 commit afd66d9
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 11 deletions.
2 changes: 0 additions & 2 deletions lib/crawler/output_sink/elasticsearch.rb
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,9 @@ def flush # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
rescue Utility::EsClient::IndexingFailedError => e
system_logger.warn("Bulk index failed: #{e}")
reset_ingestion_stats(false)

rescue StandardError => e
system_logger.warn("Bulk index failed for unexpected reason: #{e}")
reset_ingestion_stats(false)

end
end

Expand Down
8 changes: 6 additions & 2 deletions lib/utility/es_client.rb
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,14 @@ def bulk(payload = {})
retries += 1
if retries <= MAX_RETRIES
wait_time = 2**retries
@system_logger.info("Bulk index attempt #{retries} failed: '#{e.message}'. Retrying in #{wait_time} seconds...")
@system_logger.info(<<~LOG.squish)
Bulk index attempt #{retries} failed: '#{e.message}'. Retrying in #{wait_time} seconds...
LOG
sleep(wait_time.seconds) && retry
else
@system_logger.warn("Bulk index failed after #{retries} attempts: '#{e.message}'. Writing payload to file...")
@system_logger.warn(<<~LOG.squish)
Bulk index failed after #{retries} attempts: '#{e.message}'. Writing payload to file...
LOG
store_failed_payload(payload)
raise e
end
Expand Down
23 changes: 16 additions & 7 deletions spec/lib/utility/es_client_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
let(:system_logger) { double }
let(:host) { 'http://notreallyaserver' }
let(:port) { '9200' }
let(:elastic_product_headers) { { 'x-elastic-product': 'Elasticsearch'} }
let(:elastic_product_headers) { { 'x-elastic-product': 'Elasticsearch' } }
let(:config) do
{
elasticsearch: {
Expand Down Expand Up @@ -125,19 +125,22 @@

context 'when there is an error in the first attempt' do
before :each do
stub_request(:post, "#{host}:#{port}/_bulk").to_return({ status: 404, exception: 'Intermittent failure' }, {status: 200, headers: elastic_product_headers})
stub_request(:post, "#{host}:#{port}/_bulk").to_return({ status: 404, exception: 'Intermittent failure' },
{ status: 200, headers: elastic_product_headers })
end

it 'succeeds on the retry' do
result = subject.bulk(payload)
expect(result.status).to eq(200)
expect(system_logger).to have_received(:info).with("Bulk index attempt 1 failed: 'Intermittent failure'. Retrying in 2 seconds...")
expect(system_logger).to have_received(:info).with(
"Bulk index attempt 1 failed: 'Intermittent failure'. Retrying in 2 seconds..."
)
end
end

context 'when there is an error in every attempt' do
let(:fixed_time) { Time.new(2024, 1, 1, 0, 0, 0) }
let(:file_double) { double("File", puts: nil, close: nil) }
let(:file_double) { double('File', puts: nil, close: nil) }

before :each do
stub_const('Utility::EsClient::MAX_RETRIES', 1)
Expand All @@ -149,10 +152,16 @@
it 'raises an error after exhausting retries' do
expect { subject.bulk(payload) }.to raise_error(StandardError)

expect(system_logger).to have_received(:info).with("Bulk index attempt 1 failed: 'Consistent failure'. Retrying in 2 seconds...")
expect(system_logger).to have_received(:warn).with("Bulk index failed after 2 attempts: 'Consistent failure'. Writing payload to file...")
expect(system_logger).to have_received(:info).with(
"Bulk index attempt 1 failed: 'Consistent failure'. Retrying in 2 seconds..."
)
expect(system_logger).to have_received(:warn).with(
"Bulk index failed after 2 attempts: 'Consistent failure'. Writing payload to file..."
)

expect(File).to have_received(:open).with("#{Utility::EsClient::FAILED_BULKS_DIR}/crawl-id/#{fixed_time.strftime('%Y%m%d%H%M%S')}", 'w')
expect(File).to have_received(:open).with(
"#{Utility::EsClient::FAILED_BULKS_DIR}/crawl-id/#{fixed_time.strftime('%Y%m%d%H%M%S')}", 'w'
)

expect(file_double).to have_received(:puts).with(payload[:body].first)
expect(file_double).to have_received(:puts).with(payload[:body].second)
Expand Down

0 comments on commit afd66d9

Please sign in to comment.