Address feedback (backoff on http 429)

Author: remeh

fluent-plugin-datadog.gemspec

@@ -11,7 +11,7 @@ require "fluent/plugin/version.rb"
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-datadog"
-  spec.version       = Datadog::FluentPlugin::GEM_VERSION
+  spec.version       = DatadogFluentPlugin::VERSION
   spec.authors       = ["Datadog Solutions Team"]
   spec.email         = ["support@datadoghq.com"]
   spec.summary       = "Datadog output plugin for Fluent event collector"

lib/fluent/plugin/out_datadog.rb

@@ -325,7 +325,7 @@ def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_pr
       unless force_v1_routes
         @client.override_headers["DD-API-KEY"] = api_key
         @client.override_headers["DD-EVP-ORIGIN"] = "fluent"
-        @client.override_headers["DD-EVP-ORIGIN-VERSION"] = Datadog::FluentPlugin::GEM_VERSION
+        @client.override_headers["DD-EVP-ORIGIN-VERSION"] = DatadogFluentPlugin::VERSION
       end
       @client.override_headers["Content-Type"] = "application/json"
       if use_compression
@@ -342,7 +342,8 @@ def send(payload)
       request.body = payload
       response = @client.request @uri, request
       res_code = response.code.to_i
-      if res_code >= 500
+      # on a backend error or on an http 429, retry with backoff
+      if res_code >= 500 || res_code == 429
         raise RetryableError.new "Unable to send payload: #{res_code} #{response.message}"
       end
       if res_code >= 400

lib/fluent/plugin/version.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-module Datadog
-  class FluentPlugin
-    GEM_VERSION = '0.14.0'
-  end
+module DatadogFluentPlugin
+  VERSION = '0.14.0'
 end

test/plugin/test_out_datadog.rb

@@ -223,6 +223,18 @@ def create_valid_subject
       end
     end
 
+    # note that in theory, v1 routes should not return a 429 but still
+    # we added this mechanism for v1 routes while implementing v2 ones
+    test "should retry when server is returning 429 (v1 routes)" do
+      api_key = 'XXX'
+      stub_dd_request_with_return_code(api_key, 429)
+      payload = '{}'
+      client = Fluent::DatadogOutput::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 443, 80, nil, false, api_key, true
+      assert_raise(Fluent::DatadogOutput::RetryableError) do
+        client.send(payload)
+      end
+    end
+
     test "should not retry when server is returning 4XX (v1 routes)" do
       api_key = 'XXX'
       stub_dd_request_with_return_code(api_key, 400)
@@ -246,6 +258,16 @@ def create_valid_subject
       end
     end
 
+    test "should retry when server is returning 429 (v2 routes)" do
+      api_key = 'XXX'
+      stub_dd_request_with_return_code(api_key, 429, true)
+      payload = '{}'
+      client = Fluent::DatadogOutput::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 443, 80, nil, false, api_key
+      assert_raise(Fluent::DatadogOutput::RetryableError) do
+          client.send(payload)
+      end
+    end
+
     test "should not retry when server is returning 4XX (v2 routes)" do
       api_key = 'XXX'
       stub_dd_request_with_return_code(api_key, 400, true)
@@ -297,7 +319,7 @@ def stub_dd_request_v2_routes(api_key)
               'Content-Type'=>'application/json',
               'Dd-Api-Key'=> "#{api_key}",
               'Dd-Evp-Origin'=>'fluent',
-              'Dd-Evp-Origin-Version'=> "#{Datadog::FluentPlugin::GEM_VERSION}",
+              'Dd-Evp-Origin-Version'=> "#{DatadogFluentPlugin::VERSION}",
               'Keep-Alive'=>'30',
               'User-Agent'=>'Ruby'
           })