Merge pull request #5 from GoogleCloudPlatform/severity

Improve severity level support.
GoogleCloudPlatform · Apr 23, 2015 · 394c816 · 394c816
2 parents aae2863 + fe27ee4
commit 394c816
Show file tree

Hide file tree

Showing 2 changed files with 164 additions and 2 deletions.
diff --git a/lib/fluent/plugin/out_google_cloud.rb b/lib/fluent/plugin/out_google_cloud.rb
@@ -171,10 +171,11 @@ def write(chunk)
               },
             },
           }
-          # TODO: default severity?
           if record.has_key?('severity')
-            entry['metadata']['severity'] = record['severity']
+            entry['metadata']['severity'] = parse_severity(record['severity'])
             record.delete('severity')
+          else
+            entry['metadata']['severity'] = 'DEFAULT'
           end
 
           # use textPayload if the only remainaing key is 'message',
@@ -253,6 +254,68 @@ def fetch_metadata(metadata_path)
       end
     end
 
+    # Values permitted by the API for 'severity' (which is an enum).
+    VALID_SEVERITIES = Set.new [
+      'DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
+      'ALERT', 'EMERGENCY']
+
+    # Translates other severity strings to one of the valid values above.
+    SEVERITY_TRANSLATIONS = {
+      # log4j levels (both current and obsolete).
+      'WARN' => 'WARNING',
+      'FATAL' => 'CRITICAL',
+      'TRACE' => 'DEBUG',
+      'TRACE_INT' => 'DEBUG',
+      'FINE' => 'DEBUG',
+      'FINER' => 'DEBUG',
+      'FINEST' => 'DEBUG',
+      # single-letter levels.  Note E->ERROR and D->DEBUG.
+      'D' => 'DEBUG',
+      'I' => 'INFO',
+      'N' => 'NOTICE',
+      'W' => 'WARNING',
+      'E' => 'ERROR',
+      'C' => 'CRITICAL',
+      'A' => 'ALERT',
+      # other misc. translations.
+      'ERR' => 'ERROR',
+    }
+
+    def parse_severity(severity_str)
+      # The API is case insensitive, but uppercase to make things simpler.
+      severity = severity_str.upcase.strip
+
+      # If the severity is already valid, just return it.
+      if (VALID_SEVERITIES.include?(severity))
+        return severity
+      end
+
+      # If the severity is an integer (string) return it as an integer,
+      # truncated to the closest valid value (multiples of 100 between 0-800).
+      if (/\A\d+\z/.match(severity))
+        begin
+          numeric_severity = (severity.to_i / 100) * 100
+          if (numeric_severity < 0)
+            return 0
+          elsif (numeric_severity > 800)
+            return 800
+          else
+            return numeric_severity
+          end
+        rescue
+          return 'DEFAULT'
+        end
+      end
+
+      # Try to translate the severity.
+      if (SEVERITY_TRANSLATIONS.has_key?(severity))
+        return SEVERITY_TRANSLATIONS[severity]
+      end
+
+      # If all else fails, use 'DEFAULT'.
+      return 'DEFAULT'
+    end
+
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',

diff --git a/test/plugin/test_out_google_cloud.rb b/test/plugin/test_out_google_cloud.rb
@@ -207,6 +207,27 @@ def test_timestamps
     end
   end
 
+  def test_severities
+    setup_logging_stubs
+    d = create_driver(PRIVATE_KEY_CONFIG)
+    expected_severity = []
+    emit_index = 0
+    # Array of pairs of [parsed_severity, expected_severity]
+    [['INFO', 'INFO'], ['warn', 'WARNING'], ['E', 'ERROR'],
+     ['BLAH', 'DEFAULT'], ['105', 100], ['', 'DEFAULT']].each do |sev|
+      d.emit({'message' => log_entry(emit_index), 'severity' => sev[0]})
+      expected_severity.push(sev[1])
+      emit_index += 1
+    end
+    d.run
+    verify_index = 0
+    verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
+      assert_equal expected_severity[verify_index],
+        entry['metadata']['severity'], entry
+      verify_index += 1
+    end
+  end
+
   def test_multiple_logs
     setup_logging_stubs
     d = create_driver(PRIVATE_KEY_CONFIG)
@@ -312,6 +333,84 @@ def test_multiple_managed_vm_logs
     end
   end
 
+  # Make parse_severity public so we can test it.
+  class Fluent::GoogleCloudOutput
+    public :parse_severity
+  end
+
+  def test_parse_severity
+    test_obj = Fluent::GoogleCloudOutput.new
+
+    # known severities should translate to themselves, regardless of case
+    ['DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
+     'ALERT', 'EMERGENCY'].each do |severity|
+      assert_equal(severity, test_obj.parse_severity(severity))
+      assert_equal(severity, test_obj.parse_severity(severity.downcase))
+      assert_equal(severity, test_obj.parse_severity(severity.capitalize))
+    end
+
+    # numeric levels
+    assert_equal(0, test_obj.parse_severity('0'))
+    assert_equal(100, test_obj.parse_severity('100'))
+    assert_equal(200, test_obj.parse_severity('200'))
+    assert_equal(300, test_obj.parse_severity('300'))
+    assert_equal(400, test_obj.parse_severity('400'))
+    assert_equal(500, test_obj.parse_severity('500'))
+    assert_equal(600, test_obj.parse_severity('600'))
+    assert_equal(700, test_obj.parse_severity('700'))
+    assert_equal(800, test_obj.parse_severity('800'))
+
+    assert_equal(800, test_obj.parse_severity('900'))
+    assert_equal(0, test_obj.parse_severity('1'))
+    assert_equal(100, test_obj.parse_severity('105'))
+    assert_equal(400, test_obj.parse_severity('420'))
+    assert_equal(700, test_obj.parse_severity('799'))
+
+    assert_equal(100, test_obj.parse_severity('105 '))
+    assert_equal(100, test_obj.parse_severity('     105'))
+    assert_equal(100, test_obj.parse_severity('     105    '))
+
+    assert_equal('DEFAULT', test_obj.parse_severity('-100'))
+    assert_equal('DEFAULT', test_obj.parse_severity('105 100'))
+
+    # synonyms for existing log levels
+    assert_equal('ERROR', test_obj.parse_severity('ERR'))
+    assert_equal('WARNING', test_obj.parse_severity('WARN'))
+    assert_equal('CRITICAL', test_obj.parse_severity('FATAL'))
+    assert_equal('DEBUG', test_obj.parse_severity('TRACE'))
+    assert_equal('DEBUG', test_obj.parse_severity('TRACE_INT'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINE'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINER'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINEST'))
+
+    # single letters.
+    assert_equal('DEBUG', test_obj.parse_severity('D'))
+    assert_equal('INFO', test_obj.parse_severity('I'))
+    assert_equal('NOTICE', test_obj.parse_severity('N'))
+    assert_equal('WARNING', test_obj.parse_severity('W'))
+    assert_equal('ERROR', test_obj.parse_severity('E'))
+    assert_equal('CRITICAL', test_obj.parse_severity('C'))
+    assert_equal('ALERT', test_obj.parse_severity('A'))
+    assert_equal('ERROR', test_obj.parse_severity('e'))
+
+    assert_equal('DEFAULT', test_obj.parse_severity('x'))
+    assert_equal('DEFAULT', test_obj.parse_severity('-'))
+
+    # leading/trailing whitespace should be stripped
+    assert_equal('ERROR', test_obj.parse_severity('  ERROR'))
+    assert_equal('ERROR', test_obj.parse_severity('ERROR  '))
+    assert_equal('ERROR', test_obj.parse_severity('   ERROR  '))
+    assert_equal('ERROR', test_obj.parse_severity("\t  ERROR  "))
+
+    # space in the middle should not be stripped.
+    assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
+
+    # anything else should translate to 'DEFAULT'
+    assert_equal('DEFAULT', test_obj.parse_severity(''))
+    assert_equal('DEFAULT', test_obj.parse_severity('garbage'))
+    assert_equal('DEFAULT', test_obj.parse_severity('er'))
+  end
+
   private
 
   def uri_for_log(config)