feat: track config hash on config reload (#1212)

<!--
Thank you for contributing to the project! 💜
Please make sure to:
- Chat with us first if this is a big change
  - Open a new issue (or comment on an existing one)
- We want to make sure you don't spend time implementing something we
might have to say No to
- Add unit tests
- Mention any relevant issues in the PR description (e.g. "Fixes #123")

Please see our [OSS process
document](https://github.com/honeycombio/home/blob/main/honeycomb-oss-lifecycle-and-practices.md#)
to get an idea of how we operate.
-->

## Which problem is this PR solving?

To provide better visibility of current configuration used in Refinery,
this PR introduce two metrics, `config_hash` and `rule_config_hash`, for
keeping track of configuration.

## Short description of the changes
- change config change log from `info` level to `warn`
- include full config hash value in config change log
- store the decimal number of the last 4 digit of config hash value as
metrics


#967

app/app.go

@@ -26,10 +26,29 @@ type App struct {
 // program will exit.
 func (a *App) Start() error {
 	a.Logger.Debug().Logf("Starting up App...")
+	a.Metrics.Register("config_hash", "gauge")
+	a.Metrics.Register("rule_config_hash", "gauge")
 
 	a.IncomingRouter.SetVersion(a.Version)
 	a.PeerRouter.SetVersion(a.Version)
 
+	a.Config.RegisterReloadCallback(func(configHash, rulesHash string) {
+		if a.Logger != nil {
+			a.Logger.Warn().WithFields(map[string]interface{}{
+				"configHash": configHash,
+				"rulesHash":  rulesHash,
+			}).Logf("configuration change was detected and the configuration was reloaded.")
+
+			cfgMetric := config.ConfigHashMetrics(configHash)
+			ruleMetric := config.ConfigHashMetrics(rulesHash)
+
+			a.Metrics.Gauge("config_hash", cfgMetric)
+			a.Metrics.Gauge("rule_config_hash", ruleMetric)
+
+		}
+
+	})
+
 	// launch our main routers to listen for incoming event traffic from both peers
 	// and external sources
 	a.IncomingRouter.LnS("incoming")

cmd/refinery/main.go

@@ -96,12 +96,6 @@ func main() {
 		fmt.Println("Config and Rules validated successfully.")
 		os.Exit(0)
 	}
-	c.RegisterReloadCallback(func() {
-		if a.Logger != nil {
-			a.Logger.Info().Logf("configuration change was detected and the configuration was reloaded")
-		}
-	})
-
 	// get desired implementation for each dependency to inject
 	lgr := logger.GetLoggerImplementation(c)
 	collector := collect.GetCollectorImplementation(c)

collect/collect.go

@@ -141,7 +141,7 @@ func (i *InMemCollector) Start() error {
 }
 
 // sendReloadSignal will trigger the collector reloading its config, eventually.
-func (i *InMemCollector) sendReloadSignal() {
+func (i *InMemCollector) sendReloadSignal(cfgHash, ruleHash string) {
 	// non-blocking insert of the signal here so we don't leak goroutines
 	select {
 	case i.reload <- struct{}{}:

config/config.go

@@ -21,7 +21,7 @@ type Config interface {
 	// consumers of configuration set config values on startup, they should
 	// check their values haven't changed and re-start anything that needs
 	// restarting with the new values.
-	RegisterReloadCallback(callback func())
+	RegisterReloadCallback(callback ConfigReloadCallback)
 
 	// GetListenAddr returns the address and port on which to listen for
 	// incoming events
@@ -191,6 +191,8 @@ type Config interface {
 	GetParentIdFieldNames() []string
 }
 
+type ConfigReloadCallback func(configHash, ruleCfgHash string)
+
 type ConfigMetadata struct {
 	Type     string `json:"type"`
 	ID       string `json:"id"`

config/configLoadHelpers.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"strconv"
 
 	"github.com/creasty/defaults"
 	"github.com/pelletier/go-toml/v2"
@@ -240,3 +241,21 @@ func readConfigInto(dest any, location string, opts *CmdEnv) (string, error) {
 
 	return hash, nil
 }
+
+// ConfigHashMetrics takes a config hash and returns a integer value for use in metrics.
+// The value is the last 4 characters of the config hash, converted to an integer.
+// If the config hash is too short, or if there is an error converting the hash to an integer,
+// it returns 0.
+func ConfigHashMetrics(hash string) int64 {
+	// get last 4 characters of config hash
+	if len(hash) < 4 {
+		return 0
+	}
+	suffix := hash[len(hash)-4:]
+	CfgDecimal, err := strconv.ParseInt(suffix, 16, 64)
+	if err != nil {
+		return 0
+	}
+
+	return CfgDecimal
+}

config/configLoadHelpers_test.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/require"
 )
 
 func Test_formatFromFilename(t *testing.T) {
@@ -127,3 +129,21 @@ func Test_loadMemsize(t *testing.T) {
 		})
 	}
 }
+
+func Test_ConfigHashMetrics(t *testing.T) {
+	testcases := []struct {
+		name     string
+		hash     string
+		expected int64
+	}{
+		{name: "valid hash", hash: "7f1237f7db723f4e874a7a8269081a77", expected: 6775},
+		{name: "invalid length", hash: "1a8", expected: 0},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := ConfigHashMetrics(tc.hash)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}

config/config_test.go

@@ -212,7 +212,7 @@ func TestReload(t *testing.T) {
 
 	ch := make(chan interface{}, 1)
 
-	c.RegisterReloadCallback(func() {
+	c.RegisterReloadCallback(func(cfgHash, ruleHash string) {
 		close(ch)
 	})
 

config/file_config.go

@@ -38,7 +38,7 @@ type fileConfig struct {
 	rulesConfig   *V2SamplerConfig
 	rulesHash     string
 	opts          *CmdEnv
-	callbacks     []func()
+	callbacks     []ConfigReloadCallback
 	errorCallback func(error)
 	done          chan struct{}
 	ticker        *time.Ticker
@@ -412,7 +412,7 @@ func NewConfig(opts *CmdEnv, errorCallback func(error)) (Config, error) {
 		os.Exit(0)
 	}
 
-	cfg.callbacks = make([]func(), 0)
+	cfg.callbacks = make([]ConfigReloadCallback, 0)
 	cfg.errorCallback = errorCallback
 
 	if cfg.mainConfig.General.ConfigReloadInterval > 0 {
@@ -451,8 +451,9 @@ func (f *fileConfig) monitor() {
 			f.rulesConfig = cfg.rulesConfig
 			f.rulesHash = cfg.rulesHash
 			f.mux.Unlock() // can't defer -- routine never ends, and callbacks will deadlock
+
 			for _, cb := range f.callbacks {
-				cb()
+				cb(cfg.mainHash, cfg.rulesHash)
 			}
 		}
 	}
@@ -469,7 +470,7 @@ func (f *fileConfig) Stop() {
 	}
 }
 
-func (f *fileConfig) RegisterReloadCallback(cb func()) {
+func (f *fileConfig) RegisterReloadCallback(cb ConfigReloadCallback) {
 	f.mux.Lock()
 	defer f.mux.Unlock()
 

config/mock.go

@@ -9,7 +9,7 @@ import (
 // MockConfig will respond with whatever config it's set to do during
 // initialization
 type MockConfig struct {
-	Callbacks                        []func()
+	Callbacks                        []ConfigReloadCallback
 	IsAPIKeyValidFunc                func(string) bool
 	GetCollectorTypeErr              error
 	GetCollectorTypeVal              string
@@ -99,11 +99,11 @@ func (m *MockConfig) ReloadConfig() {
 	defer m.Mux.RUnlock()
 
 	for _, callback := range m.Callbacks {
-		callback()
+		callback("", "")
 	}
 }
 
-func (m *MockConfig) RegisterReloadCallback(callback func()) {
+func (m *MockConfig) RegisterReloadCallback(callback ConfigReloadCallback) {
 	m.Mux.Lock()
 	m.Callbacks = append(m.Callbacks, callback)
 	m.Mux.Unlock()

logger/honeycomb.go

@@ -122,7 +122,7 @@ func (h *HoneycombLogger) readResponses() {
 	}
 }
 
-func (h *HoneycombLogger) reloadBuilder() {
+func (h *HoneycombLogger) reloadBuilder(cfgHash, ruleHash string) {
 	h.Debug().Logf("reloading config for Honeycomb logger")
 	// preserve log level
 	h.level = h.Config.GetLoggerLevel()

metrics/legacy.go

@@ -92,7 +92,7 @@ func (h *LegacyMetrics) Start() error {
 	return nil
 }
 
-func (h *LegacyMetrics) reloadBuilder() {
+func (h *LegacyMetrics) reloadBuilder(cfgHash, ruleHash string) {
 	h.Logger.Debug().Logf("reloading config for honeycomb metrics reporter")
 	mc := h.Config.GetLegacyMetricsConfig()
 	h.libhClient.Close()

transmit/transmit.go

@@ -83,7 +83,7 @@ func (d *DefaultTransmission) Start() error {
 	return nil
 }
 
-func (d *DefaultTransmission) reloadTransmissionBuilder() {
+func (d *DefaultTransmission) reloadTransmissionBuilder(cfgHash, ruleHash string) {
 	d.Logger.Debug().Logf("reloading transmission config")
 	upstreamAPI, err := d.Config.GetHoneycombAPI()
 	if err != nil {