From 185dc6fe60f47495f4e314a012e2d87d8e28f979 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 29 Aug 2024 17:47:30 -0400 Subject: [PATCH] feat: [log retention improvements pt. 2] add flag for configuring log retention period (#2536) ## Description Log Retention Improvements: This PR allows users to configure log retention period with `--log-retention-period` flag on engine starts/restarts. Default is set to 1 week. - [x] PR 1: Introduce `LogFileLayout` and `PerWeekLogFileLayout` and adjust `LogFileManager` and `LogsDatabaseClient` to use this for retrieving log file paths - [x] PR 2: Make retention configurable via a CLI flag - [ ] PR 3: Implement `PerHourFileLayout` and swap storage format from `PerWeekFileLayout` to `PerHourFileLayout` - [ ] PR 4: Make `LogsAggregator` use `LogFileLayout` for determining storage format and set it to use `PerHourFileLayout` ------------------------------ ## Is this change user facing? NO ## References https://github.com/kurtosis-tech/kurtosis/issues/2443 https://github.com/kurtosis-tech/kurtosis/issues/2190 https://github.com/kurtosis-tech/kurtosis/pull/2534 --- CHANGELOG.md | 2 +- .../engine_consuming_kurtosis_command.go | 2 +- cli/cli/commands/cluster/set/set.go | 2 +- cli/cli/commands/enclave/add/add.go | 2 +- cli/cli/commands/engine/restart/restart.go | 20 +++++++++++++++- cli/cli/commands/engine/start/start.go | 24 ++++++++++++++++--- cli/cli/commands/github/login/login.go | 2 +- cli/cli/commands/kurtosis_context/set/set.go | 2 +- cli/cli/defaults/defaults.go | 2 ++ .../engine_existence_guarantor.go | 9 +++++++ .../helpers/engine_manager/engine_manager.go | 11 +++++---- docs/docs/cli-reference/engine-restart.md | 2 ++ docs/docs/cli-reference/engine-start.md | 1 + engine/launcher/args/args.go | 4 ++++ .../engine_server_launcher.go | 6 ++++- .../logline/logline_sender.go | 2 +- engine/server/engine/main.go | 22 ++++++++++++----- 17 files changed, 93 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e420c1f55..281703de21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Features -* [log retention improvements pt. 1] introduce file layout interface ([#2534](https://github.com/kurtosis-tech/kurtosis/issues/2534)) ([a494278](https://github.com/kurtosis-tech/kurtosis/commit/a4942781038330bd06d6dc3e9bc4c4550d2048dc)) +* log retention improvements pt. 1: introduce file layout interface ([#2534](https://github.com/kurtosis-tech/kurtosis/issues/2534)) ([a494278](https://github.com/kurtosis-tech/kurtosis/commit/a4942781038330bd06d6dc3e9bc4c4550d2048dc)) * make kurtosis service logs faster ([#2525](https://github.com/kurtosis-tech/kurtosis/issues/2525)) ([d6b246a](https://github.com/kurtosis-tech/kurtosis/commit/d6b246aebe36d81dfca38e6da7fb3bff8d22e36d)) ## [1.0.0](https://github.com/kurtosis-tech/kurtosis/compare/0.90.1...1.0.0) (2024-07-03) diff --git a/cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go b/cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go index fd72080716..d17f47e2db 100644 --- a/cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go +++ b/cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go @@ -187,7 +187,7 @@ func (cmd *EngineConsumingKurtosisCommand) getSetupFunc() func(context.Context) kurtosisBackend := engineManager.GetKurtosisBackend() dontRestartAPIContainers := false - engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain) + engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client") } diff --git a/cli/cli/commands/cluster/set/set.go b/cli/cli/commands/cluster/set/set.go index b31b22ad07..7227046ea8 100644 --- a/cli/cli/commands/cluster/set/set.go +++ b/cli/cli/commands/cluster/set/set.go @@ -124,7 +124,7 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e // TODO - fix the idempotent starter longer term if engineStatus == engine_manager.EngineStatus_Stopped { dontRestartAPIContainers := false - _, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain) + _, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod) if err != nil { return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. Its status can be retrieved "+ "running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'", diff --git a/cli/cli/commands/enclave/add/add.go b/cli/cli/commands/enclave/add/add.go index 1eb3e304e4..7c51077de8 100644 --- a/cli/cli/commands/enclave/add/add.go +++ b/cli/cli/commands/enclave/add/add.go @@ -121,7 +121,7 @@ func run( return stacktrace.Propagate(err, "An error occurred creating an engine manager.") } - engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain) + engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod) if err != nil { return stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client") } diff --git a/cli/cli/commands/engine/restart/restart.go b/cli/cli/commands/engine/restart/restart.go index 0be78b0b28..2e0a97de57 100644 --- a/cli/cli/commands/engine/restart/restart.go +++ b/cli/cli/commands/engine/restart/restart.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel" "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args" @@ -24,6 +25,7 @@ const ( logLevelFlagKey = "log-level" enclavePoolSizeFlagKey = "enclave-pool-size" githubAuthTokenOverrideFlagKey = "github-auth-token" + logRetentionPeriodFlagKey = "log-retention-period" defaultEngineVersion = "" restartEngineOnSameVersionIfAnyRunning = false @@ -92,6 +94,13 @@ var RestartCmd = &lowlevel.LowlevelKurtosisCommand{ Type: flags.FlagType_String, Default: defaultDomain, }, + { + Key: logRetentionPeriodFlagKey, + Usage: "The length of time that Kurtosis should keep logs for. Eg. if set to 168h, Kurtosis will remove all logs beyond 1 week. You can specify hours using 'h' however Kurtosis currently only supports setting retention on a weekly basis.", + Shorthand: "", + Type: flags.FlagType_String, + Default: defaults.DefaultLogRetentionPeriod, + }, }, PreValidationAndRunFunc: nil, RunFunc: run, @@ -158,9 +167,18 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error return stacktrace.Propagate(err, "An error occurred while getting the Kurtosis engine enclave manager UI domain name using the flag with key '%v'; this is a bug in Kurtosis", domainFlagKey) } + logRetentionPeriodStr, err := flags.GetString(logRetentionPeriodFlagKey) + if err != nil { + return stacktrace.Propagate(err, "An error occurred while getting the log retention period string from flag: '%v'", logRetentionPeriodStr) + } + _, err = time.ParseDuration(logRetentionPeriodStr) + if err != nil { + return stacktrace.Propagate(err, "An error occurred parsing provided log retention period '%v' into a duration. Ensure the provided value has the proper format of hours using 'h'.", logRetentionPeriodStr) + } + var engineClientCloseFunc func() error var restartEngineErr error - _, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) if restartEngineErr != nil { return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine") } diff --git a/cli/cli/commands/engine/start/start.go b/cli/cli/commands/engine/start/start.go index 4420a4b46f..a7369eda11 100644 --- a/cli/cli/commands/engine/start/start.go +++ b/cli/cli/commands/engine/start/start.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel" "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args" @@ -24,6 +25,7 @@ const ( logLevelFlagKey = "log-level" enclavePoolSizeFlagKey = "enclave-pool-size" githubAuthTokenOverrideFlagKey = "github-auth-token" + logRetentionPeriodFlagKey = "log-retention-period" defaultEngineVersion = "" kurtosisTechEngineImagePrefix = "kurtosistech/engine" @@ -93,6 +95,13 @@ var StartCmd = &lowlevel.LowlevelKurtosisCommand{ Type: flags.FlagType_String, Default: defaultDomain, }, + { + Key: logRetentionPeriodFlagKey, + Usage: "The length of time that Kurtosis should keep logs for. Eg. if set to 168h, Kurtosis will remove all logs beyond 1 week. You can specify hours using 'h' however Kurtosis currently only supports setting retention on a weekly basis.", + Shorthand: "", + Type: flags.FlagType_String, + Default: defaults.DefaultLogRetentionPeriod, + }, }, PreValidationAndRunFunc: nil, RunFunc: run, @@ -154,16 +163,25 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error return stacktrace.Propagate(err, "An error occurred while getting the Kurtosis engine enclave manager UI domain name using the flag with key '%v'; this is a bug in Kurtosis", domainFlagKey) } + logRetentionPeriodStr, err := flags.GetString(logRetentionPeriodFlagKey) + if err != nil { + return stacktrace.Propagate(err, "An error occurred while getting the log retention period string from flag: '%v'", logRetentionPeriodStr) + } + _, err = time.ParseDuration(logRetentionPeriodStr) + if err != nil { + return stacktrace.Propagate(err, "An error occurred parsing provided log retention period '%v' into a duration. Ensure the provided value has the proper format of hours using 'h'.", logRetentionPeriodStr) + } + if engineVersion == defaultEngineVersion && isDebugMode { engineDebugVersion := fmt.Sprintf("%s-%s", kurtosis_version.KurtosisVersion, defaults.DefaultKurtosisContainerDebugImageNameSuffix) logrus.Infof("Starting Kurtosis engine in debug mode from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineDebugVersion) - _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) } else if engineVersion == defaultEngineVersion { logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, kurtosis_version.KurtosisVersion) - _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) } else { logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineVersion) - _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) } if startEngineErr != nil { return stacktrace.Propagate(startEngineErr, "An error occurred starting the Kurtosis engine") diff --git a/cli/cli/commands/github/login/login.go b/cli/cli/commands/github/login/login.go index 4a7ca9b3f4..6ad37e2024 100644 --- a/cli/cli/commands/github/login/login.go +++ b/cli/cli/commands/github/login/login.go @@ -71,7 +71,7 @@ func RestartEngineAfterGitHubAuth(ctx context.Context) error { var engineClientCloseFunc func() error var restartEngineErr error dontRestartAPIContainers := false - _, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain) + _, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod) if restartEngineErr != nil { return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine") } diff --git a/cli/cli/commands/kurtosis_context/set/set.go b/cli/cli/commands/kurtosis_context/set/set.go index e6bde17d42..ff13451ad8 100644 --- a/cli/cli/commands/kurtosis_context/set/set.go +++ b/cli/cli/commands/kurtosis_context/set/set.go @@ -148,7 +148,7 @@ func SetContext( } dontRestartAPIContainers := false - _, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain) + _, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod) if startEngineErr != nil { logrus.Warnf("The context was successfully set to '%s' but Kurtosis failed to start an engine in "+ "this new context. A new engine should be started manually with '%s %s %s'. The error was:\n%v", diff --git a/cli/cli/defaults/defaults.go b/cli/cli/defaults/defaults.go index 84c52a430d..081138c00b 100644 --- a/cli/cli/defaults/defaults.go +++ b/cli/cli/defaults/defaults.go @@ -26,6 +26,8 @@ const ( DefaultGitHubAuthTokenOverride = "" DefaultDomain = "" + + DefaultLogRetentionPeriod = "168h" ) var DefaultApiContainerLogLevel = logrus.DebugLevel diff --git a/cli/cli/helpers/engine_manager/engine_existence_guarantor.go b/cli/cli/helpers/engine_manager/engine_existence_guarantor.go index c09a536a14..73a1b1e2d8 100644 --- a/cli/cli/helpers/engine_manager/engine_existence_guarantor.go +++ b/cli/cli/helpers/engine_manager/engine_existence_guarantor.go @@ -88,6 +88,9 @@ type engineExistenceGuarantor struct { // Enclave manager UI domain name domain string + + // Length of time Kurtosis will keep logs for + logRetentionPeriod string } func newEngineExistenceGuarantorWithDefaultVersion( @@ -107,6 +110,7 @@ func newEngineExistenceGuarantorWithDefaultVersion( githubAuthTokenOverride string, restartAPIContainers bool, domain string, + logRetentionPeriod string, ) *engineExistenceGuarantor { return newEngineExistenceGuarantorWithCustomVersion( ctx, @@ -126,6 +130,7 @@ func newEngineExistenceGuarantorWithDefaultVersion( githubAuthTokenOverride, restartAPIContainers, domain, + logRetentionPeriod, ) } @@ -147,6 +152,7 @@ func newEngineExistenceGuarantorWithCustomVersion( githubAuthTokenOverride string, restartAPIContainers bool, domain string, + logRetentionPeriod string, ) *engineExistenceGuarantor { return &engineExistenceGuarantor{ ctx: ctx, @@ -168,6 +174,7 @@ func newEngineExistenceGuarantorWithCustomVersion( githubAuthTokenOverride: githubAuthTokenOverride, restartAPIContainers: restartAPIContainers, domain: domain, + logRetentionPeriod: logRetentionPeriod, } } @@ -228,6 +235,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error { githubAuthToken, guarantor.restartAPIContainers, guarantor.domain, + guarantor.logRetentionPeriod, ) } else { _, _, engineLaunchErr = guarantor.engineServerLauncher.LaunchWithCustomVersion( @@ -249,6 +257,7 @@ func (guarantor *engineExistenceGuarantor) VisitStopped() error { githubAuthToken, guarantor.restartAPIContainers, guarantor.domain, + guarantor.logRetentionPeriod, ) } if engineLaunchErr != nil { diff --git a/cli/cli/helpers/engine_manager/engine_manager.go b/cli/cli/helpers/engine_manager/engine_manager.go index 564240afb4..ea727a7f1d 100644 --- a/cli/cli/helpers/engine_manager/engine_manager.go +++ b/cli/cli/helpers/engine_manager/engine_manager.go @@ -183,7 +183,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithDefaultVersion( githubAuthTokenOverride string, restartAPIContainers bool, domain string, -) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) { + logRetentionPeriodStr string) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) { status, maybeHostMachinePortBinding, engineVersion, err := manager.GetEngineStatus(ctx) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred retrieving the Kurtosis engine status, which is necessary for creating a connection to the engine") @@ -207,6 +207,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithDefaultVersion( githubAuthTokenOverride, restartAPIContainers, domain, + logRetentionPeriodStr, ) // TODO Need to handle the Kubernetes case, where a gateway needs to be started after the engine is started but // before we can return an EngineClient @@ -227,7 +228,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithCustomVersion( githubAuthTokenOverride string, restartAPIContainers bool, domain string, -) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) { + logRetentionPeriodStr string) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) { status, maybeHostMachinePortBinding, engineVersion, err := manager.GetEngineStatus(ctx) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred retrieving the Kurtosis engine status, which is necessary for creating a connection to the engine") @@ -252,6 +253,7 @@ func (manager *EngineManager) StartEngineIdempotentlyWithCustomVersion( githubAuthTokenOverride, restartAPIContainers, domain, + logRetentionPeriodStr, ) engineClient, engineClientCloseFunc, err := manager.startEngineWithGuarantor(ctx, status, engineGuarantor) if err != nil { @@ -353,6 +355,7 @@ func (manager *EngineManager) RestartEngineIdempotently( githubAuthTokenOverride string, shouldRestartAPIContainers bool, domain string, + logRetentionPeriodStr string, ) (kurtosis_engine_rpc_api_bindings.EngineServiceClient, func() error, error) { var versionOfNewEngine string // We try to do our best to restart an engine on the same version the current on is on @@ -378,9 +381,9 @@ func (manager *EngineManager) RestartEngineIdempotently( var engineClientCloseFunc func() error var restartEngineErr error if versionOfNewEngine != defaultEngineVersion { - _, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithCustomVersion(ctx, versionOfNewEngine, logLevel, poolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithCustomVersion(ctx, versionOfNewEngine, logLevel, poolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) } else { - _, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, poolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain) + _, engineClientCloseFunc, restartEngineErr = manager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, poolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr) } if restartEngineErr != nil { return nil, nil, stacktrace.Propagate(restartEngineErr, "An error occurred starting a new engine") diff --git a/docs/docs/cli-reference/engine-restart.md b/docs/docs/cli-reference/engine-restart.md index 7ed0750d0f..132280ca50 100644 --- a/docs/docs/cli-reference/engine-restart.md +++ b/docs/docs/cli-reference/engine-restart.md @@ -14,5 +14,7 @@ You may optionally pass in the following flags with this command: * `--log-level`: The level that the started engine should log at. Options include: `panic`, `fatal`, `error`, `warning`, `info`, `debug`, or `trace`. The engine logs at the `info` level by default. * `--version`: The version (Docker tag) of the Kurtosis engine that should be started. If not set, the engine will start up with the default version. * `--enclave-pool-size`: The size of the Kurtosis engine enclave pool. The enclave pool is a component of the Kurtosis engine that allows us to create and maintain 'n' number of idle enclaves for future use. This functionality allows to improve the performance for each new creation enclave request. +* `--github-auth-token`: The auth token to use for authorizing GitHub operations. If set, this will override the currently logged in GitHub user from `kurtosis github login`, if one exists. Note, this token does not persist when restarting the engine. +* `--log-retention-period`: The duration in which Kurtosis engine will keep logs for. The engine will remove any logs beyond this period. You can specify hours using `h`. The default is set to 1 week (168h). NOTE: Currently, Kurtosis only supports setting retention on weekly intervals. Ongoing work is occurring to make this interval more granular - see https://github.com/kurtosis-tech/kurtosis/pull/2534 CAUTION: The `--enclave-pool-size` flag is only available for Kubernetes. \ No newline at end of file diff --git a/docs/docs/cli-reference/engine-start.md b/docs/docs/cli-reference/engine-start.md index 39955dc329..d0737b0973 100644 --- a/docs/docs/cli-reference/engine-start.md +++ b/docs/docs/cli-reference/engine-start.md @@ -16,5 +16,6 @@ You may optionally pass in the following flags with this command: * `--version`: The version (Docker tag) of the Kurtosis engine that should be started. If not set, the engine will start up with the default version. * `--enclave-pool-size`: The size of the Kurtosis engine enclave pool. The enclave pool is a component of the Kurtosis engine that allows us to create and maintain 'n' number of idle enclaves for future use. This functionality allows to improve the performance for each new creation enclave request. * `--github-auth-token`: The auth token to use for authorizing GitHub operations. If set, this will override the currently logged in GitHub user from `kurtosis github login`, if one exists. Note, this token does not persist when restarting the engine. +* `--log-retention-period`: The duration in which Kurtosis engine will keep logs for. The engine will remove any logs beyond this period. You can specify hours using `h`. The default is set to 1 week (168h). NOTE: Currently, Kurtosis only supports setting retention on weekly intervals. Ongoing work is occurring to make this interval more granular - see https://github.com/kurtosis-tech/kurtosis/pull/2534 CAUTION: The `--enclave-pool-size` flag is only available for Kubernetes. \ No newline at end of file diff --git a/engine/launcher/args/args.go b/engine/launcher/args/args.go index 23ebdb2d55..9604837a9c 100644 --- a/engine/launcher/args/args.go +++ b/engine/launcher/args/args.go @@ -67,6 +67,8 @@ type EngineServerArgs struct { // Enclave manager UI domain name Domain string `json:"domain"` + + LogRetentionPeriod string `json:"logRetentionPeriod"` } var skipValidation = map[string]bool{ @@ -125,6 +127,7 @@ func NewEngineServerArgs( allowedCORSOrigins *[]string, restartAPIContainers bool, domain string, + logRetentionPeriod string, ) (*EngineServerArgs, error) { if enclaveEnvVars == "" { enclaveEnvVars = emptyJsonField @@ -146,6 +149,7 @@ func NewEngineServerArgs( AllowedCORSOrigins: allowedCORSOrigins, RestartAPIContainers: restartAPIContainers, Domain: domain, + LogRetentionPeriod: logRetentionPeriod, } if err := result.validate(); err != nil { return nil, stacktrace.Propagate(err, "An error occurred validating engine server args") diff --git a/engine/launcher/engine_server_launcher/engine_server_launcher.go b/engine/launcher/engine_server_launcher/engine_server_launcher.go index f736180584..f9455b811b 100644 --- a/engine/launcher/engine_server_launcher/engine_server_launcher.go +++ b/engine/launcher/engine_server_launcher/engine_server_launcher.go @@ -48,6 +48,7 @@ func (launcher *EngineServerLauncher) LaunchWithDefaultVersion( githubAuthToken string, restartAPIContainers bool, domain string, + logRetentionPeriod string, ) ( resultPublicIpAddr net.IP, resultPublicGrpcPortSpec *port_spec.PortSpec, @@ -71,7 +72,8 @@ func (launcher *EngineServerLauncher) LaunchWithDefaultVersion( shouldStartInDebugMode, githubAuthToken, restartAPIContainers, - domain) + domain, + logRetentionPeriod) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred launching the engine server container with default version tag '%v'", kurtosis_version.KurtosisVersion) } @@ -97,6 +99,7 @@ func (launcher *EngineServerLauncher) LaunchWithCustomVersion( githubAuthToken string, restartAPIContainers bool, domain string, + logRetentionPeriod string, ) ( resultPublicIpAddr net.IP, resultPublicGrpcPortSpec *port_spec.PortSpec, @@ -121,6 +124,7 @@ func (launcher *EngineServerLauncher) LaunchWithCustomVersion( allowedCORSOrigins, restartAPIContainers, domain, + logRetentionPeriod, ) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred creating the engine server args") diff --git a/engine/server/engine/centralized_logs/logline/logline_sender.go b/engine/server/engine/centralized_logs/logline/logline_sender.go index 0e76510dac..a51c274dfa 100644 --- a/engine/server/engine/centralized_logs/logline/logline_sender.go +++ b/engine/server/engine/centralized_logs/logline/logline_sender.go @@ -6,7 +6,7 @@ import ( ) const ( - batchLogsAmount = 500 + batchLogsAmount = 1 logsChanBufferSize = 300 ) diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index e9945db5c0..4670968c54 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "io/fs" + "math" "net" "net/http" "os" @@ -19,8 +20,6 @@ import ( "strings" "time" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" - "github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings" "github.com/kurtosis-tech/kurtosis/api/golang/engine/kurtosis_engine_rpc_api_bindings/kurtosis_engine_rpc_api_bindingsconnect" enclaveApi "github.com/kurtosis-tech/kurtosis/api/golang/http_rest/server/core_rest_api" @@ -64,6 +63,7 @@ const ( successExitCode = 0 failureExitCode = 1 + numHoursInAWeek = 7 * 24 grpcServerStopGracePeriod = 5 * time.Second forceColors = true @@ -175,7 +175,12 @@ func runMain() error { if err != nil { return stacktrace.Propagate(err, "An error occurred getting the Kurtosis backend for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) } - logsDatabaseClient := getLogsDatabaseClient(serverArgs.KurtosisBackendType, kurtosisBackend) + + logRetentionPeriodDuration, err := time.ParseDuration(serverArgs.LogRetentionPeriod) + if err != nil { + return stacktrace.Propagate(err, "An error occurred parsing a duration from provided log retention period string: %v", serverArgs.LogRetentionPeriod) + } + logsDatabaseClient := getLogsDatabaseClient(serverArgs.KurtosisBackendType, kurtosisBackend, logRetentionPeriodDuration) logsDatabaseClient.StartLogFileManagement(ctx) enclaveManager, err := getEnclaveManager( @@ -400,16 +405,21 @@ func getKurtosisBackend(ctx context.Context, kurtosisBackendType args.KurtosisBa } // if cluster is docker, return logs client for centralized logging, otherwise use logs db of kurtosis backend which uses k8s logs under the hood -func getLogsDatabaseClient(kurtosisBackendType args.KurtosisBackendType, kurtosisBackend backend_interface.KurtosisBackend) centralized_logs.LogsDatabaseClient { +func getLogsDatabaseClient(kurtosisBackendType args.KurtosisBackendType, kurtosisBackend backend_interface.KurtosisBackend, logRetentionPeriod time.Duration) centralized_logs.LogsDatabaseClient { var logsDatabaseClient centralized_logs.LogsDatabaseClient switch kurtosisBackendType { case args.KurtosisBackendType_Docker: realTime := logs_clock.NewRealClock() + logRetentionPeriodInWeeks := int(math.Ceil(logRetentionPeriod.Hours() / float64(numHoursInAWeek))) + if logRetentionPeriodInWeeks < 1 { + logRetentionPeriodInWeeks = 1 + } + logrus.Infof("Setting log retention period to '%v' week(s).", logRetentionPeriodInWeeks) osFs := volume_filesystem.NewOsVolumeFilesystem() perWeekFileLayout := file_layout.NewPerWeekFileLayout(realTime) - logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, perWeekFileLayout, realTime, volume_consts.LogRetentionPeriodInWeeks) - perWeekStreamLogsStrategy := stream_logs_strategy.NewPerWeekStreamLogsStrategy(realTime, volume_consts.LogRetentionPeriodInWeeks) + logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, perWeekFileLayout, realTime, logRetentionPeriodInWeeks) + perWeekStreamLogsStrategy := stream_logs_strategy.NewPerWeekStreamLogsStrategy(realTime, logRetentionPeriodInWeeks) logsDatabaseClient = persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, logFileManager, perWeekStreamLogsStrategy) case args.KurtosisBackendType_Kubernetes: