Add ability to restrict which logs are collected. (#244)

Fixes #226

This PR proposes to add a `-l` flag so that the log collection can be
restricted with a label selector. e.g

```
# collect everything but only the logs from the es-es-mixed-v2-0 pod
eck-diagnostics -r my-ns -l statefulset.kubernetes.io/pod-name=es-es-mixed-v2-0 
```
or combined with a type filter 
```
# collect everything from the Elasticsearch cluster called es but logs only from the es-es-mixed-v2-0 pod
eck-diagnostics -r my-ns -l statefulset.kubernetes.io/pod-name=es-es-mixed-v2-0 -f elasticsearch=es 
```

Operator logs are always collected and this cannot be turned off.

The implementation is somewhat different from the proposal in #226 but I
felt like a label selector was a natural choice for this kind of
filtering on k8s.

README.md

@@ -28,17 +28,19 @@ Usage:
 Flags:
       --diagnostic-image string              Diagnostic image to be used for stack diagnostics, see run-stack-diagnostics (default "docker.elastic.co/eck-dev/support-diagnostics:8.5.0")
       --eck-version string                   ECK version in use, will try to autodetect if not specified
-  -f, --filters strings                      Comma-separated list of filters in format "type=name". ex: elasticsearch=my-cluster (Supported types [agent apm beat elasticsearch enterprisesearch kibana maps])
+  -f, --filters strings                      Comma-separated list of filters in format "type=name". Example: elasticsearch=my-cluster (Supported types [agent apm beat elasticsearch enterprisesearch kibana maps logstash])
   -h, --help                                 help for eck-diagnostics
       --kubeconfig string                    optional path to kube config, defaults to $HOME/.kube/config
+  -l, --log-selectors stringArray            Label selectors to restrict the logs to be collected. Can be specified more than once. Example: -l 'elasticsearch.k8s.elastic.co/node-master=true,elasticsearch.k8s.elastic.co/node-data!=true' -l common.k8s.elastic.co/type=kibana.
   -o, --operator-namespaces strings          Comma-separated list of namespace(s) in which operator(s) are running (default [elastic-system])
       --output-directory string              Path where to output diagnostic results
-  -n, --output-name string                   Name of the output diagnostics file (default "eck-diagnostic-2023-02-21T09-16-17.zip")
+  -n, --output-name string                   Name of the output diagnostics file (default "eck-diagnostics-2024-06-25T09-28-37.zip")
   -r, --resources-namespaces strings         Comma-separated list of namespace(s) in which resources are managed
       --run-agent-diagnostics                Run diagnostics on deployed Elastic Agents. Warning: credentials will not be redacted and appear as plain text in the archive
       --run-stack-diagnostics                Run diagnostics on deployed Elasticsearch clusters and Kibana instances, requires deploying diagnostic Pods into the cluster (default true)
       --stack-diagnostics-timeout duration   Maximum time to wait for Elaticsearch and Kibana diagnostics to complete (default 5m0s)
       --verbose                              Verbose mode
+  -v, --version                              version for eck-diagnostics
 ```
 
 ## Information collected by eck-diagnostics
@@ -85,20 +87,33 @@ The ECK related custom resources are included in those namespaces as well:
 * Kibana
 
 ### Logs
-In the operator namespaces (`-o, --operator-namespaces`) all logs are collected, while in the workload resource namespaces only logs from Pods managed by ECK are collected.
+In the operator namespaces (`-o, --operator-namespaces`) the operator's logs are collected, while in the workload resource namespaces all logs from Pods managed by ECK are collected.
 
 ## Filtering collected resources
 
-The resources in the specified namespaces that are collected by eck-diagnostics can be filtered with the `-f, --filters` flag.
+The resources in the specified namespaces that are collected by eck-diagnostics can be filtered with the `-f, --filters` and `-l, --log-selectors` flags.
 
-### Usage Example
+### Usage Examples
 
 The following example will run the diagnostics for Elastic resources in namespace `a`, and will only return resources associated with either an Elasticsearch cluster named `mycluster` or a Kibana instance named `my-kb`.
 
 ```shell
 eck-diagnostics -r a -f "elasticsearch=mycluster" -f "kibana=my-kb"
 ```
 
+To restrict the amount of logs collected by eck-diagnostics use the `-l, --log-selectors` flag (repeatedly). For example to only collect the logs for Kibana and the Elasticsearch master nodes that are not data nodes: 
+
+
+```shell
+eck-diagnostics -r a -f "elasticsearch=mycluster" -f "kibana=my-kb" -l 'elasticsearch.k8s.elastic.co/node-master=true,elasticsearch.k8s.elastic.co/node-data!=true' -l common.k8s.elastic.co/type=kibana
+```
+
+The log selector flags `-l` can also be used without the filter flags `-f` and [set-based Kubernetes requirement syntax](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#set-based-requirement) is also supported:
+```shell
+eck-diagnostics -r a  -l 'apps.kubernetes.io/pod-index notin(0,1)'
+```
+
+
 ### Filtered resources
 
 Only a certain number of Kubernetes resources support filtering when filtering by an Elastic custom resource.  Along with the named Elastic custom resource type, the following resources will be returned that are associated:

cmd/main.go

@@ -22,8 +22,9 @@ const (
 )
 
 var (
-	filters    []string
-	diagParams = internal.Params{}
+	filters         []string
+	rawLogSelectors []string
+	diagParams      = internal.Params{}
 )
 
 func main() {
@@ -42,7 +43,8 @@ func main() {
 	cmd.Flags().BoolVar(&diagParams.RunAgentDiagnostics, "run-agent-diagnostics", false, "Run diagnostics on deployed Elastic Agents. Warning: credentials will not be redacted and appear as plain text in the archive")
 	cmd.Flags().StringSliceVarP(&diagParams.OperatorNamespaces, operatorNamespaces, "o", []string{"elastic-system"}, "Comma-separated list of namespace(s) in which operator(s) are running")
 	cmd.Flags().StringSliceVarP(&diagParams.ResourcesNamespaces, resourcesNamespaces, "r", nil, "Comma-separated list of namespace(s) in which resources are managed")
-	cmd.Flags().StringSliceVarP(&filters, "filters", "f", nil, fmt.Sprintf(`Comma-separated list of filters in format "type=name". ex: elasticsearch=my-cluster (Supported types %v)`, internal_filters.ValidTypes))
+	cmd.Flags().StringSliceVarP(&filters, "filters", "f", nil, fmt.Sprintf(`Comma-separated list of filters in format "type=name". Example: elasticsearch=my-cluster (Supported types %v)`, internal_filters.ValidTypes))
+	cmd.Flags().StringArrayVarP(&rawLogSelectors, "log-selectors", "l", nil, "Label selectors to restrict the logs to be collected. Can be specified more than once. Example: -l 'elasticsearch.k8s.elastic.co/node-master=true,elasticsearch.k8s.elastic.co/node-data!=true' -l common.k8s.elastic.co/type=kibana")
 	cmd.Flags().StringVar(&diagParams.ECKVersion, "eck-version", "", "ECK version in use, will try to autodetect if not specified")
 	cmd.Flags().StringVar(&diagParams.OutputDir, "output-directory", "", "Path where to output diagnostic results")
 	cmd.Flags().StringVarP(&diagParams.OutputName, "output-name", "n", fmt.Sprintf("eck-diagnostics-%s.zip", time.Now().Format("2006-01-02T15-04-05")), "Name of the output diagnostics file")
@@ -101,11 +103,16 @@ func validation(_ *cobra.Command, _ []string) error {
 }
 
 func parseFilters(_ *cobra.Command, _ []string) error {
-	filters, err := internal_filters.New(filters)
+	filters, err := internal_filters.NewTypeFilter(filters)
 	if err != nil {
 		return err
 	}
 	diagParams.Filters = filters
+	logFilters, err := internal_filters.NewLabelFilter(rawLogSelectors)
+	if err != nil {
+		return err
+	}
+	diagParams.LogFilters = logFilters
 	return nil
 }
 

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-cmp v0.6.0
 	github.com/spf13/cobra v1.8.1
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/text v0.16.0
 	k8s.io/api v0.30.2
 	k8s.io/apimachinery v0.30.2
@@ -57,6 +58,7 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect

go.sum

@@ -149,15 +149,16 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/diag.go

@@ -46,6 +46,7 @@ type Params struct {
 	Verbose                 bool
 	StackDiagnosticsTimeout time.Duration
 	Filters                 filters.Filters
+	LogFilters              filters.Filters
 }
 
 // AllNamespaces returns a slice containing all namespaces from which we want to extract diagnostic data.
@@ -146,12 +147,17 @@ func Run(params Params) error {
 		"common.k8s.elastic.co/type=logstash",          // 2.8.0
 	}
 
-	selectors := make([]labels.Selector, len(operatorLabels))
+	operatorSelectors := make([]labels.Selector, len(operatorLabels))
 	for i, label := range operatorLabels {
-		selectors[i] = label.AsSelector()
+		operatorSelectors[i] = label.AsSelector()
 		logsLabels = append(logsLabels, label.AsSelector().String())
 	}
-	namespaceFilters := params.Filters.WithSelectors(selectors)
+	// always collect operator information even in the presence of filters
+	operatorFilters := filters.LabelFilter(operatorSelectors)
+	// user defined type filters or any resource related to the operator
+	namespaceFilters := filters.Or(params.Filters, operatorFilters)
+	// for logs respect user defined log filters but add the operator filters here too so that the AND works out and we always have the operator logs
+	logsFilters := filters.And(filters.Or(params.LogFilters, operatorFilters), namespaceFilters)
 
 LOOP:
 	for ns := range allNamespaces {
@@ -220,7 +226,7 @@ LOOP:
 			},
 		})
 
-		getLogs(kubectl, zipFile, ns, namespaceFilters, logsLabels...)
+		getLogs(kubectl, zipFile, ns, logsFilters, logsLabels...)
 
 		if params.RunStackDiagnostics {
 			runStackDiagnostics(