🧹 optimized k8s asset discovery

Signed-off-by: Ivan Milchev <ivan@mondoo.com>

explorer/scan/discovery.go

@@ -205,6 +205,7 @@ func discoverAssets(rootAssetWithRuntime *AssetWithRuntime, resolvedRootAsset *i
 			if !discoveredAssets.Add(resolvedAsset, assetWithRuntime.Runtime) {
 				assetWithRuntime.Runtime.Close()
 			}
+			discoverAssets(assetWithRuntime, resolvedRootAsset, discoveredAssets, runtimeLabels, upstream, recording)
 		} else {
 			discoverAssets(assetWithRuntime, resolvedRootAsset, discoveredAssets, runtimeLabels, upstream, recording)
 			assetWithRuntime.Runtime.Close()

providers/k8s/resources/clusterrole.go

@@ -22,7 +22,7 @@ type mqlK8sRbacClusterroleInternal struct {
 }
 
 func (k *mqlK8s) clusterroles() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("clusterroles")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("clusterroles")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 		clusterRole, ok := resource.(*rbacv1.ClusterRole)
 		if !ok {

providers/k8s/resources/clusterrolebinding.go

@@ -22,7 +22,7 @@ type mqlK8sRbacClusterrolebindingInternal struct {
 }
 
 func (k *mqlK8s) clusterrolebindings() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("clusterrolebindings")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("clusterrolebindings")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		clusterRoleBinding, ok := resource.(*rbacv1.ClusterRoleBinding)

providers/k8s/resources/common.go

@@ -32,14 +32,14 @@ func gvkString(gvk schema.GroupVersionKind) string {
 	return gvk.Kind + "." + gvk.Version + "." + gvk.Group
 }
 
-func k8sResourceToMql(r *plugin.Runtime, kind string, fn resourceConvertFn) ([]interface{}, error) {
+func k8sResourceToMql(r *plugin.Runtime, kind, ns string, fn resourceConvertFn) ([]interface{}, error) {
 	kt, err := k8sProvider(r.Connection)
 	if err != nil {
 		return nil, err
 	}
 
 	// TODO: check if we are running in a namespace scope and retrieve the ns from the provider
-	result, err := kt.Resources(kind, "", "")
+	result, err := kt.Resources(kind, "", ns)
 	if err != nil {
 		return nil, err
 	}
@@ -70,6 +70,14 @@ func k8sResourceToMql(r *plugin.Runtime, kind string, fn resourceConvertFn) ([]i
 	return resp, nil
 }
 
+func getNamespaceScope(runtime *plugin.Runtime) string {
+	asset := runtime.Connection.(shared.Connection).Asset()
+	if asset.Platform.Name == "k8s-namespace" {
+		return asset.Name
+	}
+	return ""
+}
+
 func getNameAndNamespace(runtime *plugin.Runtime) (string, string, error) {
 	asset := runtime.Connection.(shared.Connection).Asset()
 	return asset.Labels["k8s.mondoo.com/name"], asset.Labels["k8s.mondoo.com/namespace"], nil

providers/k8s/resources/configmap.go

@@ -22,7 +22,7 @@ type mqlK8sConfigmapInternal struct {
 }
 
 func (k *mqlK8s) configmaps() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("configmaps")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("configmaps")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		cm, ok := resource.(*corev1.ConfigMap)

providers/k8s/resources/cronjob.go

@@ -22,7 +22,7 @@ type mqlK8sCronjobInternal struct {
 }
 
 func (k *mqlK8s) cronjobs() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(batchv1.SchemeGroupVersion.WithKind("cronjobs")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(batchv1.SchemeGroupVersion.WithKind("cronjobs")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.cronjob", map[string]*llx.RawData{

providers/k8s/resources/customresource.go

@@ -41,7 +41,7 @@ func (k *mqlK8s) customresources() ([]interface{}, error) {
 			return nil, err
 		}
 
-		mqlResources, err := k8sResourceToMql(k.MqlRuntime, crd.GetName(), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+		mqlResources, err := k8sResourceToMql(k.MqlRuntime, crd.GetName(), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 			ts := obj.GetCreationTimestamp()
 
 			r, err := CreateResource(k.MqlRuntime, "k8s.customresource", map[string]*llx.RawData{

providers/k8s/resources/daemonset.go

@@ -22,7 +22,7 @@ type mqlK8sDaemonsetInternal struct {
 }
 
 func (k *mqlK8s) daemonsets() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("daemonsets")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("daemonsets")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.daemonset", map[string]*llx.RawData{

providers/k8s/resources/deployment.go

@@ -22,7 +22,7 @@ type mqlK8sDeploymentInternal struct {
 }
 
 func (k *mqlK8s) deployments() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("deployments")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("deployments")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.deployment", map[string]*llx.RawData{

providers/k8s/resources/discovery.go

@@ -6,6 +6,7 @@ package resources
 import (
 	"bytes"
 	"fmt"
+	"slices"
 	"strings"
 
 	"github.com/gobwas/glob"
@@ -110,6 +111,19 @@ func Discover(runtime *plugin.Runtime, features cnquery.Features) (*inventory.In
 		return nil, err
 	}
 
+	if asset := conn.Asset(); asset.Platform.Name == "k8s-namespace" {
+		nsFilter = NamespaceFilterOpts{include: []string{asset.Name}}
+
+		od := NewPlatformIdOwnershipIndex(asset.PlatformIds[0])
+		assets, err := discoverNamespaceAssets(runtime, conn, invConfig, asset.PlatformIds[0], k8s, nsFilter, resFilters, od)
+		if err != nil {
+			return nil, err
+		}
+		setRelatedAssets(conn, asset, assets, od, features)
+		in.Spec.Assets = append(in.Spec.Assets, assets...)
+		return in, nil
+	}
+
 	// If we can discover the cluster asset, then we use that as root and build all
 	// platform IDs for the assets based on it. If we cannot discover the cluster, we
 	// discover the individual namespaces according to the ns filter and then build
@@ -132,14 +146,14 @@ func Discover(runtime *plugin.Runtime, features cnquery.Features) (*inventory.In
 
 		od := NewPlatformIdOwnershipIndex(assetId)
 
-		assets, err := discoverAssets(runtime, conn, invConfig, assetId, k8s, nsFilter, resFilters, od, false)
+		assets, err := discoverNamespaces(conn, invConfig, "", nsFilter, nil)
 		if err != nil {
 			return nil, err
 		}
 		setRelatedAssets(conn, root, assets, od, features)
 		in.Spec.Assets = append(in.Spec.Assets, assets...)
 	} else {
-		nss, err := discoverNamespaces(conn, invConfig, "", nil, nsFilter)
+		nss, err := discoverNamespaces(conn, invConfig, "", nsFilter, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -149,25 +163,83 @@ func Discover(runtime *plugin.Runtime, features cnquery.Features) (*inventory.In
 		}
 
 		// Discover the assets for each namespace and use the namespace platform ID as root
-		for _, ns := range nss {
-			nsFilter = NamespaceFilterOpts{include: []string{ns.Name}}
+		// for _, ns := range nss {
+		// 	nsFilter = NamespaceFilterOpts{include: []string{ns.Name}}
 
-			od := NewPlatformIdOwnershipIndex(ns.PlatformIds[0])
+		// 	od := NewPlatformIdOwnershipIndex(ns.PlatformIds[0])
+		// 	assets, err := discoverNamespaceAssets(runtime, conn, invConfig, ns.PlatformIds[0], k8s, nsFilter, resFilters, od)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	setRelatedAssets(conn, ns, assets, od, features)
+		// 	in.Spec.Assets = append(in.Spec.Assets, assets...)
+		// }
+	}
 
-			// We don't want to discover the namespaces again since we have already done this above
-			assets, err := discoverAssets(runtime, conn, invConfig, ns.PlatformIds[0], k8s, nsFilter, resFilters, od, true)
+	return in, nil
+}
+
+func discoverNamespaces(
+	conn shared.Connection,
+	invConfig *inventory.Config,
+	clusterId string,
+	nsFilter NamespaceFilterOpts,
+	od *PlatformIdOwnershipIndex,
+) ([]*inventory.Asset, error) {
+	if slices.Contains(invConfig.Discover.Targets, DiscoveryNamespaces) || slices.Contains(invConfig.Discover.Targets, DiscoveryAuto) {
+		// We don't use MQL here since we need to handle k8s permission errors
+		nss, err := conn.Namespaces()
+		if err != nil {
+			if k8sErrors.IsForbidden(err) && len(nsFilter.include) > 0 {
+				for _, ns := range nsFilter.include {
+					n, err := conn.Namespace(ns)
+					if err != nil {
+						return nil, err
+					}
+					nss = append(nss, *n)
+				}
+			} else {
+				return nil, errors.Wrap(err, "failed to list namespaces")
+			}
+		}
+
+		assetList := make([]*inventory.Asset, 0, len(nss))
+		for _, ns := range nss {
+			if skip := nsFilter.skipNamespace(ns.Name); skip {
+				continue
+			}
+
+			labels := map[string]string{}
+			for k, v := range ns.Labels {
+				labels[k] = v
+			}
+			addMondooAssetLabels(labels, &ns.ObjectMeta, clusterId)
+			platform, err := createPlatformData(ns.Kind, conn.Runtime())
 			if err != nil {
 				return nil, err
 			}
-			setRelatedAssets(conn, ns, assets, od, features)
-			in.Spec.Assets = append(in.Spec.Assets, assets...)
+			assetList = append(assetList, &inventory.Asset{
+				PlatformIds: []string{
+					shared.NewNamespacePlatformId(clusterId, ns.Name, string(ns.UID)),
+				},
+				Name:     ns.Name,
+				Platform: platform,
+				Labels:   labels,
+				// We don't want a parent connection so there is no central cache for the resources
+				// for the complete cluster. We only cache resources for a single namespace
+				Connections: []*inventory.Config{invConfig.Clone()},
+				Category:    conn.Asset().Category,
+			})
+			if od != nil {
+				od.Add(&ns)
+			}
 		}
+		return assetList, nil
 	}
-
-	return in, nil
+	return nil, nil
 }
 
-func discoverAssets(
+func discoverNamespaceAssets(
 	runtime *plugin.Runtime,
 	conn shared.Connection,
 	invConfig *inventory.Config,
@@ -176,7 +248,6 @@ func discoverAssets(
 	nsFilter NamespaceFilterOpts,
 	resFilters *ResourceFilters,
 	od *PlatformIdOwnershipIndex,
-	skipNsDiscovery bool,
 ) ([]*inventory.Asset, error) {
 	var assets []*inventory.Asset
 	var err error
@@ -252,13 +323,6 @@ func discoverAssets(
 			}
 			assets = append(assets, list...)
 		}
-		if target == DiscoveryNamespaces && !skipNsDiscovery {
-			list, err = discoverNamespaces(conn, invConfig, clusterId, od, nsFilter)
-			if err != nil {
-				return nil, err
-			}
-			assets = append(assets, list...)
-		}
 		if target == DiscoveryContainerImages || target == DiscoveryAll {
 			list, err = discoverContainerImages(conn, runtime, invConfig, clusterId, k8s, nsFilter)
 			if err != nil {
@@ -807,61 +871,6 @@ func discoverIngresses(
 	return assetList, nil
 }
 
-func discoverNamespaces(
-	conn shared.Connection,
-	invConfig *inventory.Config,
-	clusterId string,
-	od *PlatformIdOwnershipIndex,
-	nsFilter NamespaceFilterOpts,
-) ([]*inventory.Asset, error) {
-	// We don't use MQL here since we need to handle k8s permission errors
-	nss, err := conn.Namespaces()
-	if err != nil {
-		if k8sErrors.IsForbidden(err) && len(nsFilter.include) > 0 {
-			for _, ns := range nsFilter.include {
-				n, err := conn.Namespace(ns)
-				if err != nil {
-					return nil, err
-				}
-				nss = append(nss, *n)
-			}
-		} else {
-			return nil, errors.Wrap(err, "failed to list namespaces")
-		}
-	}
-
-	assetList := make([]*inventory.Asset, 0, len(nss))
-	for _, ns := range nss {
-		if skip := nsFilter.skipNamespace(ns.Name); skip {
-			continue
-		}
-
-		labels := map[string]string{}
-		for k, v := range ns.Labels {
-			labels[k] = v
-		}
-		addMondooAssetLabels(labels, &ns.ObjectMeta, clusterId)
-		platform, err := createPlatformData(ns.Kind, conn.Runtime())
-		if err != nil {
-			return nil, err
-		}
-		assetList = append(assetList, &inventory.Asset{
-			PlatformIds: []string{
-				shared.NewNamespacePlatformId(clusterId, ns.Name, string(ns.UID)),
-			},
-			Name:        ns.Name,
-			Platform:    platform,
-			Labels:      labels,
-			Connections: []*inventory.Config{invConfig.Clone(inventory.WithoutDiscovery(), inventory.WithParentConnectionId(invConfig.Id))}, // pass-in the parent connection config
-			Category:    conn.Asset().Category,
-		})
-		if od != nil {
-			od.Add(&ns)
-		}
-	}
-	return assetList, nil
-}
-
 func discoverContainerImages(conn shared.Connection, runtime *plugin.Runtime, invConfig *inventory.Config, clusterId string, k8s *mqlK8s, nsFilter NamespaceFilterOpts) ([]*inventory.Asset, error) {
 	pods := k8s.GetPods()
 	if pods.Error != nil {

providers/k8s/resources/ingress.go

@@ -26,7 +26,7 @@ type mqlK8sIngressInternal struct {
 }
 
 func (k *mqlK8s) ingresses() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(networkingv1.SchemeGroupVersion.WithKind("ingresses")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(networkingv1.SchemeGroupVersion.WithKind("ingresses")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		ingress, ok := resource.(*networkingv1.Ingress)

providers/k8s/resources/job.go

@@ -22,7 +22,7 @@ type mqlK8sJobInternal struct {
 }
 
 func (k *mqlK8s) jobs() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(batchv1.SchemeGroupVersion.WithKind("jobs")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(batchv1.SchemeGroupVersion.WithKind("jobs")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.job", map[string]*llx.RawData{

providers/k8s/resources/networkpolicy.go

@@ -21,7 +21,7 @@ type mqlK8sNetworkpolicyInternal struct {
 }
 
 func (k *mqlK8s) networkPolicies() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(networkingv1.SchemeGroupVersion.WithKind("networkpolicies")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(networkingv1.SchemeGroupVersion.WithKind("networkpolicies")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		networkPolicy, ok := resource.(*networkingv1.NetworkPolicy)

providers/k8s/resources/node.go

@@ -53,7 +53,7 @@ func initK8sNode(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[str
 
 func (k *mqlK8s) nodes() ([]interface{}, error) {
 	k.mqlK8sInternal.nodesByName = make(map[string]*mqlK8sNode)
-	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("nodes")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("nodes")), "", func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		n, ok := obj.(*corev1.Node)

providers/k8s/resources/pod.go

@@ -22,7 +22,8 @@ type mqlK8sPodInternal struct {
 }
 
 func (k *mqlK8s) pods() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("pods")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	// TODO: make sure the function below retrieves scoped down resources when the current asset is a namespace
+	return k8sResourceToMql(k.MqlRuntime, gvkString(corev1.SchemeGroupVersion.WithKind("pods")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.pod", map[string]*llx.RawData{

providers/k8s/resources/podsecuritypolicy.go

@@ -20,7 +20,7 @@ type mqlK8sPodsecuritypolicyInternal struct {
 }
 
 func (k *mqlK8s) podSecurityPolicies() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(policyv1beta1.SchemeGroupVersion.WithKind("podsecuritypolicies")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(policyv1beta1.SchemeGroupVersion.WithKind("podsecuritypolicies")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		psp, ok := resource.(*policyv1beta1.PodSecurityPolicy)

providers/k8s/resources/replicaset.go

@@ -22,7 +22,7 @@ type mqlK8sReplicasetInternal struct {
 }
 
 func (k *mqlK8s) replicasets() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("replicasets")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(appsv1.SchemeGroupVersion.WithKind("replicasets")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		r, err := CreateResource(k.MqlRuntime, "k8s.replicaset", map[string]*llx.RawData{

providers/k8s/resources/role.go

@@ -22,7 +22,7 @@ type mqlK8sRbacRoleInternal struct {
 }
 
 func (k *mqlK8s) roles() ([]interface{}, error) {
-	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("roles")), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
+	return k8sResourceToMql(k.MqlRuntime, gvkString(rbacv1.SchemeGroupVersion.WithKind("roles")), getNamespaceScope(k.MqlRuntime), func(kind string, resource runtime.Object, obj metav1.Object, objT metav1.Type) (interface{}, error) {
 		ts := obj.GetCreationTimestamp()
 
 		role, ok := resource.(*rbacv1.Role)