EDOT and Slim docker images added

dev-tools/mage/copy.go

@@ -13,9 +13,15 @@ import (
 	"regexp"
 )
 
+type SkipFn func(string) bool
+
 // Copy copies a file or a directory (recursively) and preserves the permissions.
 func Copy(src, dest string) error {
-	copy := &CopyTask{Source: src, Dest: dest}
+	return CopyWithCheck(src, dest, nil)
+}
+
+func CopyWithCheck(src, dest string, skipFn SkipFn) error {
+	copy := &CopyTask{Source: src, Dest: dest, skipFn: skipFn}
 	return copy.Execute()
 }
 
@@ -27,6 +33,7 @@ type CopyTask struct {
 	DirMode  os.FileMode      // Mode to use for copied dirs. Defaults to preserve permissions.
 	Exclude  []string         // Exclude paths that match these regular expressions.
 	excludes []*regexp.Regexp // Compiled exclude regexes.
+	skipFn   SkipFn           // external check, returns true if we skip
 }
 
 // Execute executes the copy and returns an error of there is a failure.
@@ -63,6 +70,15 @@ func (t *CopyTask) isExcluded(src string) bool {
 			return true
 		}
 	}
+
+	return false
+}
+
+func (t *CopyTask) isDestinationExcluded(dst string) bool {
+	if t.skipFn != nil {
+		return t.skipFn(dst)
+	}
+
 	return false
 }
 
@@ -78,6 +94,10 @@ func (t *CopyTask) fileCopy(src, dest string, entry fs.DirEntry) error {
 		return nil
 	}
 
+	if t.isDestinationExcluded(dest) {
+		return nil
+	}
+
 	srcFile, err := os.Open(src)
 	if err != nil {
 		return err
@@ -115,6 +135,10 @@ func (t *CopyTask) dirCopy(src, dest string, entry fs.DirEntry) error {
 		return nil
 	}
 
+	// if t.isDestinationExcluded(dest) {
+	// 	return nil
+	// }
+
 	info, err := entry.Info()
 	if err != nil {
 		return fmt.Errorf("converting dir entry: %w", err)

dev-tools/mage/dockerbuilder.go

@@ -18,6 +18,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
+	"github.com/elastic/elastic-agent/pkg/component"
+
 	"github.com/magefile/mage/sh"
 )
 
@@ -103,8 +106,35 @@ func (b *dockerBuilder) exposePorts() []string {
 
 func (b *dockerBuilder) copyFiles() error {
 	for _, f := range b.Files {
+		source := f.Source
+		var checkFn func(string) bool
 		target := filepath.Join(b.beatDir, f.Target)
-		if err := Copy(f.Source, target); err != nil {
+
+		if f.ExpandSpec {
+			specFilename := filepath.Base(source)
+			specContent, err := os.ReadFile(source)
+			if err != nil {
+				if os.IsNotExist(err) {
+					return nil
+				}
+				return fmt.Errorf("failed reading spec file for component %q: %w", specFilename, err)
+			}
+
+			// create filter
+			allowedPaths, err := component.ParseComponentFiles(specContent, specFilename, true)
+			if err != nil {
+				return fmt.Errorf("failed computing component files %q: %w", specFilename, err)
+			}
+			checkFn, err = install.SkipComponentsPathWithSubpathsFn(allowedPaths)
+			if err != nil {
+				return fmt.Errorf("failed compiling skip fn %q: %w", specFilename, err)
+			}
+
+			source = filepath.Dir(source) // change source to components directory
+			target = filepath.Dir(target) // target pointing to spec file
+		}
+
+		if err := CopyWithCheck(source, target, checkFn); err != nil {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {
 				continue
 			}

dev-tools/mage/dockervariants.go

@@ -18,6 +18,8 @@ const (
 	completeWolfi = "complete-wolfi"
 	cloud         = "cloud"
 	service       = "service"
+	edotCollector = "edot-collector"
+	slim          = "slim"
 )
 
 // DockerVariant defines the docker variant to build.
@@ -33,6 +35,8 @@ const (
 	Complete
 	Cloud
 	Service
+	EdotCollector
+	Slim
 )
 
 // String returns the name of the docker variant type.
@@ -54,6 +58,10 @@ func (typ DockerVariant) String() string {
 		return cloud
 	case Service:
 		return service
+	case EdotCollector:
+		return edotCollector
+	case Slim:
+		return slim
 	default:
 		return invalid
 	}
@@ -83,6 +91,10 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error {
 		*typ = Cloud
 	case service:
 		*typ = Service
+	case edotCollector:
+		*typ = EdotCollector
+	case slim:
+		*typ = Slim
 	default:
 		return fmt.Errorf("unknown docker variant: %v", string(text))
 	}

dev-tools/mage/pkgtypes.go

@@ -115,6 +115,10 @@ type PackageSpec struct {
 	localPostRmScript      string
 }
 
+// add new prop into package file called expand spc
+// expand spec is checked during packaging and expands to multiple files
+// if expand is not present file is copied normally
+
 // PackageFile represents a file or directory within a package.
 type PackageFile struct {
 	Source        string                  `yaml:"source,omitempty"`   // Regular source file or directory.
@@ -129,6 +133,7 @@ type PackageFile struct {
 	Owner         string                  `yaml:"owner,omitempty"`           // File Owner, for user and group name (rpm only).
 	SkipOnMissing bool                    `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing.
 	Symlink       bool                    `yaml:"symlink"`                   // Symlink marks file as a symlink pointing from target to source.
+	ExpandSpec    bool                    `yaml:"expand_spec,omitempty"`     // Optional
 }
 
 // OSArchNames defines the names of architectures for use in packages.

dev-tools/packaging/packages.yml

@@ -178,12 +178,18 @@ shared:
       skip_on_missing: true
 
   # Binary package spec (tar.gz for linux) for community beats.
+  - &agent_binary_spec_no_components
+    <<: *common
+    files:
+      <<: *agent_binary_files
+      <<: *linux_otel_files
+
   - &agent_binary_spec
     <<: *common
     files:
       <<: *agent_binary_files
-      <<: *agent_components
       <<: *linux_otel_files
+      <<: *agent_components
 
 
   - &agent_darwin_binary_spec
@@ -301,6 +307,67 @@ shared:
         source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml'
         mode: 0644
 
+  - &agent_docker_edot_spec
+    <<: *agent_binary_spec_no_components
+    docker_variant: 'edot-collector'
+    extra_vars:
+      dockerfile: 'Dockerfile.elastic-agent.tmpl'
+      docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
+      user: '{{ .BeatName }}'
+      linux_capabilities: ''
+      beats_install_path: "install"
+    files:
+      'elastic-agent.yml':
+        source: 'elastic-agent.docker.yml'
+        mode: 0600
+        config: true
+      'otel.yml':
+        source: 'otel.yml'
+        mode: 0600
+        config: true
+      '.elastic-agent.active.commit':
+        content: >
+          {{ commit }}
+        mode: 0644
+      'hints.inputs.d':
+        source: '{{ repo.RootDir }}/deploy/kubernetes/elastic-agent-standalone/templates.d'
+        mode: 0755
+      'data/{{.BeatName}}-{{ commit_short }}/components/agentbeat.spec.yml':
+        source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat.spec.yml'
+        expand_spec: true
+
+  - &agent_docker_slim_spec
+    <<: *agent_binary_spec_no_components
+    docker_variant: 'slim'
+    extra_vars:
+      dockerfile: 'Dockerfile.elastic-agent.tmpl'
+      docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
+      user: '{{ .BeatName }}'
+      linux_capabilities: ''
+      beats_install_path: "install"
+    files:
+      'elastic-agent.yml':
+        source: 'elastic-agent.docker.yml'
+        mode: 0600
+        config: true
+      'otel.yml':
+        source: 'otel.yml'
+        mode: 0600
+        config: true
+      '.elastic-agent.active.commit':
+        content: >
+          {{ commit }}
+        mode: 0644
+      'hints.inputs.d':
+        source: '{{ repo.RootDir }}/deploy/kubernetes/elastic-agent-standalone/templates.d'
+        mode: 0755
+      'data/{{.BeatName}}-{{ commit_short }}/components/agentbeat.spec.yml':
+        source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat.spec.yml'
+        expand_spec: true
+      'data/{{.BeatName}}-{{ commit_short }}/components/pf-host-agent.spec.yml':
+        source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/pf-host-agent.spec.yml'
+        expand_spec: true
+
   # includes nodejs with @elastic/synthetics
   - &agent_docker_complete_spec
     <<: *agent_docker_spec
@@ -988,6 +1055,59 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
     #### wolfi-based ####
+    #### Slim docker images ####
+    - os: linux
+      arch: amd64
+      types: [ docker ]
+      spec:
+        # The service image is always based on Wolfi
+        <<: *docker_wolfi_spec
+        <<: *docker_builder_spec
+        <<: *agent_docker_slim_spec
+        <<: *docker_elastic_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+    - os: linux
+      arch: arm64
+      types: [ docker ]
+      spec:
+        # The service image is always based on Wolfi
+        <<: *docker_wolfi_arm_spec
+        <<: *docker_builder_arm_spec
+        <<: *agent_docker_slim_spec
+        <<: *docker_elastic_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+    #### EDOT collector docker images ####
+    - os: linux
+      arch: amd64
+      types: [ docker ]
+      spec:
+        # The service image is always based on Wolfi
+        <<: *docker_wolfi_spec
+        <<: *docker_builder_spec
+        <<: *agent_docker_edot_spec
+        <<: *docker_elastic_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+    - os: linux
+      arch: arm64
+      types: [ docker ]
+      spec:
+        <<: *docker_wolfi_arm_spec
+        <<: *docker_builder_arm_spec
+        <<: *agent_docker_edot_spec
+        <<: *docker_elastic_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
     #### Cloud specific docker images ####
     - os: linux
       arch: amd64

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -118,6 +118,10 @@ ENV ELASTIC_CONTAINER="true"
 ENV PATH={{ $beatHome }}:$PATH
 ENV GODEBUG="madvdontneed=1"
 
+{{- if eq .Variant "edot-collector" }}
+ENV ELASTIC_AGENT_OTEL=true
+{{- end}}
+
 # Add an init process, check the checksum to make sure it's a match
 RUN set -e ; \
   TINI_BIN=""; \

pkg/testing/kubernetes/supported.go

@@ -72,6 +72,14 @@ var variants = []struct {
 		Name:  "service",
 		Image: "docker.elastic.co/beats-ci/elastic-agent-service",
 	},
+	{
+		Name:  "edot-collector",
+		Image: "docker.elastic.co/beats-ci/elastic-agent-edot-collector",
+	},
+	{
+		Name:  "slim",
+		Image: "docker.elastic.co/beats-ci/elastic-agent-slim",
+	},
 }
 
 // GetSupported returns the list of supported OS types for Kubernetes.