diff --git a/dev-tools/mage/copy.go b/dev-tools/mage/copy.go index fabcfbd87b4..14711bb4597 100644 --- a/dev-tools/mage/copy.go +++ b/dev-tools/mage/copy.go @@ -13,9 +13,15 @@ import ( "regexp" ) +type SkipFn func(string) bool + // Copy copies a file or a directory (recursively) and preserves the permissions. func Copy(src, dest string) error { - copy := &CopyTask{Source: src, Dest: dest} + return CopyWithCheck(src, dest, nil) +} + +func CopyWithCheck(src, dest string, skipFn SkipFn) error { + copy := &CopyTask{Source: src, Dest: dest, skipFn: skipFn} return copy.Execute() } @@ -27,6 +33,7 @@ type CopyTask struct { DirMode os.FileMode // Mode to use for copied dirs. Defaults to preserve permissions. Exclude []string // Exclude paths that match these regular expressions. excludes []*regexp.Regexp // Compiled exclude regexes. + skipFn SkipFn // external check, returns true if we skip } // Execute executes the copy and returns an error of there is a failure. @@ -63,6 +70,15 @@ func (t *CopyTask) isExcluded(src string) bool { return true } } + + return false +} + +func (t *CopyTask) isDestinationExcluded(dst string) bool { + if t.skipFn != nil { + return t.skipFn(dst) + } + return false } @@ -78,6 +94,10 @@ func (t *CopyTask) fileCopy(src, dest string, entry fs.DirEntry) error { return nil } + if t.isDestinationExcluded(dest) { + return nil + } + srcFile, err := os.Open(src) if err != nil { return err @@ -115,6 +135,10 @@ func (t *CopyTask) dirCopy(src, dest string, entry fs.DirEntry) error { return nil } + // if t.isDestinationExcluded(dest) { + // return nil + // } + info, err := entry.Info() if err != nil { return fmt.Errorf("converting dir entry: %w", err) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index cb539145645..8bc7498d9f5 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -18,6 +18,9 @@ import ( "strings" "time" + "github.com/elastic/elastic-agent/internal/pkg/agent/install" + "github.com/elastic/elastic-agent/pkg/component" + "github.com/magefile/mage/sh" ) @@ -103,8 +106,35 @@ func (b *dockerBuilder) exposePorts() []string { func (b *dockerBuilder) copyFiles() error { for _, f := range b.Files { + source := f.Source + var checkFn func(string) bool target := filepath.Join(b.beatDir, f.Target) - if err := Copy(f.Source, target); err != nil { + + if f.ExpandSpec { + specFilename := filepath.Base(source) + specContent, err := os.ReadFile(source) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed reading spec file for component %q: %w", specFilename, err) + } + + // create filter + allowedPaths, err := component.ParseComponentFiles(specContent, specFilename, true) + if err != nil { + return fmt.Errorf("failed computing component files %q: %w", specFilename, err) + } + checkFn, err = install.SkipComponentsPathWithSubpathsFn(allowedPaths) + if err != nil { + return fmt.Errorf("failed compiling skip fn %q: %w", specFilename, err) + } + + source = filepath.Dir(source) // change source to components directory + target = filepath.Dir(target) // target pointing to spec file + } + + if err := CopyWithCheck(source, target, checkFn); err != nil { if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) { continue } diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go index 40b49be181d..f3b92414097 100644 --- a/dev-tools/mage/dockervariants.go +++ b/dev-tools/mage/dockervariants.go @@ -18,6 +18,8 @@ const ( completeWolfi = "complete-wolfi" cloud = "cloud" service = "service" + edotCollector = "edot-collector" + slim = "slim" ) // DockerVariant defines the docker variant to build. @@ -33,6 +35,8 @@ const ( Complete Cloud Service + EdotCollector + Slim ) // String returns the name of the docker variant type. @@ -54,6 +58,10 @@ func (typ DockerVariant) String() string { return cloud case Service: return service + case EdotCollector: + return edotCollector + case Slim: + return slim default: return invalid } @@ -83,6 +91,10 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error { *typ = Cloud case service: *typ = Service + case edotCollector: + *typ = EdotCollector + case slim: + *typ = Slim default: return fmt.Errorf("unknown docker variant: %v", string(text)) } diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index e0b977c0a4c..b5a1068f4a8 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -115,6 +115,10 @@ type PackageSpec struct { localPostRmScript string } +// add new prop into package file called expand spc +// expand spec is checked during packaging and expands to multiple files +// if expand is not present file is copied normally + // PackageFile represents a file or directory within a package. type PackageFile struct { Source string `yaml:"source,omitempty"` // Regular source file or directory. @@ -129,6 +133,7 @@ type PackageFile struct { Owner string `yaml:"owner,omitempty"` // File Owner, for user and group name (rpm only). SkipOnMissing bool `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing. Symlink bool `yaml:"symlink"` // Symlink marks file as a symlink pointing from target to source. + ExpandSpec bool `yaml:"expand_spec,omitempty"` // Optional } // OSArchNames defines the names of architectures for use in packages. diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index a8f10c256c6..a6e961e42fa 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -178,12 +178,18 @@ shared: skip_on_missing: true # Binary package spec (tar.gz for linux) for community beats. + - &agent_binary_spec_no_components + <<: *common + files: + <<: *agent_binary_files + <<: *linux_otel_files + - &agent_binary_spec <<: *common files: <<: *agent_binary_files - <<: *agent_components <<: *linux_otel_files + <<: *agent_components - &agent_darwin_binary_spec @@ -301,6 +307,67 @@ shared: source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml' mode: 0644 + - &agent_docker_edot_spec + <<: *agent_binary_spec_no_components + docker_variant: 'edot-collector' + extra_vars: + dockerfile: 'Dockerfile.elastic-agent.tmpl' + docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' + user: '{{ .BeatName }}' + linux_capabilities: '' + beats_install_path: "install" + files: + 'elastic-agent.yml': + source: 'elastic-agent.docker.yml' + mode: 0600 + config: true + 'otel.yml': + source: 'otel.yml' + mode: 0600 + config: true + '.elastic-agent.active.commit': + content: > + {{ commit }} + mode: 0644 + 'hints.inputs.d': + source: '{{ repo.RootDir }}/deploy/kubernetes/elastic-agent-standalone/templates.d' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/agentbeat.spec.yml': + source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat.spec.yml' + expand_spec: true + + - &agent_docker_slim_spec + <<: *agent_binary_spec_no_components + docker_variant: 'slim' + extra_vars: + dockerfile: 'Dockerfile.elastic-agent.tmpl' + docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' + user: '{{ .BeatName }}' + linux_capabilities: '' + beats_install_path: "install" + files: + 'elastic-agent.yml': + source: 'elastic-agent.docker.yml' + mode: 0600 + config: true + 'otel.yml': + source: 'otel.yml' + mode: 0600 + config: true + '.elastic-agent.active.commit': + content: > + {{ commit }} + mode: 0644 + 'hints.inputs.d': + source: '{{ repo.RootDir }}/deploy/kubernetes/elastic-agent-standalone/templates.d' + mode: 0755 + 'data/{{.BeatName}}-{{ commit_short }}/components/agentbeat.spec.yml': + source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat.spec.yml' + expand_spec: true + 'data/{{.BeatName}}-{{ commit_short }}/components/pf-host-agent.spec.yml': + source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/pf-host-agent.spec.yml' + expand_spec: true + # includes nodejs with @elastic/synthetics - &agent_docker_complete_spec <<: *agent_docker_spec @@ -988,6 +1055,59 @@ specs: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} #### wolfi-based #### + #### Slim docker images #### + - os: linux + arch: amd64 + types: [ docker ] + spec: + # The service image is always based on Wolfi + <<: *docker_wolfi_spec + <<: *docker_builder_spec + <<: *agent_docker_slim_spec + <<: *docker_elastic_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + arch: arm64 + types: [ docker ] + spec: + # The service image is always based on Wolfi + <<: *docker_wolfi_arm_spec + <<: *docker_builder_arm_spec + <<: *agent_docker_slim_spec + <<: *docker_elastic_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + #### EDOT collector docker images #### + - os: linux + arch: amd64 + types: [ docker ] + spec: + # The service image is always based on Wolfi + <<: *docker_wolfi_spec + <<: *docker_builder_spec + <<: *agent_docker_edot_spec + <<: *docker_elastic_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + - os: linux + arch: arm64 + types: [ docker ] + spec: + <<: *docker_wolfi_arm_spec + <<: *docker_builder_arm_spec + <<: *agent_docker_edot_spec + <<: *docker_elastic_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} #### Cloud specific docker images #### - os: linux arch: amd64 diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index cf529e7bb2c..9dc7eb4f755 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -118,6 +118,10 @@ ENV ELASTIC_CONTAINER="true" ENV PATH={{ $beatHome }}:$PATH ENV GODEBUG="madvdontneed=1" +{{- if eq .Variant "edot-collector" }} +ENV ELASTIC_AGENT_OTEL=true +{{- end}} + # Add an init process, check the checksum to make sure it's a match RUN set -e ; \ TINI_BIN=""; \ diff --git a/pkg/testing/kubernetes/supported.go b/pkg/testing/kubernetes/supported.go index 68885bc7458..f62b3f5c254 100644 --- a/pkg/testing/kubernetes/supported.go +++ b/pkg/testing/kubernetes/supported.go @@ -72,6 +72,14 @@ var variants = []struct { Name: "service", Image: "docker.elastic.co/beats-ci/elastic-agent-service", }, + { + Name: "edot-collector", + Image: "docker.elastic.co/beats-ci/elastic-agent-edot-collector", + }, + { + Name: "slim", + Image: "docker.elastic.co/beats-ci/elastic-agent-slim", + }, } // GetSupported returns the list of supported OS types for Kubernetes.