Add support for ARM64 Docker images for all Beats (#23592)

This PR adds support for building ARM64 Docker images for all Beats on ARM64 machines.

To limit what packages should be built I have added the new environment variable named `PACKAGES`. It expects a comma-separated list of packages types. Available options: rpm, deb, tgz, dmg, docker, zip. If nothing is specified all available package types are built for the given platform.

This PR relies on the newly added crossbuild image with the suffix `base-arm-debian9`. This image is automatically selected if the runtime of the builder machine is ARM64.

.ci/packaging.groovy

@@ -40,6 +40,7 @@ pipeline {
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
     booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -83,12 +84,13 @@ pipeline {
               }
             }
             setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+            // Stash without any build/dependencies context to support different architectures.
+            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
             withMageEnv(){
               dir("${BASE_DIR}"){
                 setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
               }
             }
-            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
           }
         }
         stage('Build Packages'){
@@ -172,12 +174,73 @@ pipeline {
                 }
                 steps {
                   withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") {
-                    deleteDir()
+                    deleteWorkspace()
                     withMacOSEnv(){
                       release()
                     }
                   }
                 }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('Build Packages ARM'){
+          matrix {
+            axes {
+              axis {
+                name 'BEATS_FOLDER'
+                values (
+                  'auditbeat',
+                  'filebeat',
+                  'heartbeat',
+                  'journalbeat',
+                  'metricbeat',
+                  'packetbeat',
+                  'x-pack/auditbeat',
+                  'x-pack/elastic-agent',
+                  'x-pack/filebeat',
+                  'x-pack/heartbeat',
+                  'x-pack/metricbeat',
+                  'x-pack/packetbeat'
+                )
+              }
+            }
+            stages {
+              stage('Package Docker images for linux/arm64'){
+                agent { label 'arm' }
+                options { skipDefaultCheckout() }
+                when {
+                  beforeAgent true
+                  expression {
+                    return params.arm
+                  }
+                }
+                environment {
+                  HOME = "${env.WORKSPACE}"
+                  PACKAGES = "docker"
+                  PLATFORMS = [
+                    'linux/arm64',
+                  ].join(' ')
+                }
+                steps {
+                  withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
+                    deleteWorkspace()
+                    release()
+                    pushCIDockerImages()
+                  }
+                }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
               }
             }
           }
@@ -408,14 +471,43 @@ def getBeatsName(baseDir) {
 }
 
 def withBeatsEnv(Closure body) {
+  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+  fixPermissions()
   withMageEnv(){
     withEnv([
       "PYTHON_ENV=${WORKSPACE}/python-env"
     ]) {
-      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
       dir("${env.BASE_DIR}"){
         body()
       }
     }
   }
 }
+
+/**
+* This method fixes the filesystem permissions after the build has happenend. The reason is to
+* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental
+* issues.
+*/
+def deleteWorkspace() {
+  catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+    fixPermissions()
+    deleteDir()
+  }
+}
+
+def fixPermissions() {
+  if(isUnix()) {
+    catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+      dir("${env.BASE_DIR}") {
+        if (fileExists('script/fix_permissions.sh')) {
+          sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
+            set +x
+            source ./dev-tools/common.bash
+            docker_setup
+            script/fix_permissions.sh ${WORKSPACE}""", returnStatus: true)
+        }
+      }
+    }
+  }
+}

dev-tools/mage/build.go

@@ -92,6 +92,7 @@ func GolangCrossBuild(params BuildArgs) error {
 	}
 
 	defer DockerChown(filepath.Join(params.OutputDir, params.Name+binaryExtension(GOOS)))
+	defer DockerChown(filepath.Join(params.OutputDir))
 	return Build(params)
 }
 

dev-tools/mage/crossbuild.go

@@ -43,11 +43,26 @@ const defaultCrossBuildTarget = "golangCrossBuild"
 // See NewPlatformList for details about platform filtering expressions.
 var Platforms = BuildPlatforms.Defaults()
 
+// Types is the list of package types
+var SelectedPackageTypes []PackageType
+
 func init() {
 	// Allow overriding via PLATFORMS.
 	if expression := os.Getenv("PLATFORMS"); len(expression) > 0 {
 		Platforms = NewPlatformList(expression)
 	}
+
+	// Allow overriding via PACKAGES.
+	if packageTypes := os.Getenv("PACKAGES"); len(packageTypes) > 0 {
+		for _, pkgtype := range strings.Split(packageTypes, ",") {
+			var p PackageType
+			err := p.UnmarshalText([]byte(pkgtype))
+			if err != nil {
+				continue
+			}
+			SelectedPackageTypes = append(SelectedPackageTypes, p)
+		}
+	}
 }
 
 // CrossBuildOption defines a option to the CrossBuild target.
@@ -169,12 +184,13 @@ func CrossBuildXPack(options ...CrossBuildOption) error {
 	return CrossBuild(o...)
 }
 
-// buildMage pre-compiles the magefile to a binary using the native GOOS/GOARCH
-// values for Docker. It has the benefit of speeding up the build because the
+// buildMage pre-compiles the magefile to a binary using the GOARCH parameter.
+// It has the benefit of speeding up the build because the
 // mage -compile is done only once rather than in each Docker container.
 func buildMage() error {
-	return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "mage", "-f", "-goos=linux", "-goarch=amd64",
-		"-compile", CreateDir(filepath.Join("build", "mage-linux-amd64")))
+	arch := runtime.GOARCH
+	return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "mage", "-f", "-goos=linux", "-goarch="+arch,
+		"-compile", CreateDir(filepath.Join("build", "mage-linux-"+arch)))
 }
 
 func crossBuildImage(platform string) (string, error) {
@@ -185,6 +201,9 @@ func crossBuildImage(platform string) (string, error) {
 		tagSuffix = "darwin"
 	case strings.HasPrefix(platform, "linux/arm"):
 		tagSuffix = "arm"
+		if runtime.GOARCH == "arm64" {
+			tagSuffix = "base-arm-debian9"
+		}
 	case strings.HasPrefix(platform, "linux/mips"):
 		tagSuffix = "mips"
 	case strings.HasPrefix(platform, "linux/ppc"):
@@ -231,9 +250,10 @@ func (b GolangCrossBuilder) Build() error {
 	}
 	workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd))
 
-	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-amd64"))
+	builderArch := runtime.GOARCH
+	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch))
 	if err != nil {
-		return errors.Wrap(err, "failed to determine mage-linux-amd64 relative path")
+		return errors.Wrap(err, "failed to determine mage-linux-"+builderArch+" relative path")
 	}
 
 	dockerRun := sh.RunCmd("docker", "run")

dev-tools/mage/dockerbuilder.go

@@ -70,15 +70,17 @@ func (b *dockerBuilder) Build() error {
 		return errors.Wrap(err, "failed to prepare build")
 	}
 
+	tries := 3
 	tag, err := b.dockerBuild()
-	if err != nil {
+	for err != nil && tries != 0 {
 		fmt.Println(">> Building docker images again (after 10 seconds)")
 		// This sleep is to avoid hitting the docker build issues when resources are not available.
 		time.Sleep(10)
 		tag, err = b.dockerBuild()
-		if err != nil {
-			return errors.Wrap(err, "failed to build docker")
-		}
+		tries -= 1
+	}
+	if err != nil {
+		return errors.Wrap(err, "failed to build docker")
 	}
 
 	if err := b.dockerSave(tag); err != nil {
@@ -199,6 +201,12 @@ func (b *dockerBuilder) dockerBuild() (string, error) {
 }
 
 func (b *dockerBuilder) dockerSave(tag string) error {
+	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
+		err := os.MkdirAll(distributionsDir, 0750)
+		if err != nil {
+			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
+		}
+	}
 	// Save the container as artifact
 	outputFile := b.OutputFile
 	if outputFile == "" {

dev-tools/mage/pkg.go

@@ -45,16 +45,26 @@ func Package() error {
 	var tasks []interface{}
 	for _, target := range Platforms {
 		for _, pkg := range Packages {
-			if pkg.OS != target.GOOS() {
+			if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() {
 				continue
 			}
 
 			for _, pkgType := range pkg.Types {
+				if !isPackageTypeSelected(pkgType) {
+					log.Printf("Skipping %s package type because it is not selected", pkgType)
+					continue
+				}
+
 				if pkgType == DMG && runtime.GOOS != "darwin" {
 					log.Printf("Skipping DMG package type because build host isn't darwin")
 					continue
 				}
 
+				if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" {
+					log.Printf("Skipping Docker package type because build host isn't arm")
+					continue
+				}
+
 				packageArch, err := getOSArchName(target, pkgType)
 				if err != nil {
 					log.Printf("Skipping arch %v for package type %v: %v", target.Arch(), pkgType, err)
@@ -106,6 +116,19 @@ func Package() error {
 	return nil
 }
 
+func isPackageTypeSelected(pkgType PackageType) bool {
+	if SelectedPackageTypes != nil {
+		selected := false
+		for _, t := range SelectedPackageTypes {
+			if t == pkgType {
+				selected = true
+			}
+		}
+		return selected
+	}
+	return true
+}
+
 type packageBuilder struct {
 	Platform BuildPlatform
 	Spec     PackageSpec

dev-tools/mage/pkgtypes.go

@@ -68,6 +68,7 @@ const (
 // system using the contained PackageSpec.
 type OSPackageArgs struct {
 	OS    string        `yaml:"os"`
+	Arch  string        `yaml:"arch,omitempty"`
 	Types []PackageType `yaml:"types"`
 	Spec  PackageSpec   `yaml:"spec"`
 }
@@ -172,6 +173,7 @@ var OSArchNames = map[string]map[PackageType]map[string]string{
 		},
 		Docker: map[string]string{
 			"amd64": "amd64",
+			"arm64": "arm64",
 		},
 	},
 }