Merge pull request #2904 from fabriziopandini/capd-selfhosted

✨[CAPD] make CAPD ready for self-hosted tests

test/infrastructure/docker/api/v1alpha3/dockermachine_types.go

@@ -37,12 +37,38 @@ type DockerMachineSpec struct {
 	// +optional
 	CustomImage string `json:"customImage,omitempty"`
 
+	// PreLoadImages allows to pre-load images in a newly created machine. This can be used to
+	// speed up tests by avoiding e.g. to download CNI images on all the containers.
+	// +optional
+	PreLoadImages []string `json:"preLoadImages,omitempty"`
+
+	// ExtraMounts describes additional mount points for the node container
+	// These may be used to bind a hostPath
+	// +optional
+	ExtraMounts []Mount `json:"extraMounts,omitempty"`
+
 	// Bootstrapped is true when the kubeadm bootstrapping has been run
 	// against this machine
 	// +optional
 	Bootstrapped bool `json:"bootstrapped,omitempty"`
 }
 
+// Mount specifies a host volume to mount into a container.
+// This is a simplified version of kind v1alpha4.Mount types
+type Mount struct {
+	// Path of the mount within the container.
+	ContainerPath string `json:"containerPath,omitempty"`
+
+	// Path of the mount on the host. If the hostPath doesn't exist, then runtimes
+	// should report error. If the hostpath is a symbolic link, runtimes should
+	// follow the symlink and mount the real destination to container.
+	HostPath string `json:"hostPath,omitempty"`
+
+	// If set, the mount is read-only.
+	// +optional
+	Readonly bool `json:"readOnly,omitempty"`
+}
+
 // DockerMachineStatus defines the observed state of DockerMachine
 type DockerMachineStatus struct {
 	// Ready denotes that the machine (docker container) is ready

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml

@@ -46,6 +46,34 @@ spec:
                 description: CustomImage allows customizing the container image that
                   is used for running the machine
                 type: string
+              extraMounts:
+                description: ExtraMounts describes additional mount points for the
+                  node container These may be used to bind a hostPath
+                items:
+                  description: Mount specifies a host volume to mount into a container.
+                    This is a simplified version of kind v1alpha4.Mount types
+                  properties:
+                    containerPath:
+                      description: Path of the mount within the container.
+                      type: string
+                    hostPath:
+                      description: Path of the mount on the host. If the hostPath
+                        doesn't exist, then runtimes should report error. If the hostpath
+                        is a symbolic link, runtimes should follow the symlink and
+                        mount the real destination to container.
+                      type: string
+                    readOnly:
+                      description: If set, the mount is read-only.
+                      type: boolean
+                  type: object
+                type: array
+              preLoadImages:
+                description: PreLoadImages allows to pre-load images in a newly created
+                  machine. This can be used to speed up tests by avoiding e.g. to
+                  download CNI images on all the containers.
+                items:
+                  type: string
+                type: array
               providerID:
                 description: ProviderID will be the container name in ProviderID format
                   (docker:////<containername>)

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml

@@ -55,6 +55,36 @@ spec:
                         description: CustomImage allows customizing the container
                           image that is used for running the machine
                         type: string
+                      extraMounts:
+                        description: ExtraMounts describes additional mount points
+                          for the node container These may be used to bind a hostPath
+                        items:
+                          description: Mount specifies a host volume to mount into
+                            a container. This is a simplified version of kind v1alpha4.Mount
+                            types
+                          properties:
+                            containerPath:
+                              description: Path of the mount within the container.
+                              type: string
+                            hostPath:
+                              description: Path of the mount on the host. If the hostPath
+                                doesn't exist, then runtimes should report error.
+                                If the hostpath is a symbolic link, runtimes should
+                                follow the symlink and mount the real destination
+                                to container.
+                              type: string
+                            readOnly:
+                              description: If set, the mount is read-only.
+                              type: boolean
+                          type: object
+                        type: array
+                      preLoadImages:
+                        description: PreLoadImages allows to pre-load images in a
+                          newly created machine. This can be used to speed up tests
+                          by avoiding e.g. to download CNI images on all the containers.
+                        items:
+                          type: string
+                        type: array
                       providerID:
                         description: ProviderID will be the container name in ProviderID
                           format (docker:////<containername>)

test/infrastructure/docker/controllers/dockermachine_controller.go

@@ -170,16 +170,21 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine *
 		return ctrl.Result{}, nil
 	}
 
-	//Create the docker container hosting the machine
+	// Create the docker container hosting the machine
 	role := constants.WorkerNodeRoleValue
 	if util.IsControlPlaneMachine(machine) {
 		role = constants.ControlPlaneNodeRoleValue
 	}
 
-	if err := externalMachine.Create(ctx, role, machine.Spec.Version); err != nil {
+	if err := externalMachine.Create(ctx, role, machine.Spec.Version, dockerMachine.Spec.ExtraMounts); err != nil {
 		return ctrl.Result{}, errors.Wrap(err, "failed to create worker DockerMachine")
 	}
 
+	// Preload images into the container
+	if err := externalMachine.PreloadLoadImages(ctx, dockerMachine.Spec.PreLoadImages); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to pre-load images into the DockerMachine")
+	}
+
 	// if the machine is a control plane update the load balancer configuration
 	// we should only do this once, as reconfiguration more or less ensures
 	// node ref setting fails

test/infrastructure/docker/docker/kind_manager.go

@@ -342,7 +342,6 @@ func generateMountBindings(mounts ...v1alpha4.Mount) []string {
 		case v1alpha4.MountPropagationHostToContainer:
 			attrs = append(attrs, "rslave")
 		default:
-			fmt.Printf("[W] unknown propagation mode for hostPath %q\n", m.HostPath)
 			// Falls back to "private"
 		}
 

test/infrastructure/docker/docker/machine.go

@@ -21,16 +21,21 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/cloudinit"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/docker/types"
 	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster/constants"
+	"sigs.k8s.io/kind/pkg/exec"
 )
 
 const (
@@ -95,7 +100,7 @@ func (m *Machine) ProviderID() string {
 }
 
 // Create creates a docker container hosting a Kubernetes node.
-func (m *Machine) Create(ctx context.Context, role string, version *string) error {
+func (m *Machine) Create(ctx context.Context, role string, version *string, mounts []infrav1.Mount) error {
 	// Create if not exists.
 	if m.container == nil {
 		var err error
@@ -114,7 +119,7 @@ func (m *Machine) Create(ctx context.Context, role string, version *string) erro
 				clusterLabel(m.cluster),
 				"127.0.0.1",
 				0,
-				nil,
+				kindMounts(mounts),
 				nil,
 			)
 			if err != nil {
@@ -126,7 +131,7 @@ func (m *Machine) Create(ctx context.Context, role string, version *string) erro
 				m.ContainerName(),
 				machineImage,
 				clusterLabel(m.cluster),
-				nil,
+				kindMounts(mounts),
 				nil,
 			)
 			if err != nil {
@@ -149,6 +154,54 @@ func (m *Machine) Create(ctx context.Context, role string, version *string) erro
 	return nil
 }
 
+func kindMounts(mounts []infrav1.Mount) []v1alpha4.Mount {
+	if len(mounts) == 0 {
+		return nil
+	}
+
+	ret := make([]v1alpha4.Mount, 0, len(mounts))
+	for _, m := range mounts {
+		ret = append(ret, v1alpha4.Mount{
+			ContainerPath: m.ContainerPath,
+			HostPath:      m.HostPath,
+			Readonly:      m.Readonly,
+			Propagation:   v1alpha4.MountPropagationNone,
+		})
+	}
+	return ret
+}
+
+func (m *Machine) PreloadLoadImages(ctx context.Context, images []string) error {
+	// Save the image into a tar
+	dir, err := ioutil.TempDir("", "image-tar")
+	if err != nil {
+		return errors.Wrap(err, "failed to create tempdir")
+	}
+	defer os.RemoveAll(dir)
+
+	for i, image := range images {
+		imageTarPath := filepath.Join(dir, fmt.Sprintf("image-%d.tar", i))
+
+		err = exec.Command("docker", "save", "-o", imageTarPath, image).Run()
+		if err != nil {
+			return err
+		}
+
+		f, err := os.Open(imageTarPath)
+		if err != nil {
+			return errors.Wrap(err, "failed to open image")
+		}
+		defer f.Close()
+
+		ps := m.container.Commander.Command("ctr", "--namespace=k8s.io", "images", "import", "-")
+		ps.SetStdin(f)
+		if err := ps.Run(ctx); err != nil {
+			return errors.Wrap(err, "failed to load image")
+		}
+	}
+	return nil
+}
+
 // ExecBootstrap runs bootstrap on a node, this is generally `kubeadm <init|join>`
 func (m *Machine) ExecBootstrap(ctx context.Context, data string) error {
 	if m.container == nil {