Add GMSA support for V2 process isolated containers

* Add generated V2 schema files for Container Credential Guard
* Add new hcs calls that are necessary to setup container credential guard
instances.
* Add new resource type CCGInstance that implements ResourceCloser so a containers
ccg instance will be cleaned up on container close.
* Add tests to validate gmsa
* Remove logging from resource Release methods and just return an error.
Forego returning immediately on an error in ReleaseResources and return
afterwards if any of the releases failed.

Signed-off-by: Daniel Canter <dcanter@microsoft.com>

go.sum

@@ -112,6 +112,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

internal/hcs/service.go

@@ -0,0 +1,49 @@
+package hcs
+
+import (
+	"context"
+	"encoding/json"
+
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+	"github.com/Microsoft/hcsshim/internal/vmcompute"
+)
+
+// GetServiceProperties returns properties of the host compute service.
+func GetServiceProperties(ctx context.Context, q hcsschema.PropertyQuery) (*hcsschema.ServiceProperties, error) {
+	operation := "hcsshim::GetServiceProperties"
+
+	queryb, err := json.Marshal(q)
+	if err != nil {
+		return nil, err
+	}
+	propertiesJSON, resultJSON, err := vmcompute.HcsGetServiceProperties(ctx, string(queryb))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return nil, &HcsError{Op: operation, Err: err, Events: events}
+	}
+
+	if propertiesJSON == "" {
+		return nil, ErrUnexpectedValue
+	}
+	properties := &hcsschema.ServiceProperties{}
+	if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil {
+		return nil, err
+	}
+	return properties, nil
+}
+
+// ModifyServiceSettings modifies settings of the host compute service.
+func ModifyServiceSettings(ctx context.Context, settings hcsschema.ModificationRequest) error {
+	operation := "hcsshim::ModifyServiceSettings"
+
+	settingsJSON, err := json.Marshal(settings)
+	if err != nil {
+		return err
+	}
+	resultJSON, err := vmcompute.HcsModifyServiceSettings(ctx, string(settingsJSON))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return &HcsError{Op: operation, Err: err, Events: events}
+	}
+	return nil
+}

internal/hcs/system.go

@@ -28,8 +28,7 @@ type System struct {
 	waitBlock      chan struct{}
 	waitError      error
 	exitError      error
-
-	os, typ string
+	os, typ        string
 }
 
 func newSystem(id string) *System {

internal/hcsoci/create.go

@@ -59,6 +59,7 @@ type createOptionsInternal struct {
 	actualID               string             // Identifier for the container
 	actualOwner            string             // Owner for the container
 	actualNetworkNamespace string
+	ccgState               *hcsschema.ContainerCredentialGuardState // Container Credential Guard information to be attached to HCS container document
 }
 
 // CreateContainer creates a container. It can cope with a  wide variety of

internal/hcsoci/credentials.go

@@ -0,0 +1,131 @@
+// +build windows
+
+package hcsoci
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/log"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+)
+
+// This file holds the necessary structs and functions for adding and removing Container
+// Credential Guard instances (shortened to CCG normally) for V2 HCS schema
+// containers. Container Credential Guard is in HCS's own words "The solution to
+// allowing windows containers to have access to domain credentials for the
+// applications running in their corresponding guest." It essentially acts as
+// a way to temporarily Active Directory join a given container with a Group
+// Managed Service Account (GMSA for short) credential specification.
+// CCG will launch a process in the host that will act as a middleman for the
+// credential passthrough logic. The guest is then configured through registry
+// keys to have access to the process in the host.
+// A CCG instance needs to be created through various HCS calls and then added to
+// the V2 schema container document before being sent to HCS. For V1 HCS schema containers
+// setting up instances manually is not needed, the GMSA credential specification
+// simply needs to be present in the V1 container document.
+
+// CCGInstance stores the id used when creating a ccg instance. Used when
+// closing a container to be able to release the instance.
+type CCGInstance struct {
+	// ID of container that instance belongs to.
+	id string
+}
+
+// Release calls into hcs to remove the ccg instance. These do not get cleaned up automatically
+// they MUST be explicitly removed with a call to ModifyServiceSettings. The instances will persist
+// unless vmcompute.exe exits or they are removed manually as done here.
+func (instance *CCGInstance) Release(ctx context.Context) error {
+	if err := removeCredentialGuard(ctx, instance.id); err != nil {
+		return fmt.Errorf("failed to remove container credential guard instance: %s", err)
+	}
+	return nil
+}
+
+// CreateCredentialGuard creates a container credential guard instance and
+// returns the state object to be placed in a v2 container doc.
+func CreateCredentialGuard(ctx context.Context, id, credSpec string, hypervisorIsolated bool) (*hcsschema.ContainerCredentialGuardState, *CCGInstance, error) {
+	log.G(ctx).WithField("containerID", id).Debug("creating container credential guard instance")
+	// V2 schema ccg setup a little different as its expected to be passed
+	// through all the way to the gcs. Can no longer be enabled just through
+	// a single property. The flow is as follows
+	// ------------------------------------------------------------------------
+	// 1. Call HcsModifyServiceSettings with a ModificationRequest set with a
+	// ContainerCredentialGuardAddInstanceRequest. This is where the cred spec
+	// gets passed in. Transport either "LRPC" (Argon) or "HvSocket" (Xenon).
+	// 2. Query the instance with a call to HcsGetServiceProperties with the
+	// PropertyType "ContainerCredentialGuard". This will return all instances
+	// 3. Parse for the id of our container to find which one correlates to the
+	// container we're building the doc for, then add to the V2 doc.
+	// 4. If xenon container the hvsocketconfig will need to be in the UVMs V2
+	// schema HcsComputeSystem document before being created/sent to HCS. It must
+	// be in the doc at creation time as we do not support hot adding hvsocket
+	// service table entries.
+	// This is currently a blocker for adding support for hyper-v gmsa.
+	transport := "LRPC"
+	if hypervisorIsolated {
+		// TODO(Dcantah) Set transport to HvSocket here when this is supported
+		return nil, nil, errors.New("hypervisor isolated containers with v2 HCS schema do not support GMSA")
+	}
+	req := hcsschema.ModificationRequest{
+		PropertyType: hcsschema.PTContainerCredentialGuard,
+		Settings: &hcsschema.ContainerCredentialGuardOperationRequest{
+			Operation: hcsschema.AddInstance,
+			OperationDetails: &hcsschema.ContainerCredentialGuardAddInstanceRequest{
+				Id:             id,
+				CredentialSpec: credSpec,
+				Transport:      transport,
+			},
+		},
+	}
+	if err := hcs.ModifyServiceSettings(ctx, req); err != nil {
+		return nil, nil, fmt.Errorf("failed to generate container credential guard instance: %s", err)
+	}
+
+	q := hcsschema.PropertyQuery{
+		PropertyTypes: []hcsschema.PropertyType{hcsschema.PTContainerCredentialGuard},
+	}
+	serviceProps, err := hcs.GetServiceProperties(ctx, q)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to retrieve container credential guard instances: %s", err)
+	}
+	if len(serviceProps.Properties) != 1 {
+		return nil, nil, errors.New("wrong number of service properties present")
+	}
+
+	ccgSysInfo := &hcsschema.ContainerCredentialGuardSystemInfo{}
+	if err := json.Unmarshal(serviceProps.Properties[0], ccgSysInfo); err != nil {
+		return nil, nil, fmt.Errorf("failed to unmarshal container credential guard instances: %s", err)
+	}
+	for _, ccgInstance := range ccgSysInfo.Instances {
+		if ccgInstance.Id == id {
+			instance := &CCGInstance{
+				id,
+			}
+			return ccgInstance.CredentialGuard, instance, nil
+		}
+	}
+	return nil, nil, fmt.Errorf("failed to find credential guard instance with container ID %s", id)
+}
+
+// Removes a ContainerCredentialGuard instance by container ID.
+func removeCredentialGuard(ctx context.Context, id string) error {
+	log.G(ctx).WithField("containerID", id).Debug("removing container credential guard")
+
+	req := hcsschema.ModificationRequest{
+		PropertyType: hcsschema.PTContainerCredentialGuard,
+		Settings: &hcsschema.ContainerCredentialGuardOperationRequest{
+			Operation: hcsschema.RemoveInstance,
+			OperationDetails: &hcsschema.ContainerCredentialGuardRemoveInstanceRequest{
+				Id: id,
+			},
+		},
+	}
+	if err := hcs.ModifyServiceSettings(ctx, req); err != nil {
+		return err
+	}
+	return nil
+}

internal/hcsoci/hcsdoc_wcow.go

@@ -165,9 +165,14 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter
 		v2Container.Networking.NetworkSharedContainerName = v1.NetworkSharedContainerName
 	}
 
-	//	// TODO V2 Credentials not in the schema yet.
 	if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok {
 		v1.Credentials = cs
+		// If this is a HCS v2 schema container, we created the CCG instance
+		// with the other container resources. Pass the CCG state information
+		// as part of the container document.
+		if coi.ccgState != nil {
+			v2Container.ContainerCredentialGuard = coi.ccgState
+		}
 	}
 
 	if coi.Spec.Root == nil {

internal/hcsoci/resources.go

@@ -2,6 +2,7 @@ package hcsoci
 
 import (
 	"context"
+	"errors"
 	"os"
 
 	"github.com/Microsoft/hcsshim/internal/log"
@@ -76,29 +77,42 @@ func ReleaseResources(ctx context.Context, r *Resources, vm *uvm.UtilityVM, all
 		}
 	}
 
+	releaseErr := false
 	// Release resources in reverse order so that the most recently
-	// added are cleaned up first.
+	// added are cleaned up first. We don't return an error right away
+	// so that other resources still get cleaned up in the case of one
+	// or more failing.
 	for i := len(r.resources) - 1; i >= 0; i-- {
 		switch r.resources[i].(type) {
 		case *uvm.NetworkEndpoints:
 			if r.createdNetNS {
 				if err := r.resources[i].Release(ctx); err != nil {
-					return err
+					log.G(ctx).WithError(err).Error("failed to release container resource")
+					releaseErr = true
 				}
 				r.createdNetNS = false
 			}
+		case *CCGInstance:
+			if err := r.resources[i].Release(ctx); err != nil {
+				log.G(ctx).WithError(err).Error("failed to release container resource")
+				releaseErr = true
+			}
 		default:
 			// Don't need to check if vm != nil here anymore as they wouldnt
 			// have been added in the first place. All resources have embedded
 			// vm they belong to.
 			if all {
 				if err := r.resources[i].Release(ctx); err != nil {
-					return err
+					log.G(ctx).WithError(err).Error("failed to release container resource")
+					releaseErr = true
 				}
 			}
 		}
 	}
 	r.resources = nil
+	if releaseErr {
+		return errors.New("failed to release one or more container resources")
+	}
 
 	// cleanup container state
 	if vm != nil {

internal/hcsoci/resources_wcow.go

@@ -138,5 +138,18 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
 		}
 	}
 
+	if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok {
+		// Only need to create a CCG instance for v2 containers
+		if schemaversion.IsV21(coi.actualSchemaVersion) {
+			hypervisorIsolated := coi.HostingSystem != nil
+			ccgState, ccgInstance, err := CreateCredentialGuard(ctx, coi.actualID, cs, hypervisorIsolated)
+			if err != nil {
+				return err
+			}
+			coi.ccgState = ccgState
+			r.resources = append(r.resources, ccgInstance)
+			//TODO dcantah: If/when dynamic service table entries is supported register the RpcEndpoint with hvsocket here
+		}
+	}
 	return nil
 }

internal/schema2/container_credential_guard_add_instance_request.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardAddInstanceRequest struct {
+	Id             string `json:"Id,omitempty"`
+	CredentialSpec string `json:"CredentialSpec,omitempty"`
+	Transport      string `json:"Transport,omitempty"`
+}

internal/schema2/container_credential_guard_hv_socket_service_config.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardHvSocketServiceConfig struct {
+	ServiceId     string                 `json:"ServiceId,omitempty"`
+	ServiceConfig *HvSocketServiceConfig `json:"ServiceConfig,omitempty"`
+}

internal/schema2/container_credential_guard_instance.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardInstance struct {
+	Id              string                                         `json:"Id,omitempty"`
+	CredentialGuard *ContainerCredentialGuardState                 `json:"CredentialGuard,omitempty"`
+	HvSocketConfig  *ContainerCredentialGuardHvSocketServiceConfig `json:"HvSocketConfig,omitempty"`
+}

internal/schema2/container_credential_guard_modify_operation.go

@@ -0,0 +1,17 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardModifyOperation string
+
+const (
+	AddInstance    ContainerCredentialGuardModifyOperation = "AddInstance"
+	RemoveInstance ContainerCredentialGuardModifyOperation = "RemoveInstance"
+)

internal/schema2/container_credential_guard_operation_request.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardOperationRequest struct {
+	Operation        ContainerCredentialGuardModifyOperation `json:"Operation,omitempty"`
+	OperationDetails interface{}                             `json:"OperationDetails,omitempty"`
+}

internal/schema2/container_credential_guard_remove_instance_request.go

@@ -0,0 +1,14 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardRemoveInstanceRequest struct {
+	Id string `json:"Id,omitempty"`
+}