Add GMSA support for V2 process isolated containers

* Add generated V2 schema files for Container Credential Guard
* Add new hcs calls that are necessary to setup container credential guard
instances.
* Add new resource type CCGState that implements ResourceCloser so a containers
ccg instance will be cleaned up on container close.
* Add tests to validate gmsa

Signed-off-by: Daniel Canter <dcanter@microsoft.com>

go.sum

@@ -112,6 +112,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

internal/hcs/system.go

@@ -28,8 +28,7 @@ type System struct {
 	waitBlock      chan struct{}
 	waitError      error
 	exitError      error
-
-	os, typ string
+	os, typ        string
 }
 
 func newSystem(id string) *System {
@@ -95,6 +94,22 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in
 	return computeSystem, nil
 }
 
+// ModifyServiceSettings modifies settings of the host compute service.
+func ModifyServiceSettings(ctx context.Context, settings hcsschema.ModificationRequest) error {
+	operation := "hcsshim::ModifyServiceSettings"
+
+	settingsJSON, err := json.Marshal(settings)
+	if err != nil {
+		return err
+	}
+	resultJSON, err := vmcompute.HcsModifyServiceSettings(ctx, string(settingsJSON))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return &HcsError{Op: operation, Err: err, Events: events}
+	}
+	return nil
+}
+
 // OpenComputeSystem opens an existing compute system by ID.
 func OpenComputeSystem(ctx context.Context, id string) (*System, error) {
 	operation := "hcsshim::OpenComputeSystem"
@@ -173,6 +188,30 @@ func GetComputeSystems(ctx context.Context, q schema1.ComputeSystemQuery) ([]sch
 	return computeSystems, nil
 }
 
+// GetServiceProperties returns properties of the host compute service.
+func GetServiceProperties(ctx context.Context, q hcsschema.PropertyQuery) (*hcsschema.ServiceProperties, error) {
+	operation := "hcsshim::GetServiceProperties"
+
+	queryb, err := json.Marshal(q)
+	if err != nil {
+		return nil, err
+	}
+	propertiesJSON, resultJSON, err := vmcompute.HcsGetServiceProperties(ctx, string(queryb))
+	events := processHcsResult(ctx, resultJSON)
+	if err != nil {
+		return nil, &HcsError{Op: operation, Err: err, Events: events}
+	}
+
+	if propertiesJSON == "" {
+		return nil, ErrUnexpectedValue
+	}
+	properties := &hcsschema.ServiceProperties{}
+	if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil {
+		return nil, err
+	}
+	return properties, nil
+}
+
 // Start synchronously starts the computeSystem.
 func (computeSystem *System) Start(ctx context.Context) (err error) {
 	operation := "hcsshim::System::Start"

internal/hcsoci/create.go

@@ -193,6 +193,14 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
 			log.G(ctx).WithError(err).Debug("failed createHCSContainerDocument")
 			return nil, resources, err
 		}
+		// TODO dcantah: Not too satisfied with how this is added to resources here
+		// but its tricky.
+		if hasCCG(v2) {
+			instance := &CCGState{
+				id: coi.actualID,
+			}
+			resources.resources = append(resources.resources, instance)
+		}
 
 		if schemaversion.IsV10(coi.actualSchemaVersion) {
 			// v1 Argon or Xenon. Pass the document directly to HCS.

internal/hcsoci/credentials.go

@@ -0,0 +1,137 @@
+// +build windows
+
+package hcsoci
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/log"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+)
+
+// CCGState stores a containers Container Credential Guard state. Used when
+// closing a container to be able to release the instance.
+type CCGState struct {
+	// ID of container that instance belongs to.
+	id string
+}
+
+// Release calls into hcs to remove the ccg instance. These do not get cleaned up automatically
+// they MUST be explicitly removed with a call to ModifyServiceSettings. The instances will persist
+// unless vmcompute.exe exits or they are removed manually as done here.
+func (instance *CCGState) Release(ctx context.Context) error {
+	if err := removeCredentialGuard(ctx, instance.id); err != nil {
+		log.G(ctx).WithError(err).WithField("containerID", instance.id).Warn("failed to remove Container Credential Guard instance")
+		return err
+	}
+	return nil
+}
+
+// CreateCredentialGuard creates a container credential guard and returns the state object to be placed in a v2 container doc.
+func CreateCredentialGuard(ctx context.Context, ID, credSpec string, uvm bool) (*hcsschema.ContainerCredentialGuardState, error) {
+	log.G(ctx).WithField("containerID", ID).Debug("creating container credential guard")
+	// V2 schema ccg setup a little different as its expected to be passed
+	// through all the way to the gcs. Can no longer be enabled just through
+	// a single property. The flow is as follows
+	// ------------------------------------------------------------------------
+	// 1. Call HcsModifyServiceSettings with a ModificationRequest set with a
+	// ContainerCredentialGuardAddInstanceRequest. This is where the cred spec
+	// gets passed in. Transport either "LRPC" (Argon) or "HvSocket" (Xenon).
+	// 2. Query the instance with a call to HcsGetServiceProperties with the
+	// PropertyType "ContainerCredentialGuard". This will return all instances
+	// 3. Parse for the id of our container to find which one correlates to the
+	// container we're building the doc for, then add to the v2 doc.
+	// 4. If xenon container the hvsocketconfig will need to be attached BEFORE
+	// being created. It must be in the doc itself as we do not support hot adding
+	// service table entries. This is currently a blocker for adding support for
+	// hyper-v gmsa.
+	transport := "LRPC"
+	if uvm {
+		transport = "HvSocket"
+	}
+	req := hcsschema.ModificationRequest{
+		PropertyType: "ContainerCredentialGuard",
+		Settings: &hcsschema.ContainerCredentialGuardOperationRequest{
+			Operation: hcsschema.AddInstance,
+			OperationDetails: &hcsschema.ContainerCredentialGuardAddInstanceRequest{
+				Id:             ID,
+				CredentialSpec: credSpec,
+				Transport:      transport,
+			},
+		},
+	}
+	if err := hcs.ModifyServiceSettings(ctx, req); err != nil {
+		log.G(ctx).WithError(err).WithField("containerID", ID).Warn("failed to generate container credential guard instance")
+		return nil, err
+	}
+
+	q := hcsschema.PropertyQuery{
+		PropertyTypes: []hcsschema.PropertyType{hcsschema.PTContainerCredentialGuard},
+	}
+	serviceProps, err := hcs.GetServiceProperties(ctx, q)
+	if err != nil {
+		log.G(ctx).WithError(err).WithField("containerID", ID).Warn("failed to retrieve container credential guard instances")
+		return nil, err
+	}
+
+	ccgState, err := unmarshalCCGInstances(serviceProps, ID)
+	if err != nil {
+		return nil, err
+	}
+	return ccgState, nil
+}
+
+// Checks to see if doc has non-nil ccg field
+func hasCCG(doc *hcsschema.Container) bool {
+	if doc.ContainerCredentialGuard != nil {
+		return true
+	}
+	return false
+}
+
+// Removes a ContainerCredentialGuard instance by container ID.
+func removeCredentialGuard(ctx context.Context, ID string) error {
+	log.G(ctx).WithField("containerID", ID).Debug("removing container credential guard")
+
+	req := hcsschema.ModificationRequest{
+		PropertyType: "ContainerCredentialGuard",
+		Settings: &hcsschema.ContainerCredentialGuardOperationRequest{
+			Operation: hcsschema.RemoveInstance,
+			OperationDetails: &hcsschema.ContainerCredentialGuardRemoveInstanceRequest{
+				Id: ID,
+			},
+		},
+	}
+	if err := hcs.ModifyServiceSettings(ctx, req); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Unmarshals a ServiceProperties struct if the only property queried was 'ContainerCredentialGuard'
+// Returns the ContainerCredentialGuardInstance matching the id passed in.
+func unmarshalCCGInstances(sp *hcsschema.ServiceProperties, ID string) (*hcsschema.ContainerCredentialGuardState, error) {
+	if len(sp.Properties) != 1 {
+		return nil, fmt.Errorf("wrong number of service properties present")
+	}
+	// Properties is []interface{}
+	ccgSysInfo := &hcsschema.ContainerCredentialGuardSystemInfo{}
+	ccgJSON, err := json.Marshal(sp.Properties[0])
+	if err != nil {
+		return nil, err
+	}
+
+	if err := json.Unmarshal(ccgJSON, ccgSysInfo); err != nil {
+		return nil, err
+	}
+
+	for _, ccgInstance := range ccgSysInfo.Instances {
+		if ccgInstance.Id == ID {
+			return ccgInstance.CredentialGuard, nil
+		}
+	}
+	return nil, fmt.Errorf("failed to find credential guard instance with ID %s", ID)
+}

internal/hcsoci/hcsdoc_wcow.go

@@ -165,9 +165,20 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter
 		v2Container.Networking.NetworkSharedContainerName = v1.NetworkSharedContainerName
 	}
 
-	//	// TODO V2 Credentials not in the schema yet.
 	if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok {
 		v1.Credentials = cs
+		if coi.HostingSystem != nil {
+			return nil, nil, fmt.Errorf("v2 hyper-v isolated containers do not support gmsa")
+		}
+		// Hardcode false for uvm parameter as hyper-v not supported as of now.
+		// Later on have var xenon assigned to false above and assign true if
+		// coi.HostingSystem != nil
+		ccgState, err := CreateCredentialGuard(ctx, coi.actualID, cs, false)
+		if err != nil {
+			return nil, nil, err
+		}
+		v2Container.ContainerCredentialGuard = ccgState
+		//TODO dcantah: If/when dynamic service table entries is supported register the RpcEndpoint with hvsocket here
 	}
 
 	if coi.Spec.Root == nil {

internal/schema2/container_credential_guard_add_instance_request.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardAddInstanceRequest struct {
+	Id             string `json:"Id,omitempty"`
+	CredentialSpec string `json:"CredentialSpec,omitempty"`
+	Transport      string `json:"Transport,omitempty"`
+}

internal/schema2/container_credential_guard_hv_socket_service_config.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardHvSocketServiceConfig struct {
+	ServiceId     string                 `json:"ServiceId,omitempty"`
+	ServiceConfig *HvSocketServiceConfig `json:"ServiceConfig,omitempty"`
+}

internal/schema2/container_credential_guard_instance.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardInstance struct {
+	Id              string                                         `json:"Id,omitempty"`
+	CredentialGuard *ContainerCredentialGuardState                 `json:"CredentialGuard,omitempty"`
+	HvSocketConfig  *ContainerCredentialGuardHvSocketServiceConfig `json:"HvSocketConfig,omitempty"`
+}

internal/schema2/container_credential_guard_modify_operation.go

@@ -0,0 +1,17 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardModifyOperation string
+
+const (
+	AddInstance    ContainerCredentialGuardModifyOperation = "AddInstance"
+	RemoveInstance ContainerCredentialGuardModifyOperation = "RemoveInstance"
+)

internal/schema2/container_credential_guard_operation_request.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardOperationRequest struct {
+	Operation        ContainerCredentialGuardModifyOperation `json:"Operation,omitempty"`
+	OperationDetails interface{}                             `json:"OperationDetails,omitempty"`
+}

internal/schema2/container_credential_guard_remove_instance_request.go

@@ -0,0 +1,14 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardRemoveInstanceRequest struct {
+	Id string `json:"Id,omitempty"`
+}

internal/schema2/container_credential_guard_system_info.go

@@ -0,0 +1,14 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerCredentialGuardSystemInfo struct {
+	Instances []ContainerCredentialGuardInstance `json:"Instances,omitempty"`
+}

internal/schema2/guest_connection_info.go

@@ -17,5 +17,5 @@ type GuestConnectionInfo struct {
 
 	ProtocolVersion int32 `json:"ProtocolVersion,omitempty"`
 
-	GuestDefinedCapabilities *interface{} `json:"GuestDefinedCapabilities,omitempty"`
+	GuestDefinedCapabilities *interface{} `json:"GuestDefinedCapabilities,omitempty"` // Why a pointer?
 }

internal/schema2/modification_request.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ModificationRequest struct {
+	PropertyType PropertyType `json:"PropertyType,omitempty"`
+	Settings     interface{}  `json:"Settings,omitempty"`
+}

internal/schema2/property_type.go

@@ -18,6 +18,7 @@ const (
 	PTProcessList                 PropertyType = "ProcessList"
 	PTTerminateOnLastHandleClosed PropertyType = "TerminateOnLastHandleClosed"
 	PTSharedMemoryRegion          PropertyType = "SharedMemoryRegion"
+	PTContainerCredentialGuard    PropertyType = "ContainerCredentialGuard" // This was not originally apart of this struct
 	PTGuestConnection             PropertyType = "GuestConnection"
 	PTICHeartbeatStatus           PropertyType = "ICHeartbeatStatus"
 )