From c9e9e4ccccd59f28c42312d5b2df5cbbbe4d11d5 Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Wed, 3 Apr 2024 01:40:42 -0300 Subject: [PATCH] Wavelength provisioning with CAPA changes --- .../v2/api/v1beta2/network_types.go | 135 +++- .../pkg/cloud/services/network/natgateways.go | 8 +- .../pkg/cloud/services/network/routetables.go | 206 ++--- .../v2/pkg/cloud/services/network/subnets.go | 112 ++- pkg/asset/manifests/aws/cluster.go | 19 +- pkg/asset/manifests/aws/zones.go | 382 ++++++--- pkg/asset/manifests/aws/zones_test.go | 756 +++++++++++++++--- .../v2/api/v1beta2/network_types.go | 8 +- 8 files changed, 1216 insertions(+), 410 deletions(-) diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go index 3724aeddb6f..0bba069a61a 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go @@ -22,6 +22,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" ) const ( @@ -29,12 +30,6 @@ const ( DefaultAPIServerPort = 6443 // DefaultAPIServerPortString defines the API server port as a string for convenience. DefaultAPIServerPortString = "6443" - // ZoneTypeAvailabilityZone defines the regular AWS zones in the Region. - ZoneTypeAvailabilityZone = "availability-zone" - // ZoneTypeAvailabilityZone defines the AWS zone type in Local Zone infrastructure. - ZoneTypeLocalZone = "local-zone" - // ZoneTypeAvailabilityZone defines the AWS zone type in Wavelength infrastructure. - ZoneTypeWavelengthZone = "wavelength-zone" ) // NetworkStatus encapsulates AWS networking resources. @@ -434,34 +429,30 @@ type SubnetSpec struct { // Tags is a collection of tags describing the resource. Tags Tags `json:"tags,omitempty"` - // ZoneName defines a zone name for this subnet - // If you're already set AvailabilityZone, it will take precendence. - // - // The valid values are availability-zone, local-zone , and wavelength-zone. - ZoneName *string `json:"zoneName,omitempty"` - - // ZoneType defines a zone type for this subnet - // If you're already set AvailabilityZone, it will take precendence. + // ZoneType defines a zone type for this subnet. // - // The valid values are availability-zone, local-zone , and wavelength-zone. + // The valid values are availability-zone, local-zone, and wavelength-zone. // // Zone types local-zone or wavelength-zone is not selected to automatically create - // control plane or compute nodes. + // resources like: Nat Gateway, Network Load Balancers, control plane or compute nodes. // // When local-zone, the public subnets will be associated to the public route table, - // the private subnets will try to create a NAT Gateway, when supported, otherwise - // the subnet will be associated to the private route table in the region (preferred parent zone, zone - // type availability-zone in the region, or first available). + // the private subnets uses the zone in the region to create route tables re-using + // Nat Gateways (preferred from parent zone, the zone type availability-zone in the region, + // or first table available). // // When wavelength-zone, the public subnets will be associated to the carrier route table, // created altogether the Carrier Gateway when public subnets in AWS Wavelength Zone is defined. // The private subnets will try to create a NAT Gateway, when supported, otherwise // the subnet will be associated to the private route table in the region (preferred parent zone, zone // type availability-zone in the region, or first available). - ZoneType *string `json:"zoneType,omitempty"` + // + // +kubebuilder:validation:Enum=availability-zone;local-zone + // +optional + ZoneType *ZoneType `json:"zoneType,omitempty"` - // ParentZoneName defines a parent zone name for the zone that the subnet is created, - // when applied. Available only in zone types local-zone or wavelength-zone. + // ParentZoneName defines a parent zone name of the zone that the subnet is created. + // +optional ParentZoneName *string `json:"parentZoneName,omitempty"` } @@ -480,12 +471,16 @@ func (s *SubnetSpec) String() string { } // IsEdge returns the true when the subnet is created in the edge zone, -// Local Zone or Wavelength. +// Local Zones. func (s *SubnetSpec) IsEdge() bool { if s.ZoneType == nil { return false } - if *s.ZoneType == ZoneTypeLocalZone || *s.ZoneType == ZoneTypeWavelengthZone { + fmt.Println(s.AvailabilityZone, s.ZoneType.String()) + if s.ZoneType.Equal(ZoneTypeLocalZone) { + return true + } + if s.ZoneType.Equal(ZoneTypeWavelengthZone) { return true } return false @@ -502,6 +497,29 @@ func (s *SubnetSpec) IsEdgeWavelength() bool { return false } +// SetZoneInfo updates the subnets with zone information. +func (s *SubnetSpec) SetZoneInfo(zones []*ec2.AvailabilityZone) error { + zoneInfo := func(zoneName string) *ec2.AvailabilityZone { + for _, zone := range zones { + if aws.StringValue(zone.ZoneName) == zoneName { + return zone + } + } + return nil + } + zone := zoneInfo(s.AvailabilityZone) + if zone == nil { + return fmt.Errorf("unable to get zone information for zone %v", s.AvailabilityZone) + } + if zone.ZoneType != nil { + s.ZoneType = newZoneType(zone.ZoneType) + } + if zone.ParentZoneName != nil { + s.ParentZoneName = zone.ParentZoneName + } + return nil +} + // Subnets is a slice of Subnet. // +listType=map // +listMapKey=id @@ -521,8 +539,10 @@ func (s Subnets) ToMap() map[string]*SubnetSpec { func (s Subnets) IDs() []string { res := []string{} for _, subnet := range s { - // Do not return edge zones (Local Zone or Wavelength Zone) to regular Subnet IDs, - // to keep compatibility. Use IDsWithEdge() get the the full list of zones + // Prevent returning edge zones (Local Zone) to regular Subnet IDs. + // Edge zones should not deploy control plane nodes, and does not support Nat Gateway and + // Network Load Balancers. Any resource for the core infrastructure should not consume edge + // zones. if subnet.IsEdge() { continue } @@ -573,11 +593,9 @@ func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec { // FilterPrivate returns a slice containing all subnets marked as private. func (s Subnets) FilterPrivate() (res Subnets) { for _, x := range s { - if x.ZoneType != nil { - switch aws.StringValue(x.ZoneType) { - case ZoneTypeLocalZone, ZoneTypeWavelengthZone: - continue - } + // Subnets in AWS Local Zones should not be used by core infrastructure. + if x.IsEdge() { + continue } if !x.IsPublic { res = append(res, x) @@ -589,11 +607,9 @@ func (s Subnets) FilterPrivate() (res Subnets) { // FilterPublic returns a slice containing all subnets marked as public. func (s Subnets) FilterPublic() (res Subnets) { for _, x := range s { - if x.ZoneType != nil { - switch aws.StringValue(x.ZoneType) { - case ZoneTypeLocalZone, ZoneTypeWavelengthZone: - continue - } + // Subnets in AWS Local Zones should not be used by core infrastructure. + if x.IsEdge() { + continue } if x.IsPublic { res = append(res, x) @@ -617,7 +633,7 @@ func (s Subnets) GetUniqueZones() []string { keys := make(map[string]bool) zones := []string{} for _, x := range s { - if _, value := keys[x.AvailabilityZone]; !value { + if _, value := keys[x.AvailabilityZone]; len(x.AvailabilityZone) > 0 && !value { keys[x.AvailabilityZone] = true zones = append(zones, x.AvailabilityZone) } @@ -625,6 +641,17 @@ func (s Subnets) GetUniqueZones() []string { return zones } +// SetZoneInfo updates the subnets with zone information. +func (s Subnets) SetZoneInfo(zones []*ec2.AvailabilityZone) error { + for i := range s { + err := s[i].SetZoneInfo(zones) + if err != nil { + return err + } + } + return nil +} + // IsIPv6Enabled returns true if the IPv6 block is defined on the network spec. func (s Subnets) HasPublicSubnetWavelength() bool { for _, sub := range s { @@ -852,3 +879,37 @@ func (i *IngressRule) Equals(o *IngressRule) bool { return true } + +// ZoneType defines listener AWS Availability Zone type. +type ZoneType string + +// String returns the string representation for the zone type. +func (z ZoneType) String() string { + return string(z) +} + +// Equal compares two zone types. +func (z ZoneType) Equal(other ZoneType) bool { + return z == other +} + +// newZoneType returns the zone type based in the string value representing +// the zone name. Default zone type is availability-zone. +func newZoneType(ztype *string) *ZoneType { + switch ZoneType(*ztype) { + case ZoneTypeLocalZone: + return &ZoneTypeLocalZone + case ZoneTypeWavelengthZone: + return &ZoneTypeWavelengthZone + } + return &ZoneTypeAvailabilityZone +} + +var ( + // ZoneTypeAvailabilityZone defines the regular AWS zones in the Region. + ZoneTypeAvailabilityZone = ZoneType("availability-zone") + // ZoneTypeLocalZone defines the AWS zone type in Local Zone infrastructure. + ZoneTypeLocalZone = ZoneType("local-zone") + // ZoneTypeAvailabilityZone defines the AWS zone type in Wavelength infrastructure. + ZoneTypeWavelengthZone = ZoneType("wavelength-zone") +) diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go index 95bf30a917d..5899a557ecd 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go @@ -360,20 +360,20 @@ func (s *Service) findNatGatewayForEdgeSubnet(sn *infrav1.SubnetSpec) (string, e return gws[0], nil } - // if not, check if the parent zone public subnet has nat gateway + // Check if the parent zone public subnet has nat gateway if sn.ParentZoneName != nil { if gws, ok := azGateways[aws.StringValue(sn.ParentZoneName)]; ok && len(gws) > 0 { return gws[0], nil } } - // if not, get the first public subnet's nat gateway available + // Get the first public subnet's nat gateway available for zone, gws := range azGateways { if len(gws[0]) > 0 { - fmt.Printf("\n>> Assigning route table ID %s to zone %s from zone %s\n", gws[0], sn.AvailabilityZone, zone) + s.scope.Debug("Assigning route table", "table ID", gws[0], "source zone", zone, "target zone", sn.AvailabilityZone) return gws[0], nil } } - return "", errors.Errorf("no nat gateways available in %q for private subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways) + return "", errors.Errorf("no nat gateways available in %q for private edge subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways) } diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go index 397d1049f9a..9ec8eb82957 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go @@ -60,49 +60,10 @@ func (s *Service) reconcileRouteTables() error { for i := range subnets { sn := &subnets[i] // We need to compile the minimum routes for this subnet first, so we can compare it or create them. - var routes []*ec2.Route - // Gather routes to public subnets. - // Public subnets on Wavelength zone infrastructure requires Carrier Gateway to - // route intenet traffic from/to the carrier infrastructure. - if sn.IsPublic && !sn.IsEdgeWavelength() { - if s.scope.VPC().InternetGatewayID == nil { - return errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID) - } - routes = append(routes, s.getGatewayPublicRoute()) - if sn.IsIPv6 { - routes = append(routes, s.getGatewayPublicIPv6Route()) - } - } else if sn.IsPublic && sn.IsEdgeWavelength() { - if s.scope.VPC().CarrierGatewayID == nil { - return errors.Errorf("failed to create carrier routing table: carrier gateway for %q is nil", s.scope.VPC().ID) - } - routes = append(routes, s.getCarrierGatewayPublicIPv4Route()) - // TODO(mtulio): IS IPv6 supported in Carrier Gateway/Wavelength infra? - if sn.IsIPv6 { - routes = append(routes, s.getCarrierGatewayPublicIPv6Route()) - } - } else { - var natGatewayID string - // private subnets in the edge zones (Local or Wavelength zones) - if sn.IsEdge() { - natGatewayID, err = s.findNatGatewayForEdgeSubnet(sn) - } else { - natGatewayID, err = s.getNatGatewayForSubnet(sn) - } - if err != nil { - return err - } - - routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) - if sn.IsIPv6 { - if !s.scope.VPC().IsIPv6Enabled() { - // Safety net because EgressOnlyInternetGateway needs the ID from the ipv6 block. - // if, for whatever reason by this point that is not available, we don't want to - // panic because of a nil pointer access. This should never occur. Famous last words though. - return errors.Errorf("ipv6 block missing for ipv6 enabled subnet, can't create egress only internet gateway") - } - routes = append(routes, s.getEgressOnlyInternetGateway()) - } + routes, err := s.getRoutesForSubnet(sn) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedRouteTableRoutes", "Failed to get routes for managed RouteTable for subnet %s: %v", sn.ID, err) + return errors.Wrapf(err, "failed to discover routes on route table %s", sn.ID) } if rt, ok := subnetRouteMap[sn.GetResourceID()]; ok { @@ -164,7 +125,7 @@ func (s *Service) reconcileRouteTables() error { return nil } -func (s *Service) fixMismatchedRouting(specRoute *ec2.Route, currentRoute *ec2.Route, rt *ec2.RouteTable) error { +func (s *Service) fixMismatchedRouting(specRoute *ec2.CreateRouteInput, currentRoute *ec2.Route, rt *ec2.RouteTable) error { var input *ec2.ReplaceRouteInput if specRoute.DestinationCidrBlock != nil { if (currentRoute.DestinationCidrBlock != nil && @@ -233,6 +194,32 @@ func (s *Service) describeVpcRouteTablesBySubnet() (map[string]*ec2.RouteTable, return res, nil } +func (s *Service) deleteRouteTable(rt *ec2.RouteTable) error { + for _, as := range rt.Associations { + if as.SubnetId == nil { + continue + } + + if _, err := s.EC2Client.DisassociateRouteTableWithContext(context.TODO(), &ec2.DisassociateRouteTableInput{AssociationId: as.RouteTableAssociationId}); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDisassociateRouteTable", "Failed to disassociate managed RouteTable %q from Subnet %q: %v", *rt.RouteTableId, *as.SubnetId, err) + return errors.Wrapf(err, "failed to disassociate route table %q from subnet %q", *rt.RouteTableId, *as.SubnetId) + } + + record.Eventf(s.scope.InfraCluster(), "SuccessfulDisassociateRouteTable", "Disassociated managed RouteTable %q from subnet %q", *rt.RouteTableId, *as.SubnetId) + s.scope.Debug("Deleted association between route table and subnet", "route-table-id", *rt.RouteTableId, "subnet-id", *as.SubnetId) + } + + if _, err := s.EC2Client.DeleteRouteTableWithContext(context.TODO(), &ec2.DeleteRouteTableInput{RouteTableId: rt.RouteTableId}); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", *rt.RouteTableId, err) + return errors.Wrapf(err, "failed to delete route table %q", *rt.RouteTableId) + } + + record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteRouteTable", "Deleted managed RouteTable %q", *rt.RouteTableId) + s.scope.Info("Deleted route table", "route-table-id", *rt.RouteTableId) + + return nil +} + func (s *Service) deleteRouteTables() error { if s.scope.VPC().IsUnmanaged(s.scope.Name()) { s.scope.Trace("Skipping routing tables deletion in unmanaged mode") @@ -245,27 +232,10 @@ func (s *Service) deleteRouteTables() error { } for _, rt := range rts { - for _, as := range rt.Associations { - if as.SubnetId == nil { - continue - } - - if _, err := s.EC2Client.DisassociateRouteTableWithContext(context.TODO(), &ec2.DisassociateRouteTableInput{AssociationId: as.RouteTableAssociationId}); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedDisassociateRouteTable", "Failed to disassociate managed RouteTable %q from Subnet %q: %v", *rt.RouteTableId, *as.SubnetId, err) - return errors.Wrapf(err, "failed to disassociate route table %q from subnet %q", *rt.RouteTableId, *as.SubnetId) - } - - record.Eventf(s.scope.InfraCluster(), "SuccessfulDisassociateRouteTable", "Disassociated managed RouteTable %q from subnet %q", *rt.RouteTableId, *as.SubnetId) - s.scope.Debug("Deleted association between route table and subnet", "route-table-id", *rt.RouteTableId, "subnet-id", *as.SubnetId) - } - - if _, err := s.EC2Client.DeleteRouteTableWithContext(context.TODO(), &ec2.DeleteRouteTableInput{RouteTableId: rt.RouteTableId}); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", *rt.RouteTableId, err) - return errors.Wrapf(err, "failed to delete route table %q", *rt.RouteTableId) + err := s.deleteRouteTable(rt) + if err != nil { + return err } - - record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteRouteTable", "Deleted managed RouteTable %q", *rt.RouteTableId) - s.scope.Info("Deleted route table", "route-table-id", *rt.RouteTableId) } return nil } @@ -290,7 +260,7 @@ func (s *Service) describeVpcRouteTables() ([]*ec2.RouteTable, error) { return out.RouteTables, nil } -func (s *Service) createRouteTableWithRoutes(routes []*ec2.Route, isPublic bool, zone string) (*infrav1.RouteTable, error) { +func (s *Service) createRouteTableWithRoutes(routes []*ec2.CreateRouteInput, isPublic bool, zone string) (*infrav1.RouteTable, error) { out, err := s.EC2Client.CreateRouteTableWithContext(context.TODO(), &ec2.CreateRouteTableInput{ VpcId: aws.String(s.scope.VPC().ID), TagSpecifications: []*ec2.TagSpecification{ @@ -306,23 +276,17 @@ func (s *Service) createRouteTableWithRoutes(routes []*ec2.Route, isPublic bool, for i := range routes { route := routes[i] if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - if _, err := s.EC2Client.CreateRouteWithContext(context.TODO(), &ec2.CreateRouteInput{ - RouteTableId: out.RouteTable.RouteTableId, - DestinationCidrBlock: route.DestinationCidrBlock, - DestinationIpv6CidrBlock: route.DestinationIpv6CidrBlock, - EgressOnlyInternetGatewayId: route.EgressOnlyInternetGatewayId, - GatewayId: route.GatewayId, - InstanceId: route.InstanceId, - NatGatewayId: route.NatGatewayId, - NetworkInterfaceId: route.NetworkInterfaceId, - VpcPeeringConnectionId: route.VpcPeeringConnectionId, - }); err != nil { + route.RouteTableId = out.RouteTable.RouteTableId + if _, err := s.EC2Client.CreateRouteWithContext(context.TODO(), route); err != nil { return false, err } return true, nil }, awserrors.RouteTableNotFound, awserrors.NATGatewayNotFound, awserrors.GatewayNotFound); err != nil { - // TODO(vincepri): cleanup the route table if this fails. record.Warnf(s.scope.InfraCluster(), "FailedCreateRoute", "Failed to create route %s for RouteTable %q: %v", route.GoString(), *out.RouteTable.RouteTableId, err) + errDel := s.deleteRouteTable(out.RouteTable) + if errDel != nil { + s.scope.Trace("Failed to delete RouteTable %q: %v", *out.RouteTable.RouteTableId, errDel) + } return nil, errors.Wrapf(err, "failed to create route in route table %q: %s", *out.RouteTable.RouteTableId, route.GoString()) } record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateRoute", "Created route %s for RouteTable %q", route.GoString(), *out.RouteTable.RouteTableId) @@ -348,43 +312,43 @@ func (s *Service) associateRouteTable(rt *infrav1.RouteTable, subnetID string) e return nil } -func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.Route { - return &ec2.Route{ +func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ NatGatewayId: aws.String(natGatewayID), DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), } } -func (s *Service) getEgressOnlyInternetGateway() *ec2.Route { - return &ec2.Route{ +func (s *Service) getEgressOnlyInternetGateway() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), EgressOnlyInternetGatewayId: s.scope.VPC().IPv6.EgressOnlyInternetGatewayID, } } -func (s *Service) getGatewayPublicRoute() *ec2.Route { - return &ec2.Route{ +func (s *Service) getGatewayPublicRoute() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), GatewayId: aws.String(*s.scope.VPC().InternetGatewayID), } } -func (s *Service) getGatewayPublicIPv6Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getGatewayPublicIPv6Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), GatewayId: aws.String(*s.scope.VPC().InternetGatewayID), } } -func (s *Service) getCarrierGatewayPublicIPv4Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getCarrierGatewayPublicIPv4Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), CarrierGatewayId: aws.String(*s.scope.VPC().CarrierGatewayID), } } -func (s *Service) getCarrierGatewayPublicIPv6Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getCarrierGatewayPublicIPv6Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv6CidrBlock), CarrierGatewayId: aws.String(*s.scope.VPC().CarrierGatewayID), } @@ -415,3 +379,67 @@ func (s *Service) getRouteTableTagParams(id string, public bool, zone string) in Additional: additionalTags, } } + +func (s *Service) getRoutesToPublicSubnet(sn *infrav1.SubnetSpec) ([]*ec2.CreateRouteInput, error) { + var routes []*ec2.CreateRouteInput + + if sn.IsEdgeWavelength() { + if s.scope.VPC().CarrierGatewayID == nil { + return routes, errors.Errorf("failed to create carrier routing table: carrier gateway for %q is nil", s.scope.VPC().ID) + } + routes = append(routes, s.getCarrierGatewayPublicIPv4Route()) + // TODO(mtulio): IS IPv6 supported in Carrier Gateway/Wavelength infra? + if sn.IsIPv6 { + routes = append(routes, s.getCarrierGatewayPublicIPv6Route()) + } + return routes, nil + } + + if s.scope.VPC().InternetGatewayID == nil { + return routes, errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID) + } + routes = append(routes, s.getGatewayPublicRoute()) + if sn.IsIPv6 { + routes = append(routes, s.getGatewayPublicIPv6Route()) + } + + return routes, nil +} + +func (s *Service) getRoutesToPrivateSubnet(sn *infrav1.SubnetSpec) (routes []*ec2.CreateRouteInput, err error) { + // var routes []*ec2.Route + var natGatewayID string + + // NAT gateways in edge zones (Local Zones) are not globally supported, + // private subnets in those locations uses Nat Gateways from the + // Parent Zone or, when not available, the first zone in the Region. + if sn.IsEdge() { + natGatewayID, err = s.findNatGatewayForEdgeSubnet(sn) + } else { + natGatewayID, err = s.getNatGatewayForSubnet(sn) + } + + if err != nil { + return routes, err + } + + routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) + if sn.IsIPv6 { + if !s.scope.VPC().IsIPv6Enabled() { + // Safety net because EgressOnlyInternetGateway needs the ID from the ipv6 block. + // if, for whatever reason by this point that is not available, we don't want to + // panic because of a nil pointer access. This should never occur. Famous last words though. + return routes, errors.Errorf("ipv6 block missing for ipv6 enabled subnet, can't create egress only internet gateway") + } + routes = append(routes, s.getEgressOnlyInternetGateway()) + } + + return routes, nil +} + +func (s *Service) getRoutesForSubnet(sn *infrav1.SubnetSpec) ([]*ec2.CreateRouteInput, error) { + if sn.IsPublic { + return s.getRoutesToPublicSubnet(sn) + } + return s.getRoutesToPrivateSubnet(sn) +} diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go index 7133bd0a1aa..8654139d372 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go @@ -136,7 +136,7 @@ func (s *Service) reconcileSubnets() error { subnetTags := sub.Tags // Make sure tags are up-to-date. if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags) + buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags, existingSubnet.IsEdge()) tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client)) if err := tagsBuilder.Ensure(existingSubnet.Tags); err != nil { return false, err @@ -151,7 +151,10 @@ func (s *Service) reconcileSubnets() error { // We may not have a permission to tag unmanaged subnets. // When tagging unmanaged subnet fails, record an event and proceed. record.Warnf(s.scope.InfraCluster(), "FailedTagSubnet", "Failed tagging unmanaged Subnet %q: %v", existingSubnet.GetResourceID(), err) - break + // Review Bug(mtulio): if proceed, out of the loop, here will create subnets without AvailabilityZone, after the future + // iterations, making unexpected situations when the zone like unmanaged subnets in different zones. + // [1] "Unmanaged VPC, 2 existing matching subnets, subnet tagging fails, should succeed" + continue } // TODO(vincepri): check if subnet needs to be updated. @@ -177,43 +180,13 @@ func (s *Service) reconcileSubnets() error { return errors.New("expected at least 1 subnet but got 0") } - // Reconciling the zone attributes for the subnet - zoneNames := make([]string, 0, len(subnets)) - for i := range subnets { - if subnets[i].ZoneType == nil { - zoneNames = append(zoneNames, subnets[i].AvailabilityZone) - } + // Reconciling the zone information for the subnets. Subnets are grouped + // by regular zones (availability zones) or edge zones (local zones or wavelength zones), + // based in the zone-type attribute for zone + if err := s.reconcileZoneInfo(subnets); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedNoZoneInfo", "Expected the zone attributes to be populated to subnet") + return errors.Wrapf(err, "expected the zone attributes to be populated to subnet") } - zonesMap := make(map[string]*ec2.AvailabilityZone, len(zoneNames)) - - // s.scope.Info("\n>> Reconciling subnets> ZONES : %v", zoneNames) - fmt.Printf("\n\n >> Reconciling subnets> len(zoneNames) = %d \n\n", len(zoneNames)) - if len(zoneNames) > 0 { - out, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ - ZoneNames: aws.StringSlice(zoneNames), - }) - if err != nil { - record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) - return errors.Wrap(err, "failed to describe availability zones") - } - - for _, zone := range out.AvailabilityZones { - if _, ok := zonesMap[aws.StringValue(zone.ZoneName)]; !ok { - zonesMap[aws.StringValue(zone.ZoneName)] = zone - } - } - for i := range subnets { - sub := &subnets[i] - //if sub.ZoneType != nil { - // continue - //} - // sync required fields - sub.ZoneType = zonesMap[sub.AvailabilityZone].ZoneType - sub.ZoneName = zonesMap[sub.AvailabilityZone].ZoneName - sub.ParentZoneName = zonesMap[sub.AvailabilityZone].ParentZoneName - } - } - fmt.Printf("\n\n>> Reconciling subnets> zonesMap : %v \n\n", zonesMap) // When the VPC is managed by CAPA, we need to create the subnets. if !unmanagedVPC { @@ -249,6 +222,42 @@ func (s *Service) reconcileSubnets() error { return nil } +func (s *Service) retrieveZoneInfo(zoneNames []string) ([]*ec2.AvailabilityZone, error) { + zones, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice(zoneNames), + }) + if err != nil { + record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) + return nil, errors.Wrap(err, "failed to describe availability zones") + } + + return zones.AvailabilityZones, nil +} + +// reconcileZoneInfo discover the zones for all subnets, and retrieve +// persist the zone information from resource API, such as Type and +// Parent Zone. +func (s *Service) reconcileZoneInfo(subnets infrav1.Subnets) error { + if len(subnets) > 0 { + // zones, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + // ZoneNames: aws.StringSlice(subnets.GetUniqueZones()), + // }) + // if err != nil { + // record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) + // return errors.Wrap(err, "failed to describe availability zones") + // } + zones, err := s.retrieveZoneInfo(subnets.GetUniqueZones()) + if err != nil { + return err + } + // Extract zone attributes from resource API for each subnet. + if err := subnets.SetZoneInfo(zones); err != nil { + return err + } + } + return nil +} + func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { zones, err := s.getAvailableZones() if err != nil { @@ -466,7 +475,7 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err TagSpecifications: []*ec2.TagSpecification{ tags.BuildParamsToTagSpecification( ec2.ResourceTypeSubnet, - s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags), + s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags, sn.IsEdge()), ), }, } @@ -488,9 +497,19 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err return nil, errors.Wrapf(err, "failed to wait for subnet %q", *out.Subnet.SubnetId) } + // Retrieve zone information used later to change the zone attributes. + zones, err := s.retrieveZoneInfo([]string{sn.AvailabilityZone}) + if err != nil { + return nil, errors.Wrapf(err, "failed to discover zone information for subnet %q", *out.Subnet.SubnetId) + } + err = sn.SetZoneInfo(zones) + if err != nil { + return nil, errors.Wrapf(err, "failed to update zone information for subnet %q", *out.Subnet.SubnetId) + } + // This has to be done separately, because: // InvalidParameterCombination: Only one subnet attribute can be modified at a time - if sn.IsIPv6 { + if sn.IsIPv6 && !sn.IsEdge() { // regardless of the subnet being public or not, ipv6 address needs to be assigned // on creation. There is no such thing as private ipv6 address. if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { @@ -510,7 +529,7 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) } - if sn.IsPublic { + if sn.IsPublic && !sn.IsEdge() { if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { if _, err := s.EC2Client.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ SubnetId: out.Subnet.SubnetId, @@ -584,7 +603,7 @@ func (s *Service) deleteSubnet(id string) error { return nil } -func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags) infrav1.BuildParams { +func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags, isEdge bool) infrav1.BuildParams { var role string additionalTags := make(map[string]string) @@ -593,10 +612,15 @@ func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, if public { role = infrav1.PublicRoleTagValue - additionalTags[externalLoadBalancerTag] = "1" + // Edge subnets should not have ELB tags to be selected by CCM to create load balancers. + if !isEdge { + additionalTags[externalLoadBalancerTag] = "1" + } } else { role = infrav1.PrivateRoleTagValue - additionalTags[internalLoadBalancerTag] = "1" + if !isEdge { + additionalTags[internalLoadBalancerTag] = "1" + } } // Add tag needed for Service type=LoadBalancer diff --git a/pkg/asset/manifests/aws/cluster.go b/pkg/asset/manifests/aws/cluster.go index 8c5cbf06bbc..68f39916208 100644 --- a/pkg/asset/manifests/aws/cluster.go +++ b/pkg/asset/manifests/aws/cluster.go @@ -1,10 +1,10 @@ package aws import ( + "context" "fmt" "time" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -25,7 +25,6 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco if err != nil { return nil, fmt.Errorf("failed to get user tags: %w", err) } - mainCIDR := capiutils.CIDRFromInstallConfig(ic) awsCluster := &capa.AWSCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -163,13 +162,14 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco Protocol: capa.SecurityGroupProtocolTCP, FromPort: 22623, ToPort: 22623, - CidrBlocks: []string{mainCIDR.String()}, + CidrBlocks: []string{capiutils.CIDRFromInstallConfig(ic).String()}, }, }, }, AdditionalTags: tags, }, } + awsCluster.SetGroupVersionKind(capa.GroupVersion.WithKind("AWSCluster")) if ic.Config.Publish == types.ExternalPublishingStrategy { // FIXME: CAPA bug. Remove when fixed upstream @@ -206,17 +206,15 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco } } - // Set the VPC and zones (managed) or subnets (BYO VPC) based in the - // install-config.yaml. - err = setZones(&zoneConfigInput{ + // Set the NetworkSpec.Subnets from VPC and zones (managed) + // or subnets (BYO VPC) based in the install-config.yaml. + err = setSubnets(context.TODO(), &zonesInput{ InstallConfig: ic, - Config: ic.Config, - Meta: ic.AWS, ClusterID: clusterID, Cluster: awsCluster, }) if err != nil { - return nil, errors.Wrap(err, "failed to set cluster zones or subnets") + return nil, fmt.Errorf("failed to set cluster zones or subnets: %w", err) } manifests = append(manifests, &asset.RuntimeFile{ @@ -235,6 +233,7 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco }, }, } + id.SetGroupVersionKind(capa.GroupVersion.WithKind("AWSClusterControllerIdentity")) manifests = append(manifests, &asset.RuntimeFile{ Object: id, File: asset.File{Filename: "01_aws-cluster-controller-identity-default.yaml"}, @@ -243,7 +242,7 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, InfrastructureRef: &corev1.ObjectReference{ - APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + APIVersion: capa.GroupVersion.String(), Kind: "AWSCluster", Name: awsCluster.Name, Namespace: awsCluster.Namespace, diff --git a/pkg/asset/manifests/aws/zones.go b/pkg/asset/manifests/aws/zones.go index 8a5d1d3e64c..335045bb22f 100644 --- a/pkg/asset/manifests/aws/zones.go +++ b/pkg/asset/manifests/aws/zones.go @@ -5,43 +5,154 @@ import ( "fmt" "net" + "k8s.io/apimachinery/pkg/util/sets" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/installconfig/aws" "github.com/openshift/installer/pkg/asset/manifests/capiutils" utilscidr "github.com/openshift/installer/pkg/asset/manifests/capiutils/cidr" - "github.com/pkg/errors" - "k8s.io/utils/ptr" - capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - "github.com/openshift/installer/pkg/types" ) -type zoneConfigInput struct { +type subnetsInput struct { + vpc string + privateSubnets aws.Subnets + publicSubnets aws.Subnets + edgeSubnets aws.Subnets +} + +type zonesInput struct { InstallConfig *installconfig.InstallConfig - Config *types.InstallConfig - Meta *aws.Metadata Cluster *capa.AWSCluster ClusterID *installconfig.ClusterID + ZonesInRegion []string + Subnets *subnetsInput } -// setZones creates the CAPI NetworkSpec structures for managed or -// BYO VPC deployments from install-config.yaml. -func setZones(in *zoneConfigInput) error { - if len(in.Config.AWS.Subnets) > 0 { - return setZonesBYOVPC(in) - } else { - return setZonesManagedVPC(in) +// GatherZonesFromMetadata retrieves zones from AWS API to be used +// when building the subnets to CAPA. +func (zin *zonesInput) GatherZonesFromMetadata(ctx context.Context) (err error) { + zin.ZonesInRegion, err = zin.InstallConfig.AWS.AvailabilityZones(ctx) + if err != nil { + return fmt.Errorf("failed to get availability zones: %w", err) + } + return nil +} + +// GatherSubnetsFromMetadata retrieves subnets from AWS API to be used +// when building the subnets to CAPA. +func (zin *zonesInput) GatherSubnetsFromMetadata(ctx context.Context) (err error) { + zin.Subnets = &subnetsInput{} + if zin.Subnets.privateSubnets, err = zin.InstallConfig.AWS.PrivateSubnets(ctx); err != nil { + return fmt.Errorf("failed to get private subnets: %w", err) + } + if zin.Subnets.publicSubnets, err = zin.InstallConfig.AWS.PublicSubnets(ctx); err != nil { + return fmt.Errorf("failed to get public subnets: %w", err) } + if zin.Subnets.edgeSubnets, err = zin.InstallConfig.AWS.EdgeSubnets(ctx); err != nil { + return fmt.Errorf("failed to get public subnets: %w", err) + } + if zin.Subnets.vpc, err = zin.InstallConfig.AWS.VPC(ctx); err != nil { + return fmt.Errorf("failed to get VPC: %w", err) + } + return nil +} + +type zonesCAPI struct { + controlPlaneZones sets.Set[string] + computeZones sets.Set[string] + edgeZones sets.Set[string] +} + +// AvailabilityZones returns a sorted union of Availability Zones defined +// in the zone attribute in the pools for control plane and compute. +func (zo *zonesCAPI) AvailabilityZones() []string { + return sets.List(zo.controlPlaneZones.Union(zo.computeZones)) +} + +// EdgeZones returns a sorted union of Local Zones or Wavelength Zones +// defined in the zone attribute in the edge compute pool. +func (zo *zonesCAPI) EdgeZones() []string { + return sets.List(zo.edgeZones) } -// setZonesManagedVPC creates the CAPI NetworkSpec.Subnets setting the +// SetAvailabilityZones insert the zone to the given compute pool, and to +// the regular zone (zone type availability-zone) list. +func (zo *zonesCAPI) SetAvailabilityZones(pool string, zones []string) { + switch pool { + case types.MachinePoolControlPlaneRoleName: + zo.controlPlaneZones.Insert(zones...) + + case types.MachinePoolComputeRoleName: + zo.computeZones.Insert(zones...) + } +} + +// SetDefaultConfigZones evaluates if machine pools (control plane and workers) have been +// set the zones from install-config.yaml, if not sets the default from platform, when exists, +// otherwise set the default from the region discovered from AWS API. +func (zo *zonesCAPI) SetDefaultConfigZones(pool string, defConfig []string, defRegion []string) { + zones := []string{} + switch pool { + case types.MachinePoolControlPlaneRoleName: + if len(zo.controlPlaneZones) == 0 && len(defConfig) > 0 { + zones = defConfig + } else if len(zo.controlPlaneZones) == 0 { + zones = defRegion + } + zo.controlPlaneZones.Insert(zones...) + + case types.MachinePoolComputeRoleName: + if len(zo.computeZones) == 0 && len(defConfig) > 0 { + zones = defConfig + } else if len(zo.computeZones) == 0 { + zones = defRegion + } + zo.computeZones.Insert(zones...) + } +} + +// setSubnets is the entrypoint to create the CAPI NetworkSpec structures +// for managed or BYO VPC deployments from install-config.yaml. +// The NetworkSpec.Subnets will be populated with the desired zones. +func setSubnets(ctx context.Context, in *zonesInput) error { + if in.InstallConfig == nil { + return fmt.Errorf("failed to get installConfig") + } + if in.InstallConfig.AWS == nil { + return fmt.Errorf("failed to get AWS metadata") + } + if in.InstallConfig.Config == nil { + return fmt.Errorf("unable to get Config") + } + if in.Cluster == nil { + return fmt.Errorf("failed to get AWSCluster config") + } + if len(in.InstallConfig.Config.AWS.Subnets) > 0 { + if err := in.GatherSubnetsFromMetadata(ctx); err != nil { + return fmt.Errorf("failed to get subnets from metadata: %w", err) + } + return setSubnetsBYOVPC(in) + } + + if err := in.GatherZonesFromMetadata(ctx); err != nil { + return fmt.Errorf("failed to get availability zones from metadata: %w", err) + } + return setSubnetsManagedVPC(in) +} + +// setSubnetsBYOVPC creates the CAPI NetworkSpec.Subnets setting the // desired subnets from install-config.yaml in the BYO VPC deployment. -func setZonesBYOVPC(in *zoneConfigInput) error { - privateSubnets, err := in.Meta.PrivateSubnets(context.TODO()) - if err != nil { - return errors.Wrap(err, "failed to get private subnets") +// This function does not have support for unit test to mock for AWS API, +// so all API calls must be done prior this execution. +// TODO: create support to mock AWS API calls in the unit tests, so we can merge +// the methods GatherSubnetsFromMetadata() into this. +func setSubnetsBYOVPC(in *zonesInput) error { + in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{ + ID: in.Subnets.vpc, } - for _, subnet := range privateSubnets { + for _, subnet := range in.Subnets.privateSubnets { in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ ID: subnet.ID, CidrBlock: subnet.CIDR, @@ -50,11 +161,7 @@ func setZonesBYOVPC(in *zoneConfigInput) error { }) } - publicSubnets, err := in.Meta.PublicSubnets(context.TODO()) - if err != nil { - return errors.Wrap(err, "failed to get public subnets") - } - for _, subnet := range publicSubnets { + for _, subnet := range in.Subnets.publicSubnets { in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ ID: subnet.ID, CidrBlock: subnet.CIDR, @@ -63,142 +170,185 @@ func setZonesBYOVPC(in *zoneConfigInput) error { }) } - vpc, err := in.Meta.VPC(context.TODO()) - if err != nil { - return errors.Wrap(err, "failed to get VPC") - } - in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{ - ID: vpc, + // edgeSubnets are subnet created on AWS Local Zones or Wavelength Zone, + // discovered by ID and zone-type attribute. + for _, subnet := range in.Subnets.edgeSubnets { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: subnet.ID, + CidrBlock: subnet.CIDR, + AvailabilityZone: subnet.Zone.Name, + IsPublic: subnet.Public, + }) } return nil } -// setZonesManagedVPC creates the CAPI NetworkSpec.VPC setting the -// desired zones from install-config.yaml in the managed VPC deployment. -func setZonesManagedVPC(in *zoneConfigInput) error { - - zones, err := extractZonesFromInstallConfig(in) +// setSubnetsManagedVPC creates the CAPI NetworkSpec.VPC and the NetworkSpec.Subnets, +// setting the desired zones from install-config.yaml in the managed +// VPC deployment, when specified, otherwise default zones are set from +// the previously discovered from AWS API. +// This function does not have mock for AWS API, so all API calls must be done prior +// this execution. +// TODO: create support to mock AWS API calls in the unit tests, so we can merge +// the methods GatherZonesFromMetadata() into this. +// The CIDR blocks are calculated leaving free blocks to allow future expansions, +// in Day-2, when desired. +func setSubnetsManagedVPC(in *zonesInput) error { + out, err := extractZonesFromInstallConfig(in) if err != nil { - return errors.Wrap(err, "failed to get availability zones") + return fmt.Errorf("failed to get availability zones: %w", err) } - mainCIDR := capiutils.CIDRFromInstallConfig(in.InstallConfig) - - // Fallback to available zones in the region. - if len(zones) == 0 { - // Q? Do we need to use standard query or leave CAPA choose the zones automatically? - // zonesMeta, err := in.Config.AWS.AvailabilityZones(context.TODO()) - // if err != nil { - // return errors.Wrap(err, "failed to get availability zones") - // } - // for _, zoneMeta := range zonesMeta { - // zones = append(zones, &aws.Zone{Name: zoneMeta}) - // } - - // Leaving CAPA to discover zones - in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{ - CidrBlock: mainCIDR.String(), - AvailabilityZoneUsageLimit: ptr.To(len(zones)), - AvailabilityZoneSelection: &capa.AZSelectionSchemeOrdered, - } - return nil - } + isPublishingExternal := in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy + allAvailabilityZones := out.AvailabilityZones() + allEdgeZones := out.EdgeZones() + mainCIDR := capiutils.CIDRFromInstallConfig(in.InstallConfig) in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{ CidrBlock: mainCIDR.String(), } - // Base subnets considering only private zones, leaving one block free to allow + // Base subnets considering only private zones, leaving one free block to allow // future subnet expansions in Day-2. - numSubnets := len(zones) + 1 + numSubnets := len(allAvailabilityZones) + 1 // Public subnets consumes one range from base blocks. - isPublishingExternal := in.Config.Publish == types.ExternalPublishingStrategy if isPublishingExternal { numSubnets++ } - subnetsCIDRs, err := utilscidr.SplitIntoSubnetsIPv4(mainCIDR.String(), numSubnets) + // Edge subnets consumes one CIDR block from private CIDR, slicing it + // into smaller depending on the amount edge zones added to install config. + if len(allEdgeZones) > 0 { + numSubnets++ + } + + privateCIDRs, err := utilscidr.SplitIntoSubnetsIPv4(mainCIDR.String(), numSubnets) if err != nil { - return errors.Wrap(err, "unable to retrieve CIDR blocks for all private subnets") + return fmt.Errorf("unable to generate CIDR blocks for all private subnets: %w", err) + } + + publicCIDR := privateCIDRs[len(allAvailabilityZones)].String() + var edgeCIDR string + if len(allEdgeZones) > 0 { + publicCIDR = privateCIDRs[len(allAvailabilityZones)].String() + edgeCIDR = privateCIDRs[len(allAvailabilityZones)+1].String() } - var publicSubnetsCIDRs []*net.IPNet + + var publicCIDRs []*net.IPNet if isPublishingExternal { - publicSubnetsCIDRs, err = utilscidr.SplitIntoSubnetsIPv4(subnetsCIDRs[len(zones)].String(), len(zones)) + // The last num(zones) blocks are dedicated to the public subnets. + publicCIDRs, err = utilscidr.SplitIntoSubnetsIPv4(publicCIDR, len(allAvailabilityZones)) if err != nil { - return errors.Wrap(err, "unable to retrieve CIDR blocks for all public subnets") + return fmt.Errorf("unable to generate CIDR blocks for all public subnets: %w", err) } } - idxCIDR := 0 - // Q: Can we use the standard terraform name (without 'subnet') and tell CAPA - // to query it for Control Planes? - subnetNamePrefix := fmt.Sprintf("%s-subnet", in.ClusterID.InfraID) - for _, zone := range zones { - if len(subnetsCIDRs) < idxCIDR { - return errors.Wrap(err, "unable to define CIDR blocks for all private subnets") - } - cidr := subnetsCIDRs[idxCIDR] + // Create subnets from zone pools (control plane and compute) with type availability-zone. + if len(privateCIDRs) < len(allAvailabilityZones) { + return fmt.Errorf("unable to define CIDR blocks to all zones for private subnets") + } + if isPublishingExternal && len(publicCIDRs) < len(allAvailabilityZones) { + return fmt.Errorf("unable to define CIDR blocks to all zones for public subnets") + } + + for idxCIDR, zone := range allAvailabilityZones { in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - AvailabilityZone: zone.Name, - CidrBlock: cidr.String(), - ID: fmt.Sprintf("%s-private-%s", subnetNamePrefix, zone.Name), + AvailabilityZone: zone, + CidrBlock: privateCIDRs[idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-private-%s", in.ClusterID.InfraID, zone), IsPublic: false, }) if isPublishingExternal { - if len(publicSubnetsCIDRs) < idxCIDR { - return errors.Wrap(err, "unable to define CIDR blocks for all public subnets") - } - cidr = publicSubnetsCIDRs[idxCIDR] in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - AvailabilityZone: zone.Name, - CidrBlock: cidr.String(), - ID: fmt.Sprintf("%s-public-%s", subnetNamePrefix, zone.Name), + AvailabilityZone: zone, + CidrBlock: publicCIDRs[idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-public-%s", in.ClusterID.InfraID, zone), IsPublic: true, }) } - idxCIDR++ + } + + // Create subnets from edge zone pool with type local-zone. + if len(allEdgeZones) > 0 { + // Slice the main CIDR (edgeCIDR) into N*zones for privates subnets, + // and, when publish external, duplicate to create public subnets. + numEdgeSubnets := len(allEdgeZones) + if isPublishingExternal { + numEdgeSubnets = numEdgeSubnets * 2 + } + + // Allow one CIDR block for future expansion. + numEdgeSubnets++ + + // Slice the edgeCIDR into the amount of desired subnets. + edgeCIDRs, err := utilscidr.SplitIntoSubnetsIPv4(edgeCIDR, numEdgeSubnets) + if err != nil { + return fmt.Errorf("unable to generate CIDR blocks for all edge subnets: %w", err) + } + if len(edgeCIDRs) < len(allEdgeZones) { + return fmt.Errorf("unable to define CIDR blocks to all edge zones for private subnets") + } + if isPublishingExternal && (len(edgeCIDRs) < (len(allEdgeZones) * 2)) { + return fmt.Errorf("unable to define CIDR blocks to all edge zones for public subnets") + } + + // Create subnets from zone pool with type local-zone or wavelength-zone (edge zones) + for idxCIDR, zone := range allEdgeZones { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + AvailabilityZone: zone, + CidrBlock: edgeCIDRs[idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-private-%s", in.ClusterID.InfraID, zone), + IsPublic: false, + }) + if isPublishingExternal { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + AvailabilityZone: zone, + CidrBlock: edgeCIDRs[len(allEdgeZones)+idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-public-%s", in.ClusterID.InfraID, zone), + IsPublic: true, + }) + } + } } return nil } -// extractZonesFromInstallConfig extract all zones defined in the install-config, -// otherwise discover it based in the AWS metadata when none is defined. -// TODO: Open Question: What is the expected behavior when only one pool defines the -// zones? Should the cluster be limited to those zones? Eg when worker defines single -// zone, and no controlPlane.platform.aws.zones is defined. -func extractZonesFromInstallConfig(in *zoneConfigInput) ([]*aws.Zone, error) { - - var zones []*aws.Zone - zonesMap := make(map[string]struct{}) +// extractZonesFromInstallConfig extracts zones defined in the install-config. +func extractZonesFromInstallConfig(in *zonesInput) (*zonesCAPI, error) { + out := zonesCAPI{ + controlPlaneZones: sets.New[string](), + computeZones: sets.New[string](), + edgeZones: sets.New[string](), + } - if in.Config == nil { - return nil, errors.New("unable to retreive Config") + cfg := in.InstallConfig.Config + defaultZones := []string{} + if cfg.AWS != nil && cfg.AWS.DefaultMachinePlatform != nil && len(cfg.AWS.DefaultMachinePlatform.Zones) > 0 { + defaultZones = cfg.AWS.DefaultMachinePlatform.Zones } - cfg := in.Config - if cfg.ControlPlane != nil && cfg.ControlPlane.Platform.AWS != nil && - len(cfg.ControlPlane.Platform.AWS.Zones) > 0 { - for _, zone := range cfg.ControlPlane.Platform.AWS.Zones { - if _, ok := zonesMap[zone]; !ok { - zonesMap[zone] = struct{}{} - zones = append(zones, &aws.Zone{Name: zone}) - } - } + if cfg.ControlPlane != nil && cfg.ControlPlane.Platform.AWS != nil { + out.SetAvailabilityZones(types.MachinePoolControlPlaneRoleName, cfg.ControlPlane.Platform.AWS.Zones) } + out.SetDefaultConfigZones(types.MachinePoolControlPlaneRoleName, defaultZones, in.ZonesInRegion) - for _, compute := range cfg.Compute { - if len(compute.Platform.AWS.Zones) > 0 { - for _, zone := range compute.Platform.AWS.Zones { - if _, ok := zonesMap[zone]; !ok { - zonesMap[zone] = struct{}{} - zones = append(zones, &aws.Zone{Name: zone}) - } - } + for _, pool := range cfg.Compute { + if pool.Platform.AWS == nil { + continue + } + if len(pool.Platform.AWS.Zones) > 0 { + out.SetAvailabilityZones(pool.Name, pool.Platform.AWS.Zones) } + // Ignoring as edge pool is not yet supported by CAPA. + // See https://github.com/openshift/installer/pull/8173 + if pool.Name == types.MachinePoolEdgeRoleName { + out.edgeZones.Insert(pool.Platform.AWS.Zones...) + continue + } + out.SetDefaultConfigZones(types.MachinePoolComputeRoleName, defaultZones, in.ZonesInRegion) } - - return zones, nil + return &out, nil } diff --git a/pkg/asset/manifests/aws/zones_test.go b/pkg/asset/manifests/aws/zones_test.go index 49a3cce2b6e..beaae661833 100644 --- a/pkg/asset/manifests/aws/zones_test.go +++ b/pkg/asset/manifests/aws/zones_test.go @@ -1,19 +1,21 @@ package aws import ( - "fmt" "reflect" + "sort" "testing" - "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/installconfig/aws" "github.com/openshift/installer/pkg/asset/manifests/capiutils" "github.com/openshift/installer/pkg/ipnet" "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ) var ( @@ -31,14 +33,6 @@ func stubInstallConfig() *installconfig.InstallConfig { return ic } -func stubInstallConfigComplete() *installconfig.InstallConfig { - ic := stubInstallConfig() - ic.Config = stubInstallConfigType() - ic.Config.Compute = stubInstallCOnfigPoolCompute() - ic.Config.ControlPlane = stubInstallCOnfigPoolControl() - return ic -} - func stubInstallConfigType() *types.InstallConfig { return &types.InstallConfig{ TypeMeta: metav1.TypeMeta{ @@ -57,27 +51,6 @@ func stubInstallConfigType() *types.InstallConfig { }, } } -func stubInstallConfigTypeZones() *types.InstallConfig { - c := &types.InstallConfig{ - TypeMeta: metav1.TypeMeta{ - APIVersion: types.InstallConfigVersion, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - }, - Publish: types.ExternalPublishingStrategy, - Networking: &types.Networking{ - MachineNetwork: []types.MachineNetworkEntry{ - { - CIDR: *ipnet.MustParseCIDR(stubDefaultCIDR), - }, - }, - }, - } - c.ControlPlane = stubInstallCOnfigPoolControl() - c.Compute = stubInstallCOnfigPoolCompute() - return c -} func stubInstallCOnfigPoolCompute() []types.MachinePool { return []types.MachinePool{ @@ -92,7 +65,7 @@ func stubInstallCOnfigPoolCompute() []types.MachinePool { } } -func stubInstallCOnfigPoolControl() *types.MachinePool { +func stubInstallConfigPoolControl() *types.MachinePool { return &types.MachinePool{ Name: "master", Platform: types.MachinePoolPlatform{ @@ -103,183 +76,758 @@ func stubInstallCOnfigPoolControl() *types.MachinePool { } } -func stubAwsCluster() *capa.AWSCluster { - return &capa.AWSCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "infraId", - Namespace: capiutils.Namespace, - }, - Spec: capa.AWSClusterSpec{}, +var stubAwsCluster = &capa.AWSCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "infraId", + Namespace: capiutils.Namespace, + }, + Spec: capa.AWSClusterSpec{}, +} + +func tSortCapaSubnetsByID(in capa.Subnets) capa.Subnets { + subnetIds := []string{} + subnetsMap := make(map[string]capa.SubnetSpec, len(in)) + for _, subnet := range in { + subnetsMap[subnet.ID] = subnet + subnetIds = append(subnetIds, subnet.ID) } + sort.Strings(subnetIds) + out := capa.Subnets{} + for _, sid := range subnetIds { + out = append(out, subnetsMap[sid]) + } + return out } func Test_extractZonesFromInstallConfig(t *testing.T) { type args struct { - in *zoneConfigInput + in *zonesInput } tests := []struct { name string args args - want []*aws.Zone + want *zonesCAPI wantErr bool }{ - { - name: "empty install config", - args: args{ - in: &zoneConfigInput{ - Config: nil, - }, - }, - wantErr: true, - }, { name: "default zones", args: args{ - in: &zoneConfigInput{ - Config: stubInstallConfigType(), + in: &zonesInput{ + InstallConfig: func() *installconfig.InstallConfig { + ic := stubInstallConfig() + ic.Config = stubInstallConfigType() + return ic + }(), }, }, - want: nil, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, }, { name: "custom zones control plane pool", args: args{ - in: &zoneConfigInput{ - Config: func() *types.InstallConfig { - config := types.InstallConfig{ - ControlPlane: stubInstallCOnfigPoolControl(), + in: &zonesInput{ + InstallConfig: func() *installconfig.InstallConfig { + ic := stubInstallConfig() + ic.Config = &types.InstallConfig{ + ControlPlane: stubInstallConfigPoolControl(), Compute: nil, } - return &config + return ic }(), }, }, - want: []*aws.Zone{{Name: "a"}, {Name: "b"}}, + want: &zonesCAPI{ + controlPlaneZones: sets.New("a", "b"), + computeZones: sets.Set[string]{}, + }, }, { name: "custom zones compute pool", args: args{ - in: &zoneConfigInput{ - Config: func() *types.InstallConfig { - config := types.InstallConfig{ + in: &zonesInput{ + InstallConfig: func() *installconfig.InstallConfig { + ic := stubInstallConfig() + ic.Config = &types.InstallConfig{ ControlPlane: nil, Compute: stubInstallCOnfigPoolCompute(), } - return &config + return ic }(), }, }, - want: []*aws.Zone{{Name: "b"}, {Name: "c"}}, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b", "c"), + }, }, { name: "custom zones control plane and compute pools", args: args{ - in: &zoneConfigInput{ - Config: func() *types.InstallConfig { - c := &types.InstallConfig{} - c.ControlPlane = stubInstallCOnfigPoolControl() - c.Compute = stubInstallCOnfigPoolCompute() - return c + in: &zonesInput{ + InstallConfig: func() *installconfig.InstallConfig { + ic := stubInstallConfig() + ic.Config = &types.InstallConfig{ + ControlPlane: stubInstallConfigPoolControl(), + Compute: stubInstallCOnfigPoolCompute(), + } + return ic }(), }, }, - want: []*aws.Zone{{Name: "a"}, {Name: "b"}, {Name: "c"}}, + want: &zonesCAPI{ + controlPlaneZones: sets.New("a", "b"), + computeZones: sets.New("b", "c"), + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := extractZonesFromInstallConfig(tt.args.in) if (err != nil) != tt.wantErr { - t.Errorf("extractZonesFromInstallConfig() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("extractZonesFromInstallConfig() error: %v, wantErr: %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - spew.Printf("Got: %v\n", got) - t.Errorf("extractZonesFromInstallConfig() = %v, want %v", got, tt.want) + t.Errorf("extractZonesFromInstallConfig() err=%v\ngot : %#v,\nwant: %#v\n", err, got, tt.want) } }) } } -func Test_setZonesManagedVPC(t *testing.T) { +func Test_setSubnetsManagedVPC(t *testing.T) { type args struct { - in *zoneConfigInput + in *zonesInput } tests := []struct { name string args args wantErr bool - want *capa.AWSCluster + want *capa.NetworkSpec }{ { - name: "empty clusterx", + name: "regular zones from config", args: args{ - in: &zoneConfigInput{ - ClusterID: stubClusterID(), - InstallConfig: stubInstallConfigComplete(), - Config: stubInstallConfigTypeZones(), - Cluster: stubAwsCluster(), + in: &zonesInput{ + ClusterID: stubClusterID(), + InstallConfig: func() *installconfig.InstallConfig { + ic := stubInstallConfig() + ic.Config = &types.InstallConfig{ + Publish: types.ExternalPublishingStrategy, + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{ + { + CIDR: *ipnet.MustParseCIDR(stubDefaultCIDR), + }, + }, + }, + } + return ic + }(), + Cluster: stubAwsCluster, + ZonesInRegion: []string{"a", "b", "c"}, }, }, - want: func() *capa.AWSCluster { + want: func() *capa.NetworkSpec { c := capa.AWSCluster{} c.Spec.NetworkSpec.VPC = capa.VPCSpec{CidrBlock: stubDefaultCIDR} c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-a", + ID: "infra-id-subnet-private-a", AvailabilityZone: "a", IsPublic: false, CidrBlock: "10.0.0.0/19", }) c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-b", + ID: "infra-id-subnet-private-b", AvailabilityZone: "b", IsPublic: false, CidrBlock: "10.0.32.0/19", }) c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-c", + ID: "infra-id-subnet-private-c", AvailabilityZone: "c", IsPublic: false, CidrBlock: "10.0.64.0/19", }) c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-a", + ID: "infra-id-subnet-public-a", AvailabilityZone: "a", IsPublic: true, CidrBlock: "10.0.96.0/21", }) c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-b", + ID: "infra-id-subnet-public-b", AvailabilityZone: "b", IsPublic: true, CidrBlock: "10.0.104.0/21", }) c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ - ID: "infra-id-subnet-c", + ID: "infra-id-subnet-public-c", AvailabilityZone: "c", IsPublic: true, CidrBlock: "10.0.112.0/21", }) - return &c + return &c.Spec.NetworkSpec }(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := setZonesManagedVPC(tt.args.in) + err := setSubnetsManagedVPC(tt.args.in) if (err != nil) != tt.wantErr { - t.Errorf("setZonesManagedVPC() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("setSubnetsManagedVPC() #1 error: %+v,\nwantErr %+v\n", err, tt.wantErr) } - var got *capa.AWSCluster - if tt.args.in.Cluster != nil { - got = tt.args.in.Cluster + var got *capa.NetworkSpec + if tt.args.in != nil && tt.args.in.Cluster != nil { + got = &tt.args.in.Cluster.Spec.NetworkSpec + } else { + if !tt.wantErr { + t.Errorf("setSubnetsManagedVPC() #2 error: %v, wantErr: %v", err, tt.wantErr) + } + return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("setZonesManagedVPC() = %v, want %v", got, tt.want) - fmt.Println("Want:") - spew.Dump(tt.want) - fmt.Println("Got:") - spew.Dump(got) + if len(got.Subnets) == 0 { + if !tt.wantErr { + t.Errorf("setSubnetsManagedVPC() #2 error: %v, wantErr: %v", err, tt.wantErr) + } + return + } + got.Subnets = tSortCapaSubnetsByID(got.Subnets) + if tt.want != nil { + assert.EqualValuesf(t, tt.want, got, "%v failed.\nWant: %+v\nGot: %+v", tt.name) + } + }) + } +} + +func Test_setSubnetsBYOVPC(t *testing.T) { + type args struct { + in *zonesInput + } + tests := []struct { + name string + args args + want *capa.NetworkSpec + wantErr bool + }{ + { + name: "default byo vpc", + args: args{ + in: &zonesInput{ + Cluster: &capa.AWSCluster{}, + Subnets: &subnetsInput{ + vpc: "vpc-name", + privateSubnets: aws.Subnets{ + "subnetId-a-private": aws.Subnet{ + ID: "subnetId-a-private", + CIDR: "10.0.1.0/24", + Zone: &aws.Zone{ + Name: "a", + }, + Public: false, + }, + "subnetId-b-private": aws.Subnet{ + ID: "subnetId-b-private", + CIDR: "10.0.2.0/24", + Zone: &aws.Zone{ + Name: "b", + }, + Public: false, + }, + "subnetId-c-private": aws.Subnet{ + ID: "subnetId-c-private", + CIDR: "10.0.3.0/24", + Zone: &aws.Zone{ + Name: "c", + }, + Public: false, + }, + }, + publicSubnets: aws.Subnets{ + "subnetId-a-public": aws.Subnet{ + ID: "subnetId-a-public", + CIDR: "10.0.4.0/24", + Zone: &aws.Zone{ + Name: "a", + }, + Public: true, + }, + "subnetId-b-public": aws.Subnet{ + ID: "subnetId-b-public", + CIDR: "10.0.5.0/24", + Zone: &aws.Zone{ + Name: "b", + }, + Public: true, + }, + "subnetId-c-public": aws.Subnet{ + ID: "subnetId-c-public", + CIDR: "10.0.6.0/24", + Zone: &aws.Zone{ + Name: "c", + }, + Public: true, + }, + }, + }, + }, + }, + want: func() *capa.NetworkSpec { + c := capa.AWSCluster{} + c.Spec.NetworkSpec.VPC = capa.VPCSpec{ + ID: "vpc-name", + } + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-a-private", + AvailabilityZone: "a", + IsPublic: false, + CidrBlock: "10.0.1.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-a-public", + AvailabilityZone: "a", + IsPublic: true, + CidrBlock: "10.0.4.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-b-private", + AvailabilityZone: "b", + IsPublic: false, + CidrBlock: "10.0.2.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-b-public", + AvailabilityZone: "b", + IsPublic: true, + CidrBlock: "10.0.5.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-c-private", + AvailabilityZone: "c", + IsPublic: false, + CidrBlock: "10.0.3.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-c-public", + AvailabilityZone: "c", + IsPublic: true, + CidrBlock: "10.0.6.0/24", + }) + return &c.Spec.NetworkSpec + }(), + }, + { + name: "byo vpc only private subnets", + args: args{ + in: &zonesInput{ + Cluster: &capa.AWSCluster{}, + Subnets: &subnetsInput{ + vpc: "vpc-name", + privateSubnets: aws.Subnets{ + "subnetId-a-private": aws.Subnet{ + ID: "subnetId-a-private", + CIDR: "10.0.1.0/24", + Zone: &aws.Zone{ + Name: "a", + }, + Public: false, + }, + "subnetId-b-private": aws.Subnet{ + ID: "subnetId-b-private", + CIDR: "10.0.2.0/24", + Zone: &aws.Zone{ + Name: "b", + }, + Public: false, + }, + "subnetId-c-private": aws.Subnet{ + ID: "subnetId-c-private", + CIDR: "10.0.3.0/24", + Zone: &aws.Zone{ + Name: "c", + }, + Public: false, + }, + }, + }, + }, + }, + want: func() *capa.NetworkSpec { + c := capa.AWSCluster{ + Spec: capa.AWSClusterSpec{ + NetworkSpec: capa.NetworkSpec{ + VPC: capa.VPCSpec{ + ID: "vpc-name", + }, + }, + }, + } + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-a-private", + AvailabilityZone: "a", + IsPublic: false, + CidrBlock: "10.0.1.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-b-private", + AvailabilityZone: "b", + IsPublic: false, + CidrBlock: "10.0.2.0/24", + }) + c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: "subnetId-c-private", + AvailabilityZone: "c", + IsPublic: false, + CidrBlock: "10.0.3.0/24", + }) + return &c.Spec.NetworkSpec + }(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := setSubnetsBYOVPC(tt.args.in) + if (err != nil) != tt.wantErr { + t.Errorf("setSubnetsBYOVPC() #1 error: %v, wantErr: %v", err, tt.wantErr) + return + } + var got *capa.NetworkSpec + if tt.args.in != nil && tt.args.in.Cluster != nil { + got = &tt.args.in.Cluster.Spec.NetworkSpec + } else { + if !tt.wantErr { + t.Errorf("setSubnetsBYOVPC() #2 error: %v, wantErr: %v", err, tt.wantErr) + } + return + } + if len(got.Subnets) == 0 { + if !tt.wantErr { + t.Errorf("setSubnetsBYOVPC() #2 error: %v, wantErr: %v", err, tt.wantErr) + } + return + } + got.Subnets = tSortCapaSubnetsByID(got.Subnets) + if tt.want != nil { + assert.EqualValuesf(t, tt.want, got, "%v failed.\nWant: %+v\nGot: %+v", tt.name) + } + }) + } +} + +func Test_zonesCAPI_SetAvailabilityZones(t *testing.T) { + type fields struct { + controlPlaneZones sets.Set[string] + computeZones sets.Set[string] + } + type args struct { + pool string + zones []string + } + tests := []struct { + name string + fields fields + args args + want *zonesCAPI + }{ + { + name: "empty", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + zones: []string{}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + }, + { + name: "set zones for control plane pool", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + zones: []string{"a", "b"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.New("a", "b"), + computeZones: sets.Set[string]{}, + }, + }, + { + name: "set zones for compute pool", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolComputeRoleName, + zones: []string{"b", "c"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b", "c"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + zo := &zonesCAPI{ + controlPlaneZones: tt.fields.controlPlaneZones, + computeZones: tt.fields.computeZones, + } + zo.SetAvailabilityZones(tt.args.pool, tt.args.zones) + if tt.want != nil { + assert.EqualValuesf(t, tt.want, zo, "%v failed", tt.name) + } + }) + } +} + +func Test_zonesCAPI_SetDefaultConfigZones(t *testing.T) { + type fields struct { + AvailabilityZones sets.Set[string] + controlPlaneZones sets.Set[string] + computeZones sets.Set[string] + } + type args struct { + pool string + defConfig []string + defRegion []string + } + tests := []struct { + name string + fields fields + args args + want *zonesCAPI + }{ + { + name: "empty", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + defConfig: []string{}, + defRegion: []string{}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + }, + { + name: "platform defaults when control plane pool exists", + fields: fields{ + controlPlaneZones: sets.New("a"), + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + defConfig: []string{"d"}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.New("a"), + computeZones: sets.Set[string]{}, + }, + }, + { + name: "platform defaults when control plane pool not exists", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + defConfig: []string{"d"}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.New("d"), + computeZones: sets.Set[string]{}, + }, + }, + { + name: "region defaults when control plane pool exists", + fields: fields{ + controlPlaneZones: sets.New("a"), + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + defConfig: []string{}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.New("a"), + computeZones: sets.Set[string]{}, + }, + }, + { + name: "region defaults when control plane pool not exists", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolControlPlaneRoleName, + defConfig: []string{}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.New("f"), + computeZones: sets.Set[string]{}, + }, + }, + { + name: "platform defaults when compute pool exists", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b"), + }, + args: args{ + pool: types.MachinePoolComputeRoleName, + defConfig: []string{"d"}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b"), + }, + }, + { + name: "platform defaults when compute pool not exists", + fields: fields{ + AvailabilityZones: sets.Set[string]{}, + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolComputeRoleName, + defConfig: []string{"d"}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("d"), + }, + }, + { + name: "region defaults when compute pool exists", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b"), + }, + args: args{ + pool: types.MachinePoolComputeRoleName, + defConfig: []string{}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("b"), + }, + }, + { + name: "region defaults when compute pool not exists", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + args: args{ + pool: types.MachinePoolComputeRoleName, + defConfig: []string{}, + defRegion: []string{"f"}, + }, + want: &zonesCAPI{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("f"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + zo := &zonesCAPI{ + controlPlaneZones: tt.fields.controlPlaneZones, + computeZones: tt.fields.computeZones, + } + zo.SetDefaultConfigZones(tt.args.pool, tt.args.defConfig, tt.args.defRegion) + if tt.want != nil { + assert.EqualValuesf(t, tt.want, zo, "%v failed", tt.name) + } + }) + } +} + +func Test_zonesCAPI_AvailabilityZones(t *testing.T) { + type fields struct { + controlPlaneZones sets.Set[string] + computeZones sets.Set[string] + } + tests := []struct { + name string + fields fields + want []string + }{ + // TODO: Add test cases. + { + name: "empty", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.Set[string]{}, + }, + want: []string{}, + }, + { + name: "sorted", + fields: fields{ + controlPlaneZones: sets.New("a", "b"), + computeZones: sets.New("b", "c"), + }, + want: []string{"a", "b", "c"}, + }, + { + name: "unsorted", + fields: fields{ + controlPlaneZones: sets.New("x", "a"), + computeZones: sets.New("b", "a"), + }, + want: []string{"a", "b", "x"}, + }, + { + name: "control planes only", + fields: fields{ + controlPlaneZones: sets.New("x", "a"), + computeZones: sets.Set[string]{}, + }, + want: []string{"a", "x"}, + }, + { + name: "compute only", + fields: fields{ + controlPlaneZones: sets.Set[string]{}, + computeZones: sets.New("x", "a"), + }, + want: []string{"a", "x"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + zo := &zonesCAPI{ + controlPlaneZones: tt.fields.controlPlaneZones, + computeZones: tt.fields.computeZones, + } + if got := zo.AvailabilityZones(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("zonesCAPI.AvailabilityZones() = %v, want %v", got, tt.want) } }) } diff --git a/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go b/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go index b254bf535ef..460e81d3752 100644 --- a/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go +++ b/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go @@ -421,12 +421,6 @@ type SubnetSpec struct { // Tags is a collection of tags describing the resource. Tags Tags `json:"tags,omitempty"` - // ZoneName defines a zone name for this subnet - // If you're already set AvailabilityZone, it will take precendence. - // - // The valid values are availability-zone , local-zone , and wavelength-zone - ZoneName *string `json:"zoneName,omitempty"` - // ZoneType defines a zone type for this subnet // If you're already set AvailabilityZone, it will take precendence. // @@ -447,6 +441,8 @@ type SubnetSpec struct { // type availability-zone in the region, or first available). ZoneType *string `json:"zoneType,omitempty"` + // ParentZoneName defines a parent zone name of the zone that the subnet is created. + // +optional ParentZoneName *string `json:"parentZoneName,omitempty"` }