From 4e62e6aa14dc56945ea71ea34f488fd0a8e14312 Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Wed, 3 Apr 2024 01:40:42 -0300 Subject: [PATCH] Wavelength provisioning with CAPA changes --- .../v2/api/v1beta2/network_types.go | 135 ++++++++---- .../pkg/cloud/services/network/natgateways.go | 8 +- .../pkg/cloud/services/network/routetables.go | 206 ++++++++++-------- .../v2/pkg/cloud/services/network/subnets.go | 112 ++++++---- pkg/asset/manifests/aws/zones.go | 102 ++++++++- .../v2/api/v1beta2/network_types.go | 8 +- 6 files changed, 381 insertions(+), 190 deletions(-) diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go index 3724aeddb6f..0bba069a61a 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go @@ -22,6 +22,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" ) const ( @@ -29,12 +30,6 @@ const ( DefaultAPIServerPort = 6443 // DefaultAPIServerPortString defines the API server port as a string for convenience. DefaultAPIServerPortString = "6443" - // ZoneTypeAvailabilityZone defines the regular AWS zones in the Region. - ZoneTypeAvailabilityZone = "availability-zone" - // ZoneTypeAvailabilityZone defines the AWS zone type in Local Zone infrastructure. - ZoneTypeLocalZone = "local-zone" - // ZoneTypeAvailabilityZone defines the AWS zone type in Wavelength infrastructure. - ZoneTypeWavelengthZone = "wavelength-zone" ) // NetworkStatus encapsulates AWS networking resources. @@ -434,34 +429,30 @@ type SubnetSpec struct { // Tags is a collection of tags describing the resource. Tags Tags `json:"tags,omitempty"` - // ZoneName defines a zone name for this subnet - // If you're already set AvailabilityZone, it will take precendence. - // - // The valid values are availability-zone, local-zone , and wavelength-zone. - ZoneName *string `json:"zoneName,omitempty"` - - // ZoneType defines a zone type for this subnet - // If you're already set AvailabilityZone, it will take precendence. + // ZoneType defines a zone type for this subnet. // - // The valid values are availability-zone, local-zone , and wavelength-zone. + // The valid values are availability-zone, local-zone, and wavelength-zone. // // Zone types local-zone or wavelength-zone is not selected to automatically create - // control plane or compute nodes. + // resources like: Nat Gateway, Network Load Balancers, control plane or compute nodes. // // When local-zone, the public subnets will be associated to the public route table, - // the private subnets will try to create a NAT Gateway, when supported, otherwise - // the subnet will be associated to the private route table in the region (preferred parent zone, zone - // type availability-zone in the region, or first available). + // the private subnets uses the zone in the region to create route tables re-using + // Nat Gateways (preferred from parent zone, the zone type availability-zone in the region, + // or first table available). // // When wavelength-zone, the public subnets will be associated to the carrier route table, // created altogether the Carrier Gateway when public subnets in AWS Wavelength Zone is defined. // The private subnets will try to create a NAT Gateway, when supported, otherwise // the subnet will be associated to the private route table in the region (preferred parent zone, zone // type availability-zone in the region, or first available). - ZoneType *string `json:"zoneType,omitempty"` + // + // +kubebuilder:validation:Enum=availability-zone;local-zone + // +optional + ZoneType *ZoneType `json:"zoneType,omitempty"` - // ParentZoneName defines a parent zone name for the zone that the subnet is created, - // when applied. Available only in zone types local-zone or wavelength-zone. + // ParentZoneName defines a parent zone name of the zone that the subnet is created. + // +optional ParentZoneName *string `json:"parentZoneName,omitempty"` } @@ -480,12 +471,16 @@ func (s *SubnetSpec) String() string { } // IsEdge returns the true when the subnet is created in the edge zone, -// Local Zone or Wavelength. +// Local Zones. func (s *SubnetSpec) IsEdge() bool { if s.ZoneType == nil { return false } - if *s.ZoneType == ZoneTypeLocalZone || *s.ZoneType == ZoneTypeWavelengthZone { + fmt.Println(s.AvailabilityZone, s.ZoneType.String()) + if s.ZoneType.Equal(ZoneTypeLocalZone) { + return true + } + if s.ZoneType.Equal(ZoneTypeWavelengthZone) { return true } return false @@ -502,6 +497,29 @@ func (s *SubnetSpec) IsEdgeWavelength() bool { return false } +// SetZoneInfo updates the subnets with zone information. +func (s *SubnetSpec) SetZoneInfo(zones []*ec2.AvailabilityZone) error { + zoneInfo := func(zoneName string) *ec2.AvailabilityZone { + for _, zone := range zones { + if aws.StringValue(zone.ZoneName) == zoneName { + return zone + } + } + return nil + } + zone := zoneInfo(s.AvailabilityZone) + if zone == nil { + return fmt.Errorf("unable to get zone information for zone %v", s.AvailabilityZone) + } + if zone.ZoneType != nil { + s.ZoneType = newZoneType(zone.ZoneType) + } + if zone.ParentZoneName != nil { + s.ParentZoneName = zone.ParentZoneName + } + return nil +} + // Subnets is a slice of Subnet. // +listType=map // +listMapKey=id @@ -521,8 +539,10 @@ func (s Subnets) ToMap() map[string]*SubnetSpec { func (s Subnets) IDs() []string { res := []string{} for _, subnet := range s { - // Do not return edge zones (Local Zone or Wavelength Zone) to regular Subnet IDs, - // to keep compatibility. Use IDsWithEdge() get the the full list of zones + // Prevent returning edge zones (Local Zone) to regular Subnet IDs. + // Edge zones should not deploy control plane nodes, and does not support Nat Gateway and + // Network Load Balancers. Any resource for the core infrastructure should not consume edge + // zones. if subnet.IsEdge() { continue } @@ -573,11 +593,9 @@ func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec { // FilterPrivate returns a slice containing all subnets marked as private. func (s Subnets) FilterPrivate() (res Subnets) { for _, x := range s { - if x.ZoneType != nil { - switch aws.StringValue(x.ZoneType) { - case ZoneTypeLocalZone, ZoneTypeWavelengthZone: - continue - } + // Subnets in AWS Local Zones should not be used by core infrastructure. + if x.IsEdge() { + continue } if !x.IsPublic { res = append(res, x) @@ -589,11 +607,9 @@ func (s Subnets) FilterPrivate() (res Subnets) { // FilterPublic returns a slice containing all subnets marked as public. func (s Subnets) FilterPublic() (res Subnets) { for _, x := range s { - if x.ZoneType != nil { - switch aws.StringValue(x.ZoneType) { - case ZoneTypeLocalZone, ZoneTypeWavelengthZone: - continue - } + // Subnets in AWS Local Zones should not be used by core infrastructure. + if x.IsEdge() { + continue } if x.IsPublic { res = append(res, x) @@ -617,7 +633,7 @@ func (s Subnets) GetUniqueZones() []string { keys := make(map[string]bool) zones := []string{} for _, x := range s { - if _, value := keys[x.AvailabilityZone]; !value { + if _, value := keys[x.AvailabilityZone]; len(x.AvailabilityZone) > 0 && !value { keys[x.AvailabilityZone] = true zones = append(zones, x.AvailabilityZone) } @@ -625,6 +641,17 @@ func (s Subnets) GetUniqueZones() []string { return zones } +// SetZoneInfo updates the subnets with zone information. +func (s Subnets) SetZoneInfo(zones []*ec2.AvailabilityZone) error { + for i := range s { + err := s[i].SetZoneInfo(zones) + if err != nil { + return err + } + } + return nil +} + // IsIPv6Enabled returns true if the IPv6 block is defined on the network spec. func (s Subnets) HasPublicSubnetWavelength() bool { for _, sub := range s { @@ -852,3 +879,37 @@ func (i *IngressRule) Equals(o *IngressRule) bool { return true } + +// ZoneType defines listener AWS Availability Zone type. +type ZoneType string + +// String returns the string representation for the zone type. +func (z ZoneType) String() string { + return string(z) +} + +// Equal compares two zone types. +func (z ZoneType) Equal(other ZoneType) bool { + return z == other +} + +// newZoneType returns the zone type based in the string value representing +// the zone name. Default zone type is availability-zone. +func newZoneType(ztype *string) *ZoneType { + switch ZoneType(*ztype) { + case ZoneTypeLocalZone: + return &ZoneTypeLocalZone + case ZoneTypeWavelengthZone: + return &ZoneTypeWavelengthZone + } + return &ZoneTypeAvailabilityZone +} + +var ( + // ZoneTypeAvailabilityZone defines the regular AWS zones in the Region. + ZoneTypeAvailabilityZone = ZoneType("availability-zone") + // ZoneTypeLocalZone defines the AWS zone type in Local Zone infrastructure. + ZoneTypeLocalZone = ZoneType("local-zone") + // ZoneTypeAvailabilityZone defines the AWS zone type in Wavelength infrastructure. + ZoneTypeWavelengthZone = ZoneType("wavelength-zone") +) diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go index 95bf30a917d..5899a557ecd 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/natgateways.go @@ -360,20 +360,20 @@ func (s *Service) findNatGatewayForEdgeSubnet(sn *infrav1.SubnetSpec) (string, e return gws[0], nil } - // if not, check if the parent zone public subnet has nat gateway + // Check if the parent zone public subnet has nat gateway if sn.ParentZoneName != nil { if gws, ok := azGateways[aws.StringValue(sn.ParentZoneName)]; ok && len(gws) > 0 { return gws[0], nil } } - // if not, get the first public subnet's nat gateway available + // Get the first public subnet's nat gateway available for zone, gws := range azGateways { if len(gws[0]) > 0 { - fmt.Printf("\n>> Assigning route table ID %s to zone %s from zone %s\n", gws[0], sn.AvailabilityZone, zone) + s.scope.Debug("Assigning route table", "table ID", gws[0], "source zone", zone, "target zone", sn.AvailabilityZone) return gws[0], nil } } - return "", errors.Errorf("no nat gateways available in %q for private subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways) + return "", errors.Errorf("no nat gateways available in %q for private edge subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways) } diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go index 397d1049f9a..9ec8eb82957 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/routetables.go @@ -60,49 +60,10 @@ func (s *Service) reconcileRouteTables() error { for i := range subnets { sn := &subnets[i] // We need to compile the minimum routes for this subnet first, so we can compare it or create them. - var routes []*ec2.Route - // Gather routes to public subnets. - // Public subnets on Wavelength zone infrastructure requires Carrier Gateway to - // route intenet traffic from/to the carrier infrastructure. - if sn.IsPublic && !sn.IsEdgeWavelength() { - if s.scope.VPC().InternetGatewayID == nil { - return errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID) - } - routes = append(routes, s.getGatewayPublicRoute()) - if sn.IsIPv6 { - routes = append(routes, s.getGatewayPublicIPv6Route()) - } - } else if sn.IsPublic && sn.IsEdgeWavelength() { - if s.scope.VPC().CarrierGatewayID == nil { - return errors.Errorf("failed to create carrier routing table: carrier gateway for %q is nil", s.scope.VPC().ID) - } - routes = append(routes, s.getCarrierGatewayPublicIPv4Route()) - // TODO(mtulio): IS IPv6 supported in Carrier Gateway/Wavelength infra? - if sn.IsIPv6 { - routes = append(routes, s.getCarrierGatewayPublicIPv6Route()) - } - } else { - var natGatewayID string - // private subnets in the edge zones (Local or Wavelength zones) - if sn.IsEdge() { - natGatewayID, err = s.findNatGatewayForEdgeSubnet(sn) - } else { - natGatewayID, err = s.getNatGatewayForSubnet(sn) - } - if err != nil { - return err - } - - routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) - if sn.IsIPv6 { - if !s.scope.VPC().IsIPv6Enabled() { - // Safety net because EgressOnlyInternetGateway needs the ID from the ipv6 block. - // if, for whatever reason by this point that is not available, we don't want to - // panic because of a nil pointer access. This should never occur. Famous last words though. - return errors.Errorf("ipv6 block missing for ipv6 enabled subnet, can't create egress only internet gateway") - } - routes = append(routes, s.getEgressOnlyInternetGateway()) - } + routes, err := s.getRoutesForSubnet(sn) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedRouteTableRoutes", "Failed to get routes for managed RouteTable for subnet %s: %v", sn.ID, err) + return errors.Wrapf(err, "failed to discover routes on route table %s", sn.ID) } if rt, ok := subnetRouteMap[sn.GetResourceID()]; ok { @@ -164,7 +125,7 @@ func (s *Service) reconcileRouteTables() error { return nil } -func (s *Service) fixMismatchedRouting(specRoute *ec2.Route, currentRoute *ec2.Route, rt *ec2.RouteTable) error { +func (s *Service) fixMismatchedRouting(specRoute *ec2.CreateRouteInput, currentRoute *ec2.Route, rt *ec2.RouteTable) error { var input *ec2.ReplaceRouteInput if specRoute.DestinationCidrBlock != nil { if (currentRoute.DestinationCidrBlock != nil && @@ -233,6 +194,32 @@ func (s *Service) describeVpcRouteTablesBySubnet() (map[string]*ec2.RouteTable, return res, nil } +func (s *Service) deleteRouteTable(rt *ec2.RouteTable) error { + for _, as := range rt.Associations { + if as.SubnetId == nil { + continue + } + + if _, err := s.EC2Client.DisassociateRouteTableWithContext(context.TODO(), &ec2.DisassociateRouteTableInput{AssociationId: as.RouteTableAssociationId}); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDisassociateRouteTable", "Failed to disassociate managed RouteTable %q from Subnet %q: %v", *rt.RouteTableId, *as.SubnetId, err) + return errors.Wrapf(err, "failed to disassociate route table %q from subnet %q", *rt.RouteTableId, *as.SubnetId) + } + + record.Eventf(s.scope.InfraCluster(), "SuccessfulDisassociateRouteTable", "Disassociated managed RouteTable %q from subnet %q", *rt.RouteTableId, *as.SubnetId) + s.scope.Debug("Deleted association between route table and subnet", "route-table-id", *rt.RouteTableId, "subnet-id", *as.SubnetId) + } + + if _, err := s.EC2Client.DeleteRouteTableWithContext(context.TODO(), &ec2.DeleteRouteTableInput{RouteTableId: rt.RouteTableId}); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", *rt.RouteTableId, err) + return errors.Wrapf(err, "failed to delete route table %q", *rt.RouteTableId) + } + + record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteRouteTable", "Deleted managed RouteTable %q", *rt.RouteTableId) + s.scope.Info("Deleted route table", "route-table-id", *rt.RouteTableId) + + return nil +} + func (s *Service) deleteRouteTables() error { if s.scope.VPC().IsUnmanaged(s.scope.Name()) { s.scope.Trace("Skipping routing tables deletion in unmanaged mode") @@ -245,27 +232,10 @@ func (s *Service) deleteRouteTables() error { } for _, rt := range rts { - for _, as := range rt.Associations { - if as.SubnetId == nil { - continue - } - - if _, err := s.EC2Client.DisassociateRouteTableWithContext(context.TODO(), &ec2.DisassociateRouteTableInput{AssociationId: as.RouteTableAssociationId}); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedDisassociateRouteTable", "Failed to disassociate managed RouteTable %q from Subnet %q: %v", *rt.RouteTableId, *as.SubnetId, err) - return errors.Wrapf(err, "failed to disassociate route table %q from subnet %q", *rt.RouteTableId, *as.SubnetId) - } - - record.Eventf(s.scope.InfraCluster(), "SuccessfulDisassociateRouteTable", "Disassociated managed RouteTable %q from subnet %q", *rt.RouteTableId, *as.SubnetId) - s.scope.Debug("Deleted association between route table and subnet", "route-table-id", *rt.RouteTableId, "subnet-id", *as.SubnetId) - } - - if _, err := s.EC2Client.DeleteRouteTableWithContext(context.TODO(), &ec2.DeleteRouteTableInput{RouteTableId: rt.RouteTableId}); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", *rt.RouteTableId, err) - return errors.Wrapf(err, "failed to delete route table %q", *rt.RouteTableId) + err := s.deleteRouteTable(rt) + if err != nil { + return err } - - record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteRouteTable", "Deleted managed RouteTable %q", *rt.RouteTableId) - s.scope.Info("Deleted route table", "route-table-id", *rt.RouteTableId) } return nil } @@ -290,7 +260,7 @@ func (s *Service) describeVpcRouteTables() ([]*ec2.RouteTable, error) { return out.RouteTables, nil } -func (s *Service) createRouteTableWithRoutes(routes []*ec2.Route, isPublic bool, zone string) (*infrav1.RouteTable, error) { +func (s *Service) createRouteTableWithRoutes(routes []*ec2.CreateRouteInput, isPublic bool, zone string) (*infrav1.RouteTable, error) { out, err := s.EC2Client.CreateRouteTableWithContext(context.TODO(), &ec2.CreateRouteTableInput{ VpcId: aws.String(s.scope.VPC().ID), TagSpecifications: []*ec2.TagSpecification{ @@ -306,23 +276,17 @@ func (s *Service) createRouteTableWithRoutes(routes []*ec2.Route, isPublic bool, for i := range routes { route := routes[i] if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - if _, err := s.EC2Client.CreateRouteWithContext(context.TODO(), &ec2.CreateRouteInput{ - RouteTableId: out.RouteTable.RouteTableId, - DestinationCidrBlock: route.DestinationCidrBlock, - DestinationIpv6CidrBlock: route.DestinationIpv6CidrBlock, - EgressOnlyInternetGatewayId: route.EgressOnlyInternetGatewayId, - GatewayId: route.GatewayId, - InstanceId: route.InstanceId, - NatGatewayId: route.NatGatewayId, - NetworkInterfaceId: route.NetworkInterfaceId, - VpcPeeringConnectionId: route.VpcPeeringConnectionId, - }); err != nil { + route.RouteTableId = out.RouteTable.RouteTableId + if _, err := s.EC2Client.CreateRouteWithContext(context.TODO(), route); err != nil { return false, err } return true, nil }, awserrors.RouteTableNotFound, awserrors.NATGatewayNotFound, awserrors.GatewayNotFound); err != nil { - // TODO(vincepri): cleanup the route table if this fails. record.Warnf(s.scope.InfraCluster(), "FailedCreateRoute", "Failed to create route %s for RouteTable %q: %v", route.GoString(), *out.RouteTable.RouteTableId, err) + errDel := s.deleteRouteTable(out.RouteTable) + if errDel != nil { + s.scope.Trace("Failed to delete RouteTable %q: %v", *out.RouteTable.RouteTableId, errDel) + } return nil, errors.Wrapf(err, "failed to create route in route table %q: %s", *out.RouteTable.RouteTableId, route.GoString()) } record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateRoute", "Created route %s for RouteTable %q", route.GoString(), *out.RouteTable.RouteTableId) @@ -348,43 +312,43 @@ func (s *Service) associateRouteTable(rt *infrav1.RouteTable, subnetID string) e return nil } -func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.Route { - return &ec2.Route{ +func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ NatGatewayId: aws.String(natGatewayID), DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), } } -func (s *Service) getEgressOnlyInternetGateway() *ec2.Route { - return &ec2.Route{ +func (s *Service) getEgressOnlyInternetGateway() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), EgressOnlyInternetGatewayId: s.scope.VPC().IPv6.EgressOnlyInternetGatewayID, } } -func (s *Service) getGatewayPublicRoute() *ec2.Route { - return &ec2.Route{ +func (s *Service) getGatewayPublicRoute() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), GatewayId: aws.String(*s.scope.VPC().InternetGatewayID), } } -func (s *Service) getGatewayPublicIPv6Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getGatewayPublicIPv6Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), GatewayId: aws.String(*s.scope.VPC().InternetGatewayID), } } -func (s *Service) getCarrierGatewayPublicIPv4Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getCarrierGatewayPublicIPv4Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), CarrierGatewayId: aws.String(*s.scope.VPC().CarrierGatewayID), } } -func (s *Service) getCarrierGatewayPublicIPv6Route() *ec2.Route { - return &ec2.Route{ +func (s *Service) getCarrierGatewayPublicIPv6Route() *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ DestinationCidrBlock: aws.String(services.AnyIPv6CidrBlock), CarrierGatewayId: aws.String(*s.scope.VPC().CarrierGatewayID), } @@ -415,3 +379,67 @@ func (s *Service) getRouteTableTagParams(id string, public bool, zone string) in Additional: additionalTags, } } + +func (s *Service) getRoutesToPublicSubnet(sn *infrav1.SubnetSpec) ([]*ec2.CreateRouteInput, error) { + var routes []*ec2.CreateRouteInput + + if sn.IsEdgeWavelength() { + if s.scope.VPC().CarrierGatewayID == nil { + return routes, errors.Errorf("failed to create carrier routing table: carrier gateway for %q is nil", s.scope.VPC().ID) + } + routes = append(routes, s.getCarrierGatewayPublicIPv4Route()) + // TODO(mtulio): IS IPv6 supported in Carrier Gateway/Wavelength infra? + if sn.IsIPv6 { + routes = append(routes, s.getCarrierGatewayPublicIPv6Route()) + } + return routes, nil + } + + if s.scope.VPC().InternetGatewayID == nil { + return routes, errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID) + } + routes = append(routes, s.getGatewayPublicRoute()) + if sn.IsIPv6 { + routes = append(routes, s.getGatewayPublicIPv6Route()) + } + + return routes, nil +} + +func (s *Service) getRoutesToPrivateSubnet(sn *infrav1.SubnetSpec) (routes []*ec2.CreateRouteInput, err error) { + // var routes []*ec2.Route + var natGatewayID string + + // NAT gateways in edge zones (Local Zones) are not globally supported, + // private subnets in those locations uses Nat Gateways from the + // Parent Zone or, when not available, the first zone in the Region. + if sn.IsEdge() { + natGatewayID, err = s.findNatGatewayForEdgeSubnet(sn) + } else { + natGatewayID, err = s.getNatGatewayForSubnet(sn) + } + + if err != nil { + return routes, err + } + + routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) + if sn.IsIPv6 { + if !s.scope.VPC().IsIPv6Enabled() { + // Safety net because EgressOnlyInternetGateway needs the ID from the ipv6 block. + // if, for whatever reason by this point that is not available, we don't want to + // panic because of a nil pointer access. This should never occur. Famous last words though. + return routes, errors.Errorf("ipv6 block missing for ipv6 enabled subnet, can't create egress only internet gateway") + } + routes = append(routes, s.getEgressOnlyInternetGateway()) + } + + return routes, nil +} + +func (s *Service) getRoutesForSubnet(sn *infrav1.SubnetSpec) ([]*ec2.CreateRouteInput, error) { + if sn.IsPublic { + return s.getRoutesToPublicSubnet(sn) + } + return s.getRoutesToPrivateSubnet(sn) +} diff --git a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go index 7133bd0a1aa..8654139d372 100644 --- a/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go +++ b/cluster-api/providers/aws/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/network/subnets.go @@ -136,7 +136,7 @@ func (s *Service) reconcileSubnets() error { subnetTags := sub.Tags // Make sure tags are up-to-date. if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags) + buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags, existingSubnet.IsEdge()) tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client)) if err := tagsBuilder.Ensure(existingSubnet.Tags); err != nil { return false, err @@ -151,7 +151,10 @@ func (s *Service) reconcileSubnets() error { // We may not have a permission to tag unmanaged subnets. // When tagging unmanaged subnet fails, record an event and proceed. record.Warnf(s.scope.InfraCluster(), "FailedTagSubnet", "Failed tagging unmanaged Subnet %q: %v", existingSubnet.GetResourceID(), err) - break + // Review Bug(mtulio): if proceed, out of the loop, here will create subnets without AvailabilityZone, after the future + // iterations, making unexpected situations when the zone like unmanaged subnets in different zones. + // [1] "Unmanaged VPC, 2 existing matching subnets, subnet tagging fails, should succeed" + continue } // TODO(vincepri): check if subnet needs to be updated. @@ -177,43 +180,13 @@ func (s *Service) reconcileSubnets() error { return errors.New("expected at least 1 subnet but got 0") } - // Reconciling the zone attributes for the subnet - zoneNames := make([]string, 0, len(subnets)) - for i := range subnets { - if subnets[i].ZoneType == nil { - zoneNames = append(zoneNames, subnets[i].AvailabilityZone) - } + // Reconciling the zone information for the subnets. Subnets are grouped + // by regular zones (availability zones) or edge zones (local zones or wavelength zones), + // based in the zone-type attribute for zone + if err := s.reconcileZoneInfo(subnets); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedNoZoneInfo", "Expected the zone attributes to be populated to subnet") + return errors.Wrapf(err, "expected the zone attributes to be populated to subnet") } - zonesMap := make(map[string]*ec2.AvailabilityZone, len(zoneNames)) - - // s.scope.Info("\n>> Reconciling subnets> ZONES : %v", zoneNames) - fmt.Printf("\n\n >> Reconciling subnets> len(zoneNames) = %d \n\n", len(zoneNames)) - if len(zoneNames) > 0 { - out, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ - ZoneNames: aws.StringSlice(zoneNames), - }) - if err != nil { - record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) - return errors.Wrap(err, "failed to describe availability zones") - } - - for _, zone := range out.AvailabilityZones { - if _, ok := zonesMap[aws.StringValue(zone.ZoneName)]; !ok { - zonesMap[aws.StringValue(zone.ZoneName)] = zone - } - } - for i := range subnets { - sub := &subnets[i] - //if sub.ZoneType != nil { - // continue - //} - // sync required fields - sub.ZoneType = zonesMap[sub.AvailabilityZone].ZoneType - sub.ZoneName = zonesMap[sub.AvailabilityZone].ZoneName - sub.ParentZoneName = zonesMap[sub.AvailabilityZone].ParentZoneName - } - } - fmt.Printf("\n\n>> Reconciling subnets> zonesMap : %v \n\n", zonesMap) // When the VPC is managed by CAPA, we need to create the subnets. if !unmanagedVPC { @@ -249,6 +222,42 @@ func (s *Service) reconcileSubnets() error { return nil } +func (s *Service) retrieveZoneInfo(zoneNames []string) ([]*ec2.AvailabilityZone, error) { + zones, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice(zoneNames), + }) + if err != nil { + record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) + return nil, errors.Wrap(err, "failed to describe availability zones") + } + + return zones.AvailabilityZones, nil +} + +// reconcileZoneInfo discover the zones for all subnets, and retrieve +// persist the zone information from resource API, such as Type and +// Parent Zone. +func (s *Service) reconcileZoneInfo(subnets infrav1.Subnets) error { + if len(subnets) > 0 { + // zones, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + // ZoneNames: aws.StringSlice(subnets.GetUniqueZones()), + // }) + // if err != nil { + // record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err) + // return errors.Wrap(err, "failed to describe availability zones") + // } + zones, err := s.retrieveZoneInfo(subnets.GetUniqueZones()) + if err != nil { + return err + } + // Extract zone attributes from resource API for each subnet. + if err := subnets.SetZoneInfo(zones); err != nil { + return err + } + } + return nil +} + func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { zones, err := s.getAvailableZones() if err != nil { @@ -466,7 +475,7 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err TagSpecifications: []*ec2.TagSpecification{ tags.BuildParamsToTagSpecification( ec2.ResourceTypeSubnet, - s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags), + s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags, sn.IsEdge()), ), }, } @@ -488,9 +497,19 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err return nil, errors.Wrapf(err, "failed to wait for subnet %q", *out.Subnet.SubnetId) } + // Retrieve zone information used later to change the zone attributes. + zones, err := s.retrieveZoneInfo([]string{sn.AvailabilityZone}) + if err != nil { + return nil, errors.Wrapf(err, "failed to discover zone information for subnet %q", *out.Subnet.SubnetId) + } + err = sn.SetZoneInfo(zones) + if err != nil { + return nil, errors.Wrapf(err, "failed to update zone information for subnet %q", *out.Subnet.SubnetId) + } + // This has to be done separately, because: // InvalidParameterCombination: Only one subnet attribute can be modified at a time - if sn.IsIPv6 { + if sn.IsIPv6 && !sn.IsEdge() { // regardless of the subnet being public or not, ipv6 address needs to be assigned // on creation. There is no such thing as private ipv6 address. if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { @@ -510,7 +529,7 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) } - if sn.IsPublic { + if sn.IsPublic && !sn.IsEdge() { if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { if _, err := s.EC2Client.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ SubnetId: out.Subnet.SubnetId, @@ -584,7 +603,7 @@ func (s *Service) deleteSubnet(id string) error { return nil } -func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags) infrav1.BuildParams { +func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags, isEdge bool) infrav1.BuildParams { var role string additionalTags := make(map[string]string) @@ -593,10 +612,15 @@ func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, if public { role = infrav1.PublicRoleTagValue - additionalTags[externalLoadBalancerTag] = "1" + // Edge subnets should not have ELB tags to be selected by CCM to create load balancers. + if !isEdge { + additionalTags[externalLoadBalancerTag] = "1" + } } else { role = infrav1.PrivateRoleTagValue - additionalTags[internalLoadBalancerTag] = "1" + if !isEdge { + additionalTags[internalLoadBalancerTag] = "1" + } } // Add tag needed for Service type=LoadBalancer diff --git a/pkg/asset/manifests/aws/zones.go b/pkg/asset/manifests/aws/zones.go index 8dc2db9fc33..741c142b16b 100644 --- a/pkg/asset/manifests/aws/zones.go +++ b/pkg/asset/manifests/aws/zones.go @@ -19,6 +19,7 @@ type subnetsInput struct { vpc string privateSubnets aws.Subnets publicSubnets aws.Subnets + edgeSubnets aws.Subnets } type zonesInput struct { @@ -49,6 +50,9 @@ func (zin *zonesInput) GatherSubnetsFromMetadata(ctx context.Context) (err error if zin.Subnets.publicSubnets, err = zin.InstallConfig.AWS.PublicSubnets(ctx); err != nil { return fmt.Errorf("failed to get public subnets: %w", err) } + if zin.Subnets.edgeSubnets, err = zin.InstallConfig.AWS.EdgeSubnets(ctx); err != nil { + return fmt.Errorf("failed to get public subnets: %w", err) + } if zin.Subnets.vpc, err = zin.InstallConfig.AWS.VPC(ctx); err != nil { return fmt.Errorf("failed to get VPC: %w", err) } @@ -58,14 +62,21 @@ func (zin *zonesInput) GatherSubnetsFromMetadata(ctx context.Context) (err error type zonesCAPI struct { controlPlaneZones sets.Set[string] computeZones sets.Set[string] + edgeZones sets.Set[string] } // AvailabilityZones returns a sorted union of Availability Zones defined -// in the zone attribute in the pools for control plane and compute zones. +// in the zone attribute in the pools for control plane and compute. func (zo *zonesCAPI) AvailabilityZones() []string { return sets.List(zo.controlPlaneZones.Union(zo.computeZones)) } +// EdgeZones returns a sorted union of Local Zones or Wavelength Zones +// defined in the zone attribute in the edge compute pool. +func (zo *zonesCAPI) EdgeZones() []string { + return sets.List(zo.edgeZones) +} + // SetAvailabilityZones insert the zone to the given compute pool, and to // the regular zone (zone type availability-zone) list. func (zo *zonesCAPI) SetAvailabilityZones(pool string, zones []string) { @@ -159,6 +170,17 @@ func setSubnetsBYOVPC(in *zonesInput) error { }) } + // edgeSubnets are subnet created on AWS Local Zones or Wavelength Zone, + // discovered by ID and zone-type attribute. + for _, subnet := range in.Subnets.edgeSubnets { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + ID: subnet.ID, + CidrBlock: subnet.CIDR, + AvailabilityZone: subnet.Zone.Name, + IsPublic: subnet.Public, + }) + } + return nil } @@ -178,8 +200,10 @@ func setSubnetsManagedVPC(in *zonesInput) error { return fmt.Errorf("failed to get availability zones: %w", err) } - allZones := out.AvailabilityZones() isPublishingExternal := in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy + allAvailabilityZones := out.AvailabilityZones() + allEdgeZones := out.EdgeZones() + mainCIDR := capiutils.CIDRFromInstallConfig(in.InstallConfig) in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{ CidrBlock: mainCIDR.String(), @@ -187,36 +211,49 @@ func setSubnetsManagedVPC(in *zonesInput) error { // Base subnets considering only private zones, leaving one free block to allow // future subnet expansions in Day-2. - numSubnets := len(allZones) + 1 + numSubnets := len(allAvailabilityZones) + 1 // Public subnets consumes one range from base blocks. if isPublishingExternal { numSubnets++ } + // Edge subnets consumes one CIDR block from private CIDR, slicing it + // into smaller depending on the amount edge zones added to install config. + if len(allEdgeZones) > 0 { + numSubnets++ + } + privateCIDRs, err := utilscidr.SplitIntoSubnetsIPv4(mainCIDR.String(), numSubnets) if err != nil { - return fmt.Errorf("unable to retrieve CIDR blocks for all private subnets: %w", err) + return fmt.Errorf("unable to generate CIDR blocks for all private subnets: %w", err) + } + + publicCIDR := privateCIDRs[len(allAvailabilityZones)].String() + var edgeCIDR string + if len(allEdgeZones) > 0 { + publicCIDR = privateCIDRs[len(allAvailabilityZones)].String() + edgeCIDR = privateCIDRs[len(allAvailabilityZones)+1].String() } var publicCIDRs []*net.IPNet if isPublishingExternal { // The last num(zones) blocks are dedicated to the public subnets. - publicCIDRs, err = utilscidr.SplitIntoSubnetsIPv4(privateCIDRs[len(allZones)].String(), len(allZones)) + publicCIDRs, err = utilscidr.SplitIntoSubnetsIPv4(publicCIDR, len(allAvailabilityZones)) if err != nil { - return fmt.Errorf("unable to retrieve CIDR blocks for all public subnets: %w", err) + return fmt.Errorf("unable to generate CIDR blocks for all public subnets: %w", err) } } - // Create subnets from zone pool with type availability-zone - if len(privateCIDRs) < len(allZones) { + // Create subnets from zone pools (control plane and compute) with type availability-zone. + if len(privateCIDRs) < len(allAvailabilityZones) { return fmt.Errorf("unable to define CIDR blocks to all zones for private subnets") } - if isPublishingExternal && len(publicCIDRs) < len(allZones) { + if isPublishingExternal && len(publicCIDRs) < len(allAvailabilityZones) { return fmt.Errorf("unable to define CIDR blocks to all zones for public subnets") } - for idxCIDR, zone := range allZones { + for idxCIDR, zone := range allAvailabilityZones { in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ AvailabilityZone: zone, CidrBlock: privateCIDRs[idxCIDR].String(), @@ -232,6 +269,49 @@ func setSubnetsManagedVPC(in *zonesInput) error { }) } } + + // Create subnets from edge zone pool with type local-zone. + if len(allEdgeZones) > 0 { + // Slice the main CIDR (edgeCIDR) into N*zones for privates subnets, + // and, when publish external, duplicate to create public subnets. + numEdgeSubnets := len(allEdgeZones) + if isPublishingExternal { + numEdgeSubnets = numEdgeSubnets * 2 + } + + // Allow one CIDR block for future expansion. + numEdgeSubnets++ + + // Slice the edgeCIDR into the amount of desired subnets. + edgeCIDRs, err := utilscidr.SplitIntoSubnetsIPv4(edgeCIDR, numEdgeSubnets) + if err != nil { + return fmt.Errorf("unable to generate CIDR blocks for all edge subnets: %w", err) + } + if len(edgeCIDRs) < len(allEdgeZones) { + return fmt.Errorf("unable to define CIDR blocks to all edge zones for private subnets") + } + if isPublishingExternal && (len(edgeCIDRs) < (len(allEdgeZones) * 2)) { + return fmt.Errorf("unable to define CIDR blocks to all edge zones for public subnets") + } + + // Create subnets from zone pool with type local-zone or wavelength-zone (edge zones) + for idxCIDR, zone := range allEdgeZones { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + AvailabilityZone: zone, + CidrBlock: edgeCIDRs[idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-private-%s", in.ClusterID.InfraID, zone), + IsPublic: false, + }) + if isPublishingExternal { + in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{ + AvailabilityZone: zone, + CidrBlock: edgeCIDRs[len(allEdgeZones)+idxCIDR].String(), + ID: fmt.Sprintf("%s-subnet-public-%s", in.ClusterID.InfraID, zone), + IsPublic: true, + }) + } + } + } return nil } @@ -240,6 +320,7 @@ func extractZonesFromInstallConfig(in *zonesInput) (*zonesCAPI, error) { out := zonesCAPI{ controlPlaneZones: sets.New[string](), computeZones: sets.New[string](), + edgeZones: sets.New[string](), } cfg := in.InstallConfig.Config @@ -263,6 +344,7 @@ func extractZonesFromInstallConfig(in *zonesInput) (*zonesCAPI, error) { // Ignoring as edge pool is not yet supported by CAPA. // See https://github.com/openshift/installer/pull/8173 if pool.Name == types.MachinePoolEdgeRoleName { + out.edgeZones.Insert(pool.Platform.AWS.Zones...) continue } out.SetDefaultConfigZones(types.MachinePoolComputeRoleName, defaultZones, in.ZonesInRegion) diff --git a/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go b/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go index b254bf535ef..460e81d3752 100644 --- a/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go +++ b/vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2/network_types.go @@ -421,12 +421,6 @@ type SubnetSpec struct { // Tags is a collection of tags describing the resource. Tags Tags `json:"tags,omitempty"` - // ZoneName defines a zone name for this subnet - // If you're already set AvailabilityZone, it will take precendence. - // - // The valid values are availability-zone , local-zone , and wavelength-zone - ZoneName *string `json:"zoneName,omitempty"` - // ZoneType defines a zone type for this subnet // If you're already set AvailabilityZone, it will take precendence. // @@ -447,6 +441,8 @@ type SubnetSpec struct { // type availability-zone in the region, or first available). ZoneType *string `json:"zoneType,omitempty"` + // ParentZoneName defines a parent zone name of the zone that the subnet is created. + // +optional ParentZoneName *string `json:"parentZoneName,omitempty"` }