[v15] Machine ID: Generate "includable" ssh_configs (#46685)

* Hack on single-cluster SSH config

* More thorough testing and adjusted header

* Switch to warn level message

* Fix tests

lib/config/openssh/openssh.go

@@ -321,3 +321,72 @@ func (c *SSHConfig) GetMuxedSSHConfig(sb *strings.Builder, config *MuxedSSHConfi
 
 	return nil
 }
+
+var clusterSSHConfigTmpl = template.Must(template.New("cluster-ssh-config").Funcs(template.FuncMap{
+	"proxyCommandQuote": proxyCommandQuote,
+}).Parse(
+	`# Cluster-specific ssh_config generated by {{ .AppName }} for cluster '{{ .ClusterName }}' via proxy '{{ .ProxyHost }}:{{ .ProxyPort }}'
+UserKnownHostsFile "{{ .KnownHostsPath }}"
+IdentityFile "{{ .IdentityFilePath }}"
+CertificateFile "{{ .CertificateFilePath }}"
+HostKeyAlgorithms {{ if .NewerHostKeyAlgorithmsSupported }}rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,{{ end }}ssh-rsa-cert-v01@openssh.com
+Port {{ .Port }}
+ProxyCommand {{ proxyCommandQuote .ExecutablePath }} ssh-proxy-command --destination-dir={{ proxyCommandQuote .DestinationDir }} --proxy-server={{ proxyCommandQuote (print .ProxyHost ":" .ProxyPort) }} --cluster={{ proxyCommandQuote .ClusterName }} {{ if .TLSRouting }}--tls-routing{{ else }}--no-tls-routing{{ end }} {{ if .ConnectionUpgrade }}--connection-upgrade{{ else }}--no-connection-upgrade{{ end }} {{ if .Resume }}--resume{{ else }}--no-resume{{ end }} --user=%r --host=%h --port=%p
+`))
+
+// ClusterSSHConfigParameters is the parameter set for GetClusterSSHConfig.
+type ClusterSSHConfigParameters struct {
+	AppName             SSHConfigApps
+	ClusterName         string
+	KnownHostsPath      string
+	IdentityFilePath    string
+	CertificateFilePath string
+	ProxyHost           string
+	ProxyPort           string
+	ExecutablePath      string
+	DestinationDir      string
+	Port                int
+	ConnectionUpgrade   bool
+	TLSRouting          bool
+	Insecure            bool
+	FIPS                bool
+	Resume              bool
+}
+
+type clusterSSHConfigTmplParams struct {
+	ClusterSSHConfigParameters
+	sshConfigOptions
+}
+
+// GetClusterSSHConfig generate a ssh_config that proxies SSH connections via
+// tbot and through to a single Teleport cluster. It performs no matching on
+// the hostname.
+//
+// As it does not use the Host match directive, it is also includable within
+// another ssh_config, which allows for more complex and customized
+// configurations.
+func (c *SSHConfig) GetClusterSSHConfig(sb *strings.Builder, config *ClusterSSHConfigParameters) error {
+	var sshOptions *sshConfigOptions
+	version, err := c.getSSHVersion()
+	if err != nil {
+		c.log.WithError(err).Debugf("Could not determine SSH version, using default SSH config")
+		sshOptions = getDefaultSSHConfigOptions()
+	} else {
+		c.log.Debugf("Found OpenSSH version %s", version)
+		sshOptions = getSSHConfigOptions(version)
+	}
+	if config.Port == 0 {
+		config.Port = defaults.SSHServerListenPort
+	}
+
+	c.log.Debugf("Using SSH options: %s", sshOptions)
+
+	if err := clusterSSHConfigTmpl.Execute(sb, clusterSSHConfigTmplParams{
+		ClusterSSHConfigParameters: *config,
+		sshConfigOptions:           *sshOptions,
+	}); err != nil {
+		return trace.Wrap(err)
+	}
+
+	return nil
+}

lib/config/openssh/openssh_test.go

@@ -238,3 +238,73 @@ func TestSSHConfig_GetMuxedSSHConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestSSHConfig_GetClusterSSHConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		sshVersion string
+		config     *ClusterSSHConfigParameters
+	}{
+		{
+			name:       "legacy OpenSSH",
+			sshVersion: "7.4.0",
+			config: &ClusterSSHConfigParameters{
+				AppName:             TbotApp,
+				ClusterName:         "example.teleport.sh",
+				DestinationDir:      "/opt/machine-id",
+				KnownHostsPath:      "/opt/machine-id/example.teleport.sh.known_hosts",
+				CertificateFilePath: "/opt/machine-id/key-cert.pub",
+				IdentityFilePath:    "/opt/machine-id/key",
+				ExecutablePath:      "/bin/tbot",
+				ProxyHost:           "example.teleport.sh",
+				ProxyPort:           "443",
+				Port:                1234,
+				Insecure:            true,
+				FIPS:                true,
+				TLSRouting:          true,
+				ConnectionUpgrade:   true,
+				Resume:              true,
+			},
+		},
+		{
+			name:       "modern OpenSSH",
+			sshVersion: "9.0.0",
+			config: &ClusterSSHConfigParameters{
+				AppName:             TbotApp,
+				ClusterName:         "example.teleport.sh",
+				DestinationDir:      "/opt/machine-id",
+				KnownHostsPath:      "/opt/machine-id/example.teleport.sh.known_hosts",
+				CertificateFilePath: "/opt/machine-id/key-cert.pub",
+				IdentityFilePath:    "/opt/machine-id/key",
+				ExecutablePath:      "/bin/tbot",
+				ProxyHost:           "example.teleport.sh",
+				ProxyPort:           "443",
+				Port:                1234,
+				Insecure:            false,
+				FIPS:                false,
+				TLSRouting:          false,
+				ConnectionUpgrade:   false,
+				Resume:              false,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &SSHConfig{
+				getSSHVersion: func() (*semver.Version, error) {
+					return semver.New(tt.sshVersion), nil
+				},
+				log: logrus.New(),
+			}
+
+			sb := &strings.Builder{}
+			err := c.GetClusterSSHConfig(sb, tt.config)
+			if golden.ShouldSet() {
+				golden.Set(t, []byte(sb.String()))
+			}
+			require.NoError(t, err)
+			require.Equal(t, string(golden.Get(t)), sb.String())
+		})
+	}
+}

lib/config/openssh/testdata/TestSSHConfig_GetClusterSSHConfig/legacy_OpenSSH.golden

@@ -0,0 +1,7 @@
+# Cluster-specific ssh_config generated by tbot for cluster 'example.teleport.sh' via proxy 'example.teleport.sh:443'
+UserKnownHostsFile "/opt/machine-id/example.teleport.sh.known_hosts"
+IdentityFile "/opt/machine-id/key"
+CertificateFile "/opt/machine-id/key-cert.pub"
+HostKeyAlgorithms ssh-rsa-cert-v01@openssh.com
+Port 1234
+ProxyCommand '/bin/tbot' ssh-proxy-command --destination-dir='/opt/machine-id' --proxy-server='example.teleport.sh:443' --cluster='example.teleport.sh' --tls-routing --connection-upgrade --resume --user=%r --host=%h --port=%p

lib/config/openssh/testdata/TestSSHConfig_GetClusterSSHConfig/modern_OpenSSH.golden

@@ -0,0 +1,7 @@
+# Cluster-specific ssh_config generated by tbot for cluster 'example.teleport.sh' via proxy 'example.teleport.sh:443'
+UserKnownHostsFile "/opt/machine-id/example.teleport.sh.known_hosts"
+IdentityFile "/opt/machine-id/key"
+CertificateFile "/opt/machine-id/key-cert.pub"
+HostKeyAlgorithms rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com
+Port 1234
+ProxyCommand '/bin/tbot' ssh-proxy-command --destination-dir='/opt/machine-id' --proxy-server='example.teleport.sh:443' --cluster='example.teleport.sh' --no-tls-routing --no-connection-upgrade --no-resume --user=%r --host=%h --port=%p

lib/tbot/service_identity_output.go

@@ -186,6 +186,7 @@ func (s *IdentityOutputService) generate(ctx context.Context) error {
 		}
 		if err := renderSSHConfig(
 			ctx,
+			s.log,
 			proxyPing,
 			clusterNames,
 			s.cfg.Destination,
@@ -249,6 +250,7 @@ type alpnTester interface {
 
 func renderSSHConfig(
 	ctx context.Context,
+	log *slog.Logger,
 	proxyPing *webclient.PingResponse,
 	clusterNames []string,
 	dest bot.Destination,
@@ -275,7 +277,7 @@ func renderSSHConfig(
 
 	// We'll write known_hosts regardless of Destination type, it's still
 	// useful alongside a manually-written ssh_config.
-	knownHosts, err := ssh.GenerateKnownHosts(
+	knownHosts, clusterKnownHosts, err := ssh.GenerateKnownHosts(
 		ctx,
 		certAuthGetter,
 		clusterNames,
@@ -332,7 +334,8 @@ func renderSSHConfig(
 			}
 		}
 
-		// Generate SSH config
+		// Generate the primary SSH config which has the cluster-specific
+		// host blocks.
 		if err := sshConf.GetSSHConfig(&sshConfigBuilder, &openssh.SSHConfigParameters{
 			AppName:             openssh.TbotApp,
 			ClusterNames:        clusterNames,
@@ -357,6 +360,53 @@ func renderSSHConfig(
 		}); err != nil {
 			return trace.Wrap(err)
 		}
+
+		// Generate the per cluster files
+		for _, clusterName := range clusterNames {
+			sshConfigName := fmt.Sprintf("%s.%s", clusterName, ssh.ConfigName)
+			knownHostsName := fmt.Sprintf("%s.%s", clusterName, ssh.KnownHostsName)
+			knownHostsPath := filepath.Join(absDestPath, knownHostsName)
+
+			sb := &strings.Builder{}
+			if err := sshConf.GetClusterSSHConfig(sb, &openssh.ClusterSSHConfigParameters{
+				AppName:             openssh.TbotApp,
+				ClusterName:         clusterName,
+				KnownHostsPath:      knownHostsPath,
+				IdentityFilePath:    identityFilePath,
+				CertificateFilePath: certificateFilePath,
+				ProxyHost:           proxyHost,
+				ProxyPort:           proxyPort,
+				ExecutablePath:      executablePath,
+				DestinationDir:      absDestPath,
+
+				Insecure:          botCfg.Insecure,
+				FIPS:              botCfg.FIPS,
+				TLSRouting:        proxyPing.Proxy.TLSRoutingEnabled,
+				ConnectionUpgrade: connUpgradeRequired,
+				// Session resumption is enabled by default, this can be
+				// configurable at a later date if we discover reasons for this to
+				// be disabled.
+				Resume: true,
+			}); err != nil {
+				return trace.Wrap(err)
+			}
+			if err := destDirectory.Write(ctx, sshConfigName, []byte(sb.String())); err != nil {
+				return trace.Wrap(err)
+			}
+
+			knownHosts, ok := clusterKnownHosts[clusterName]
+			if !ok {
+				log.WarnContext(
+					ctx,
+					"No generated known_hosts for cluster, will skip",
+					"cluster", clusterName,
+				)
+				continue
+			}
+			if err := destDirectory.Write(ctx, knownHostsName, []byte(knownHosts)); err != nil {
+				return trace.Wrap(err)
+			}
+		}
 	} else {
 		// Deprecated: this block will be removed in v17. It exists so users can
 		// revert to the old behavior if necessary.

lib/tbot/service_identity_output_test.go

@@ -21,6 +21,7 @@ package tbot
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
@@ -36,6 +37,7 @@ import (
 	"github.com/gravitational/teleport/lib/tbot/botfs"
 	"github.com/gravitational/teleport/lib/tbot/config"
 	"github.com/gravitational/teleport/lib/tbot/ssh"
+	"github.com/gravitational/teleport/lib/utils"
 	"github.com/gravitational/teleport/lib/utils/golden"
 )
 
@@ -172,6 +174,7 @@ func Test_renderSSHConfig(t *testing.T) {
 
 			err := renderSSHConfig(
 				context.Background(),
+				utils.NewSlogLoggerForTests(),
 				&webclient.PingResponse{
 					ClusterName: mockClusterName,
 					Proxy: webclient.ProxySettings{
@@ -224,6 +227,39 @@ func Test_renderSSHConfig(t *testing.T) {
 			require.Equal(
 				t, string(golden.GetNamed(t, "ssh_config")), string(sshConfigBytes),
 			)
+
+			// TODO(noah): In v17, we can move these assertions into the main
+			// block as the legacy proxycommand mode will be removed.
+			if tc.Env[sshConfigProxyModeEnv] == "new" {
+				for clusterType, clusterName := range map[string]string{
+					"local":  mockClusterName,
+					"remote": mockRemoteClusterName,
+				} {
+					clusterKnownHostBytes, err := os.ReadFile(
+						filepath.Join(dir, fmt.Sprintf("%s.%s", clusterName, ssh.KnownHostsName)),
+					)
+					require.NoError(t, err)
+					clusterKnownHostBytes = replaceTestDir(clusterKnownHostBytes)
+					clusterSSHConfigBytes, err := os.ReadFile(
+						filepath.Join(dir, fmt.Sprintf("%s.%s", clusterName, ssh.ConfigName)),
+					)
+					require.NoError(t, err)
+					clusterSSHConfigBytes = replaceTestDir(clusterSSHConfigBytes)
+
+					configGolden := fmt.Sprintf("%s_cluster_ssh_config", clusterType)
+					knownHostsGolden := fmt.Sprintf("%s_cluster_known_hosts", clusterType)
+					if golden.ShouldSet() {
+						golden.SetNamed(t, knownHostsGolden, clusterKnownHostBytes)
+						golden.SetNamed(t, configGolden, clusterSSHConfigBytes)
+					}
+					require.Equal(
+						t, string(golden.GetNamed(t, knownHostsGolden)), string(clusterKnownHostBytes),
+					)
+					require.Equal(
+						t, string(golden.GetNamed(t, configGolden)), string(clusterSSHConfigBytes),
+					)
+				}
+			}
 		})
 	}
 }

lib/tbot/service_ssh_multiplexer.go

@@ -158,7 +158,7 @@ func (s *SSHMultiplexerService) writeArtifacts(
 	}
 
 	// Generate known hosts
-	knownHosts, err := ssh.GenerateKnownHosts(
+	knownHosts, _, err := ssh.GenerateKnownHosts(
 		ctx,
 		s.botAuthClient,
 		clusterNames,

lib/tbot/ssh/ssh.go

@@ -48,39 +48,59 @@ type certAuthorityGetter interface {
 
 // GenerateKnownHosts generates a known_hosts file for the provided cluster
 // names and proxy hosts.
+//
+// It produces:
+//   - A main known_hosts file that includes all the clusters, with each
+//     cluster's CA limited to the wildcard of the cluster's domain name.
+//   - A known_hosts file per cluster that will match any hostname.
 func GenerateKnownHosts(
 	ctx context.Context,
 	bot certAuthorityGetter,
 	clusterNames []string,
 	proxyHosts string,
-) (string, error) {
+) (string, map[string]string, error) {
 	certAuthorities := make([]types.CertAuthority, 0, len(clusterNames))
 	for _, cn := range clusterNames {
 		ca, err := bot.GetCertAuthority(ctx, types.CertAuthID{
 			Type:       types.HostCA,
 			DomainName: cn,
 		}, false)
 		if err != nil {
-			return "", trace.Wrap(err)
+			return "", nil, trace.Wrap(err)
 		}
 		certAuthorities = append(certAuthorities, ca)
 	}
 
+	perCluster := make(map[string]string)
 	var sb strings.Builder
 	for _, auth := range authclient.AuthoritiesToTrustedCerts(certAuthorities) {
 		pubKeys, err := auth.SSHCertPublicKeys()
 		if err != nil {
-			return "", trace.Wrap(err)
+			return "", nil, trace.Wrap(err)
 		}
 
+		var perClusterSB strings.Builder
+		fmt.Fprintf(
+			&perClusterSB,
+			"# Cluster specific known_hosts generated for cluster '%s'\n",
+			auth.ClusterName,
+		)
 		for _, pubKey := range pubKeys {
 			bytes := ssh.MarshalAuthorizedKey(pubKey)
 			fmt.Fprintf(&sb,
 				"@cert-authority %s,%s,*.%s %s type=host\n",
-				proxyHosts, auth.ClusterName, auth.ClusterName, strings.TrimSpace(string(bytes)),
+				proxyHosts,
+				auth.ClusterName,
+				auth.ClusterName,
+				strings.TrimSpace(string(bytes)),
+			)
+			fmt.Fprintf(&perClusterSB,
+				"@cert-authority * %s type=host\n",
+				strings.TrimSpace(string(bytes)),
 			)
 		}
+		perCluster[auth.ClusterName] = perClusterSB.String()
 	}
 
-	return sb.String(), nil
+	return sb.String(), perCluster, nil
 }

lib/tbot/testdata/Test_renderSSHConfig/latest_OpenSSH_no_tls_routing/local_cluster_known_hosts.golden

@@ -0,0 +1,3 @@
+# Cluster specific known_hosts generated for cluster 'tele.blackmesa.gov'
+@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8kYdyZA1ZSNjZ4pqybDXvWplHQHkU6fPL+cAYHUkAT5CiQV4GOjwaSTcvZNK5U2fQ0jm6jknCnsZi1t9JujCjXUT3bYHCnSwWhXN55QzIu530Q/MeXz5W8TxYRrWULgPhqqtq8B9N554+s40higG21fmhhdDtpmQzw3vJLspY05mnL1+fW+RIKkM4rb150sdZXKINxfNQvERteE8WX0vL2yG4RuqJzYtGCDEGeHd+HLne7xfmqPxun7bUYaxAlplhm1z2J41hqaj8pBwDSEV9SBOZXvh6FjS9nvJCT7Z1bbZwWrAO/7E2ac0eV+5iEc0J+TyufO3F9uod+J+AICtB type=host
+@cert-authority * ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8kYdyZA1ZSNjZ4pqybDXvWplHQHkU6fPL+cAYHUkAT5CiQV4GOjwaSTcvZNK5U2fQ0jm6jknCnsZi1t9JujCjXUT3bYHCnSwWhXN55QzIu530Q/MeXz5W8TxYRrWULgPhqqtq8B9N554+s40higG21fmhhdDtpmQzw3vJLspY05mnL1+fW+RIKkM4rb150sdZXKINxfNQvERteE8WX0vL2yG4RuqJzYtGCDEGeHd+HLne7xfmqPxun7bUYaxAlplhm1z2J41hqaj8pBwDSEV9SBOZXvh6FjS9nvJCT7Z1bbZwWrAO/7E2ac0eV+5iEc0J+TyufO3F9uod+J+AICtB type=host