diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 60239d609a08a..1c0c3c8cc1301 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -13,6 +13,7 @@ import org.apache.shiro.SecurityUtils; import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.UsernamePasswordToken; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.common.Randomness; import org.opensearch.identity.IdentityService; import org.opensearch.identity.Subject; @@ -35,6 +36,7 @@ import org.passay.PasswordGenerator; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.opensearch.identity.noop.NoopTokenManager.NOOP_AUTH_TOKEN; /** * Extracts Shiro's {@link AuthenticationToken} from different types of auth headers @@ -75,6 +77,11 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) return token; } + @Override + public AuthToken issueServiceAccountToken(String extensionUniqueId) throws OpenSearchSecurityException { + return NOOP_AUTH_TOKEN; + } + @Override public Subject authenticateToken(AuthToken authToken) { return new NoopSubject(); diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index ae10f38943e73..2b2f10ec9c210 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -268,6 +268,20 @@ public String get(String setting, String defaultValue) { return retVal == null ? defaultValue : retVal; } + /** + * Returns a setting value based on the setting key. + */ + public Settings getNestedSettings(String key) { + return (Settings) settings.get(key); + } + + /** + * Returns a setting value based on the setting key. + */ + public List getNestedListOfSettings(String key) { + return (List) settings.get(key); + } + /** * Returns the setting value (as float) associated with the setting key. If it does not exists, * returns the default value provided. @@ -664,6 +678,7 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder fromXContent(parser, keyBuilder, builder, allowNullValues); } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { List list = new ArrayList<>(); + List listOfObjects = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { list.add(parser.text()); @@ -672,12 +687,19 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder } else if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { list.add(String.valueOf(parser.text())); } else { - throw new IllegalStateException("only value lists are allowed in serialized settings"); + listOfObjects.add(fromXContent(parser, true, false)); + // throw new IllegalStateException("only value lists are allowed in serialized settings"); } } String key = keyBuilder.toString(); validateValue(key, list, parser, allowNullValues); builder.putList(key, list); + if (!listOfObjects.isEmpty()) { + builder.putListOfObjects(key, listOfObjects); + } + if (!list.isEmpty() && !listOfObjects.isEmpty()) { + throw new IllegalStateException("list cannot contain both values and objects"); + } } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { String key = keyBuilder.toString(); validateValue(key, null, parser, allowNullValues); @@ -783,6 +805,20 @@ public String get(String key) { return Settings.toString(map.get(key)); } + /** + * Returns a setting value based on the setting key. + */ + public Settings getNestedSettings(String key) { + return (Settings) map.get(key); + } + + /** + * Returns a setting value based on the setting key. + */ + public List getNestedListOfSettings(String key) { + return (List) map.get(key); + } + /** Return the current secure settings, or {@code null} if none have been set. */ public SecureSettings getSecureSettings() { return secureSettings.get(); @@ -1020,6 +1056,19 @@ public Builder putList(String setting, List values) { return this; } + /** + * Sets the setting with the provided setting key and a list of values. + * + * @param setting The setting key + * @param values The values + * @return The builder + */ + public Builder putListOfObjects(String setting, List values) { + remove(setting); + map.put(setting, new ArrayList<>(values)); + return this; + } + /** * Sets all the provided settings including secure settings */ diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java new file mode 100644 index 0000000000000..530d69e418b25 --- /dev/null +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.discovery; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.transport.TransportRequest; + +import java.io.IOException; +import java.util.Objects; + +/** + * InitializeExtensionRequest to initialize plugin + * + * @opensearch.internal + */ +public class InitializeExtensionSecurityRequest extends TransportRequest { + + private final String serviceAccountToken; + + public InitializeExtensionSecurityRequest(String serviceAccountToken) { + this.serviceAccountToken = serviceAccountToken; + } + + public InitializeExtensionSecurityRequest(StreamInput in) throws IOException { + super(in); + serviceAccountToken = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(serviceAccountToken); + } + + public String getServiceAccountToken() { + return serviceAccountToken; + } + + @Override + public String toString() { + return "InitializeExtensionsRequest{" + "serviceAccountToken= " + serviceAccountToken + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityRequest that = (InitializeExtensionSecurityRequest) o; + return Objects.equals(serviceAccountToken, that.serviceAccountToken); + } + + @Override + public int hashCode() { + return Objects.hash(serviceAccountToken); + } +} diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java new file mode 100644 index 0000000000000..31d1675ead2ec --- /dev/null +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.discovery; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.transport.TransportResponse; + +import java.io.IOException; +import java.util.Objects; + +/** + * PluginResponse to intialize plugin + * + * @opensearch.internal + */ +public class InitializeExtensionSecurityResponse extends TransportResponse { + private String name; + + public InitializeExtensionSecurityResponse(String name) { + this.name = name; + } + + public InitializeExtensionSecurityResponse(StreamInput in) throws IOException { + name = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + } + + /** + * @return the node that is currently leading, according to the responding node. + */ + + public String getName() { + return this.name; + } + + @Override + public String toString() { + return "InitializeExtensionResponse{" + "name = " + name + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityResponse that = (InitializeExtensionSecurityResponse) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 61a6e2faa09e0..46caf5d689c5a 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -28,6 +28,8 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.discovery.InitializeExtensionRequest; import org.opensearch.discovery.InitializeExtensionResponse; +import org.opensearch.discovery.InitializeExtensionSecurityRequest; +import org.opensearch.discovery.InitializeExtensionSecurityResponse; import org.opensearch.env.EnvironmentSettingsResponse; import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.extensions.action.ExtensionActionRequest; @@ -41,6 +43,7 @@ import org.opensearch.extensions.settings.CustomSettingsRequestHandler; import org.opensearch.extensions.settings.RegisterCustomSettingsRequest; import org.opensearch.identity.IdentityService; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.TransportException; @@ -55,6 +58,9 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; /** @@ -70,6 +76,7 @@ public class ExtensionsManager { public static final String REQUEST_EXTENSION_ADD_SETTINGS_UPDATE_CONSUMER = "internal:discovery/addsettingsupdateconsumer"; public static final String REQUEST_EXTENSION_UPDATE_SETTINGS = "internal:discovery/updatesettings"; public static final String REQUEST_EXTENSION_DEPENDENCY_INFORMATION = "internal:discovery/dependencyinformation"; + public static final String REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS = "internal:discovery/registersecuritysettings"; public static final String REQUEST_EXTENSION_REGISTER_CUSTOM_SETTINGS = "internal:discovery/registercustomsettings"; public static final String REQUEST_EXTENSION_REGISTER_REST_ACTIONS = "internal:discovery/registerrestactions"; public static final String REQUEST_EXTENSION_REGISTER_TRANSPORT_ACTIONS = "internal:discovery/registertransportactions"; @@ -97,6 +104,8 @@ public static enum OpenSearchRequestType { private CustomSettingsRequestHandler customSettingsRequestHandler; private TransportService transportService; private ClusterService clusterService; + + private IdentityService identityService; private final Set> additionalSettings; private Settings environmentSettings; private AddSettingsUpdateConsumerRequestHandler addSettingsUpdateConsumerRequestHandler; @@ -403,10 +412,62 @@ protected void doRun() throws Exception { new InitializeExtensionRequest(transportService.getLocalNode(), extension), initializeExtensionResponseHandler ); + initializeExtensionSecurity(extension); } }); } + private void initializeExtensionSecurity(DiscoveryExtensionNode extension) { + final CompletableFuture inProgressFuture = new CompletableFuture<>(); + final TransportResponseHandler initializeExtensionSecurityResponseHandler = + new TransportResponseHandler() { + + @Override + public InitializeExtensionSecurityResponse read(StreamInput in) throws IOException { + return new InitializeExtensionSecurityResponse(in); + } + + @Override + public void handleResponse(InitializeExtensionSecurityResponse response) { + System.out.println("Registered security settings for " + response.getName()); + inProgressFuture.complete(response); + } + + @Override + public void handleException(TransportException exp) { + logger.error(new ParameterizedMessage("Extension initialization failed"), exp); + inProgressFuture.completeExceptionally(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + }; + try { + logger.info("Sending extension request type: " + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS); + AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); + transportService.sendRequest( + extension, + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS, + new InitializeExtensionSecurityRequest(serviceAccountToken.asAuthHeaderValue()), + initializeExtensionSecurityResponseHandler + ); + + inProgressFuture.orTimeout(EXTENSION_REQUEST_WAIT_TIMEOUT, TimeUnit.SECONDS).join(); + } catch (CompletionException | ConnectTransportException e) { + if (e.getCause() instanceof TimeoutException || e instanceof ConnectTransportException) { + logger.info("No response from extension to request.", e); + } else if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } else if (e.getCause() instanceof Error) { + throw (Error) e.getCause(); + } else { + throw new RuntimeException(e.getCause()); + } + } + } + /** * Handles an {@link ExtensionRequest}. * @@ -491,6 +552,10 @@ void setClusterService(ClusterService clusterService) { this.clusterService = clusterService; } + public void setIdentityService(IdentityService identityService) { + this.identityService = identityService; + } + CustomSettingsRequestHandler getCustomSettingsRequestHandler() { return customSettingsRequestHandler; } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 4b622b841a040..ecc46bca2ea3b 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -62,6 +62,53 @@ public RestInitializeExtensionAction(ExtensionsManager extensionsManager) { this.extensionsManager = extensionsManager; } + private static Map unflattenMap(Map flatMap) { + Map unflattenedMap = new HashMap<>(); + + for (Map.Entry entry : flatMap.entrySet()) { + String[] keys = entry.getKey().split("\\."); + putNested(unflattenedMap, keys, entry.getValue()); + } + + return unflattenedMap; + } + + private static void putNested(Map map, String[] keys, Object value) { + for (int i = 0; i < keys.length; i++) { + String key = keys[i]; + + if (i == keys.length - 1) { + map.put(key, value); + } else if (keys[i + 1].matches("\\d+")) { + int index = Integer.parseInt(keys[++i]); + + List> list; + if (map.containsKey(key)) { + list = (List>) map.get(key); + } else { + list = new ArrayList<>(); + map.put(key, list); + } + + while (list.size() <= index) { + list.add(new HashMap<>()); + } + + map = list.get(index); + } else { + Map nestedMap; + if (map.containsKey(key)) { + nestedMap = (Map) map.get(key); + } else { + nestedMap = new HashMap<>(); + map.put(key, nestedMap); + } + + map = nestedMap; + } + } + } + @Override public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { String name = null; @@ -124,13 +171,26 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client } } - Map additionalSettingsMap = extensionMap.entrySet() + Map additionalSettingsMap = extensionMap.entrySet() .stream() - .filter(kv -> additionalSettingsKeys.contains(kv.getKey())) + .filter(kv -> additionalSettingsKeys.stream().anyMatch(k -> { + if (k.endsWith(".")) { + return kv.getKey().startsWith(k); + } else { + return kv.getKey().equals(k); + } + })) .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + System.out.println("additionalSettingsKeys: " + additionalSettingsKeys); + System.out.println("additionalSettingsMap: " + additionalSettingsMap); + + Map unflattenedMap = unflattenMap(additionalSettingsMap); + + System.out.println("unflattenedMap: " + unflattenedMap); + Settings.Builder output = Settings.builder(); - output.loadFromMap(additionalSettingsMap); + output.loadFromMap(unflattenedMap); extAdditionalSettings.applySettings(output.build()); // Create extension read from initialization request diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 33f44a913dd8a..b9c3e2dd211ac 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -239,7 +239,6 @@ public String executor() { }; try { - // Will be replaced with ExtensionTokenProcessor and PrincipalIdentifierToken classes from feature/identity Map> filteredHeaders = filterHeaders(headers, allowList, denyList); diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java index 1255e822cea6e..024498e2f4cd2 100644 --- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java +++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.identity.IdentityService; import org.opensearch.identity.Subject; import org.opensearch.identity.tokens.AuthToken; @@ -23,11 +24,22 @@ public class NoopTokenManager implements TokenManager { private static final Logger log = LogManager.getLogger(IdentityService.class); + public static AuthToken NOOP_AUTH_TOKEN = new AuthToken() { + @Override + public String asAuthHeaderValue() { + return ""; + } + }; + /** * Issue a new Noop Token * @return a new Noop Token */ @Override + public AuthToken issueServiceAccountToken(String extensionUniqueId) throws OpenSearchSecurityException { + return NOOP_AUTH_TOKEN; + } + public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims) { return new AuthToken() { @Override diff --git a/server/src/main/java/org/opensearch/identity/tokens/StandardTokenClaims.java b/server/src/main/java/org/opensearch/identity/tokens/StandardTokenClaims.java new file mode 100644 index 0000000000000..09f60521d0e30 --- /dev/null +++ b/server/src/main/java/org/opensearch/identity/tokens/StandardTokenClaims.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.identity.tokens; + +/** + * This enum represents the standard claims that are used in the JWTs for OnBehalfOf tokens + */ +public enum StandardTokenClaims { + + ISSUER("iss"), + SUBJECT("sub"), + AUDIENCE("aud"), + EXPIRATION_TIME("exp"), + NOT_BEFORE("nbf"), + ISSUED_AT("iat"), + JWT_ID("jti"); + + private final String name; + + StandardTokenClaims(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return name; + } + + public static StandardTokenClaims fromString(String name) { + return StandardTokenClaims.valueOf(name); + } +} diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 4f6ddeb48dea3..29f48e2b1d5f2 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -8,6 +8,7 @@ package org.opensearch.identity.tokens; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.identity.Subject; /** @@ -18,11 +19,17 @@ public interface TokenManager { /** * Create a new on behalf of token * + * @param subject: The subject of the token * @param claims: A list of claims for the token to be generated with * @return A new auth token */ public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims); + /** + * Issue a service account token for an extension's service account + * */ + AuthToken issueServiceAccountToken(String extensionUniqueId) throws OpenSearchSecurityException; + /** * Authenticates a provided authToken * @param authToken: The authToken to authenticate diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 51cc7c9007159..2c8da3f3648b4 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -485,6 +485,7 @@ protected Node( additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } this.extensionsManager = new ExtensionsManager(additionalSettings); + this.extensionsManager.setIdentityService(identityService); } else { this.extensionsManager = new NoopExtensionsManager(); } diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 697cc92e82a5e..5ba7ff2d432df 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -61,6 +61,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -94,6 +95,7 @@ public class ExtensionsManagerTests extends OpenSearchTestCase { private NodeClient client; private MockNioTransport transport; private IdentityService identityService; + private Path extensionDir; private final ThreadPool threadPool = new TestThreadPool(ExtensionsManagerTests.class.getSimpleName()); private final Settings settings = Settings.builder()