merge main into security extension feature branch

README.md

@@ -23,6 +23,7 @@ OpenSearch Security is a plugin for OpenSearch that offers encryption, authentic
   - [Test and Build](#test-and-build)
   - [Config hot reloading](#config-hot-reloading)
   - [Onboarding new APIs](#onboarding-new-apis)
+    - [System Index Protection](#system-index-protection)
   - [Contributing](#contributing)
   - [Getting Help](#getting-help)
   - [Code of Conduct](#code-of-conduct)
@@ -119,6 +120,38 @@ It is common practice to create new transport actions to perform different tasks
 2. Register the action in the [OpenSearch Security plugin](https://github.com/opensearch-project/security). Each new action is registered in the plugin as a new permission. Usually, plugins will define different roles for their plugin (e.g., read-only access, write access). Each role will contain a set of permissions. An example of adding a new permission to the `anomaly_read_access` role for the [Anomaly Detection plugin](https://github.com/opensearch-project/anomaly-detection) can be found in [this PR](https://github.com/opensearch-project/security/pull/997/files).
 3. Register the action in the [OpenSearch Dashboards Security plugin](https://github.com/opensearch-project/security-dashboards-plugin). This plugin maintains the full list of possible permissions, so users can see all of them when creating new roles or searching permissions via Dashboards. An example of adding different permissions can be found in [this PR](https://github.com/opensearch-project/security-dashboards-plugin/pull/689/files).
 
+```mermaid
+sequenceDiagram
+    participant Client
+    participant OpenSearch
+    participant SecurityPlugin
+    participant Cluster as Plugin
+    
+    Client->>OpenSearch: Request
+    OpenSearch->>SecurityPlugin: Request
+    SecurityPlugin->>SecurityPlugin: Add Auth information to request context
+    OpenSearch->>Cluster: Client Request
+    Cluster->>SecurityPlugin: Execute transport layer action
+    SecurityPlugin->>SecurityPlugin: Check if action is allowed
+    alt Allowed
+        SecurityPlugin->>OpenSearch: Continue request
+        OpenSearch-->>Cluster: Transport layer action result
+    else Denied
+        SecurityPlugin-->>OpenSearch: Return 403 Forbidden
+        OpenSearch-->>Client: 403 Forbidden
+    end
+    alt Plugin run outside user context
+    Cluster->>Cluster: Stash context
+    Cluster->>SecurityPlugin: Execute transport layer action outside user context
+    SecurityPlugin-->>SecurityPlugin: Check if action is allowed
+    SecurityPlugin->>OpenSearch: Continue request
+    OpenSearch-->>Cluster: Transport layer action result
+    Cluster->>Cluster: Restore user context
+    end
+    Cluster-->>SecurityPlugin: Result
+    SecurityPlugin-->>OpenSearch: Result
+    OpenSearch-->>Client: Result
+```
 
 ### System Index Protection
 

TRIAGING.md

@@ -24,7 +24,7 @@ Meetings are lightly structured as follows:
 
 1. Announcements: If there are any announcements to be made they will happen at the start of the meeting.
 2. Review of new issues: The meetings always start with reviewing all untriaged [issues](https://github.com/search?q=label%3Auntriaged+is%3Aopen++repo%3Aopensearch-project%2Fsecurity+repo%3Aopensearch-project%2Fsecurity-dashboards-plugin&type=issues&ref=advsearch&s=created&o=desc) for the security and security-dashboards repositories.
-3. Untriaged items: Review any [issues](https://github.com/search?q=-label%3Atriaged+is%3Aopen++repo%3Aopensearch-project%2Fsecurity+repo%3Aopensearch-project%2Fsecurity-dashboards-plugin&type=issues&ref=advsearch&s=created&o=desc) that might have had the 'untriaged' label removed but require additional triage discussion.
+3. Untriaged items: Review any [issues](https://github.com/search?q=-label%3Atriaged+is%3Aopen++is%3Aissue+repo%3Aopensearch-project%2Fsecurity+repo%3Aopensearch-project%2Fsecurity-dashboards-plugin&type=issues) that might have had the 'untriaged' label removed but require additional triage discussion.
 4. Open discussion: Next, we open the floor in case anyone wants to highlight an issue.
 5. Backlog discussion: Then, we review issues from the [backlogs](https://github.com/search?q=label%3A%22sprint+backlog%22+is%3Aopen++repo%3Aopensearch-project%2Fsecurity+repo%3Aopensearch-project%2Fsecurity-dashboards-plugin&type=issues&ref=advsearch&s=created&o=desc) of the security and security-dashboards repositories.
 6. Least recent discussed issue: Finally, to close out the meeting we will [review the oldest](https://github.com/search?q=+is%3Aopen++repo%3Aopensearch-project%2Fsecurity+repo%3Aopensearch-project%2Fsecurity-dashboards-plugin&type=issues&ref=advsearch&s=updated&o=asc) issues from both repositories, security and security-dashboards, to help identify issues that have languished.

build.gradle

@@ -75,16 +75,16 @@ licenseFile = rootProject.file('LICENSE.txt')
 noticeFile = rootProject.file('NOTICE.txt')
 
 spotless {
-  java {
-    // note: you can use an empty string for all the imports you didn't specify explicitly, and '\\#` prefix for static imports
-    importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
-    targetExclude('src/integrationTest/**')
-  }
-  format("integrationTest", JavaExtension) {
-      target('src/integrationTest/java/**/*.java')
-      importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
-      indentWithTabs(4)
-  }
+    java {
+        // note: you can use an empty string for all the imports you didn't specify explicitly, and '\\#` prefix for static imports
+        importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
+        targetExclude('src/integrationTest/**')
+    }
+    format("integrationTest", JavaExtension) {
+        target('src/integrationTest/java/**/*.java')
+        importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
+        indentWithTabs(4)
+    }
 }
 
 spotbugs {
@@ -135,17 +135,17 @@ test {
     }
     jacoco {
         excludes = [
-            "com.sun.jndi.dns.*",
-            "com.sun.security.sasl.gsskerb.*",
-            "java.sql.*",
-            "javax.script.*",
-            "org.jcp.xml.dsig.internal.dom.*",
-            "sun.nio.cs.ext.*",
-            "sun.security.ec.*",
-            "sun.security.jgss.*",
-            "sun.security.pkcs11.*",
-            "sun.security.smartcardio.*",
-            "sun.util.resources.provider.*"
+                "com.sun.jndi.dns.*",
+                "com.sun.security.sasl.gsskerb.*",
+                "java.sql.*",
+                "javax.script.*",
+                "org.jcp.xml.dsig.internal.dom.*",
+                "sun.nio.cs.ext.*",
+                "sun.security.ec.*",
+                "sun.security.jgss.*",
+                "sun.security.pkcs11.*",
+                "sun.security.smartcardio.*",
+                "sun.util.resources.provider.*"
         ]
     }
 }
@@ -159,17 +159,17 @@ task opensslTest(type: Test) {
     }
     jacoco {
         excludes = [
-            "com.sun.jndi.dns.*",
-            "com.sun.security.sasl.gsskerb.*",
-            "java.sql.*",
-            "javax.script.*",
-            "org.jcp.xml.dsig.internal.dom.*",
-            "sun.nio.cs.ext.*",
-            "sun.security.ec.*",
-            "sun.security.jgss.*",
-            "sun.security.pkcs11.*",
-            "sun.security.smartcardio.*",
-            "sun.util.resources.provider.*"
+                "com.sun.jndi.dns.*",
+                "com.sun.security.sasl.gsskerb.*",
+                "java.sql.*",
+                "javax.script.*",
+                "org.jcp.xml.dsig.internal.dom.*",
+                "sun.nio.cs.ext.*",
+                "sun.security.ec.*",
+                "sun.security.jgss.*",
+                "sun.security.pkcs11.*",
+                "sun.security.smartcardio.*",
+                "sun.util.resources.provider.*"
         ]
     }
 }
@@ -462,7 +462,7 @@ dependencies {
     testCompileOnly 'org.apiguardian:apiguardian-api:1.0.0'
     // Kafka test execution
     testRuntimeOnly 'org.springframework.retry:spring-retry:1.3.3'
-    testRuntimeOnly ('org.springframework:spring-core:5.3.26') {
+    testRuntimeOnly ('org.springframework:spring-core:5.3.27') {
         exclude(group:'org.springframework', module: 'spring-jcl' )
     }
     testRuntimeOnly 'org.scala-lang:scala-library:2.13.9'

bwc-test/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+distributionSha256Sum=e111cb9948407e26351227dabce49822fb88c37ee72f1d1582a69c68af2e702f

bwc-test/gradlew

@@ -85,9 +85,6 @@ done
 APP_BASE_NAME=${0##*/}
 APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
 
@@ -197,6 +194,10 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
 # Collect all arguments for the java command;
 #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
 #     shell script including quotes and variable substitutions, so put them in

checkstyle/sun_checks.xml

@@ -204,14 +204,21 @@
     <property name="severity" value="error"/>
   </module>
 
+  <module name="RegexpSingleline">
+    <property name="format" value="extension"/>
+    <property name="ignoreCase" value="true"/>
+    <property name="message" value="Extension should only be used sparingly to keep implementations as generic as possible" />
+    <property name="severity" value="error"/>
+  </module>
+
   <module name="SuppressWithPlainTextCommentFilter">
     <property name="offCommentFormat" value="CS-SUPPRESS-ALL: .+"/> <!-- Require an explaination after surpressing -->
     <property name="onCommentFormat" value="CS-ENFORCE-ALL"/>
   </module>
 
   <module name="SuppressWithPlainTextCommentFilter">
     <property name="offCommentFormat" value="CS-SUPPRESS-SINGLE\: ([\w\|]+) .+"/> <!-- Require an explaination after surpressing -->
-    <property name="onCommentFormat" value="CS-ENFORCE-SINGLE"/>
+    <property name="onCommentFormat" value="CS-ENFORCE-SINGLE()"/>
     <property name="checkFormat" value="$1"/>
   </module>
 

config/roles.yml

@@ -297,6 +297,8 @@ security_analytics_read_access:
   reserved: true
   cluster_permissions:
     - 'cluster:admin/opensearch/securityanalytics/alerts/get'
+    - 'cluster:admin/opensearch/securityanalytics/correlations/findings'
+    - 'cluster:admin/opensearch/securityanalytics/correlations/list'
     - 'cluster:admin/opensearch/securityanalytics/detector/get'
     - 'cluster:admin/opensearch/securityanalytics/detector/search'
     - 'cluster:admin/opensearch/securityanalytics/findings/get'
@@ -310,6 +312,7 @@ security_analytics_full_access:
   reserved: true
   cluster_permissions:
     - 'cluster:admin/opensearch/securityanalytics/alerts/*'
+    - 'cluster:admin/opensearch/securityanalytics/correlations/*'
     - 'cluster:admin/opensearch/securityanalytics/detector/*'
     - 'cluster:admin/opensearch/securityanalytics/findings/*'
     - 'cluster:admin/opensearch/securityanalytics/mapping/*'

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+distributionSha256Sum=e111cb9948407e26351227dabce49822fb88c37ee72f1d1582a69c68af2e702f

gradlew

@@ -85,9 +85,6 @@ done
 APP_BASE_NAME=${0##*/}
 APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
 
@@ -197,6 +194,10 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
 # Collect all arguments for the java command;
 #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
 #     shell script including quotes and variable substitutions, so put them in

src/integrationTest/java/org/opensearch/security/SnapshotSteps.java

@@ -41,7 +41,7 @@ public SnapshotSteps(RestHighLevelClient restHighLevelClient) {
 
 	// CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here
 	public org.opensearch.action.support.master.AcknowledgedResponse createSnapshotRepository(String repositoryName, String snapshotDirPath, String type)
-	//CS-ENFORCE-SINGLE
+	// CS-ENFORCE-SINGLE
 		throws IOException {
 		PutRepositoryRequest createRepositoryRequest = new PutRepositoryRequest().name(repositoryName).type(type)
 			.settings(Map.of("location", snapshotDirPath));
@@ -70,7 +70,7 @@ public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshotR
 		return snapshotClient.deleteRepository(request, DEFAULT);
 	}
 
-	//CS-SUPPRESS-SINGLE: RegexpSingleline: It is not possible to use phrase "cluster manager" instead of master here
+	//CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here
 	public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshot(String repositoryName, String snapshotName) throws IOException {
 	//CS-ENFORCE-SINGLE
 		return snapshotClient.delete(new DeleteSnapshotRequest(repositoryName, snapshotName), DEFAULT);

src/integrationTest/java/org/opensearch/test/framework/certificate/CertificateMetadata.java

@@ -9,7 +9,7 @@
 */
 package org.opensearch.test.framework.certificate;
 
-
+// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions, keeping this rule disable for the whole file
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -218,3 +218,4 @@ ExtendedKeyUsage getExtendedKeyUsage() {
 		return new ExtendedKeyUsage(usages);
 	}
 }
+// CS-ENFORCE-SINGLE

src/integrationTest/java/org/opensearch/test/framework/certificate/CertificatesIssuer.java

@@ -26,6 +26,7 @@
 
 package org.opensearch.test.framework.certificate;
 
+// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions, keeping this rule disable for the whole file
 import java.math.BigInteger;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -222,3 +223,4 @@ private BigInteger generateNextCertificateSerialNumber() {
 		return BigInteger.valueOf(ID_COUNTER.incrementAndGet());
 	}
 }
+// CS-ENFORCE-SINGLE

src/integrationTest/java/org/opensearch/test/framework/certificate/PublicKeyUsage.java

@@ -14,6 +14,7 @@
 import org.bouncycastle.asn1.x509.KeyPurposeId;
 import org.bouncycastle.asn1.x509.KeyUsage;
 
+// CS-SUPPRESS-SINGLE: RegexpSingleline Extension is used to refer to certificate extensions
 /**
 * The class is associated with certificate extensions related to key usages. These extensions are defined by
 * <a href="https://www.rfc-editor.org/rfc/rfc5280.html">RFC 5280</a> and describes allowed usage of public kay which is embedded in
@@ -25,6 +26,7 @@
 *
 * @see <a href="https://www.rfc-editor.org/rfc/rfc5280.html">RFC 5280</a>
 */
+// CS-ENFORCE-SINGLE
 enum PublicKeyUsage {
 	DIGITAL_SIGNATURE(KeyUsage.digitalSignature),
 	KEY_CERT_SIGN(KeyUsage.keyCertSign),

src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java

@@ -60,8 +60,8 @@ public class TestCertificates {
 	private static final String CA_SUBJECT = "DC=com,DC=example,O=Example Com Inc.,OU=Example Com Inc. Root CA,CN=Example Com Inc. Root CA";
 	private static final String ADMIN_DN = "CN=kirk,OU=client,O=client,L=test,C=de";
 	private static final int CERTIFICATE_VALIDITY_DAYS = 365;
-	private static final String CERTIFICATE_FILE_EXTENSION = ".cert";
-	private static final String KEY_FILE_EXTENSION = ".key";
+	private static final String CERTIFICATE_FILE_EXT = ".cert";
+	private static final String KEY_FILE_EXT = ".key";
 	private final CertificateData caCertificate;
 	private final CertificateData adminCertificate;
 	private final List<CertificateData> nodeCertificates;
@@ -106,7 +106,7 @@ public CertificateData createSelfSignedCertificate(String distinguishedName) {
 	* @return file which contains certificate in PEM format, defined by <a href="https://www.rfc-editor.org/rfc/rfc1421.txt">RFC 1421</a>
 	*/
 	public File getRootCertificate() {
-		return createTempFile("root", CERTIFICATE_FILE_EXTENSION, caCertificate.certificateInPemFormat());
+		return createTempFile("root", CERTIFICATE_FILE_EXT, caCertificate.certificateInPemFormat());
 	}
 
 	public CertificateData getRootCertificateData() {
@@ -120,7 +120,7 @@ public CertificateData getRootCertificateData() {
 	*/
 	public File getNodeCertificate(int node) {
 		CertificateData certificateData = getNodeCertificateData(node);
-		return createTempFile("node-" + node, CERTIFICATE_FILE_EXTENSION, certificateData.certificateInPemFormat());
+		return createTempFile("node-" + node, CERTIFICATE_FILE_EXT, certificateData.certificateInPemFormat());
 	}
 
 	public CertificateData getNodeCertificateData(int node) {
@@ -178,7 +178,7 @@ public CertificateData getLdapCertificateData() {
 	*/
 	public File getNodeKey(int node, String privateKeyPassword) {
 		CertificateData certificateData = nodeCertificates.get(node);
-		return createTempFile("node-" + node, KEY_FILE_EXTENSION, certificateData.privateKeyInPemFormat(privateKeyPassword));
+		return createTempFile("node-" + node, KEY_FILE_EXT, certificateData.privateKeyInPemFormat(privateKeyPassword));
 	}
 
 	/**
@@ -187,7 +187,7 @@ public File getNodeKey(int node, String privateKeyPassword) {
 	* @return file which contains certificate in PEM format, defined by <a href="https://www.rfc-editor.org/rfc/rfc1421.txt">RFC 1421</a>
 	*/
 	public File getAdminCertificate() {
-		return createTempFile("admin", CERTIFICATE_FILE_EXTENSION, adminCertificate.certificateInPemFormat());
+		return createTempFile("admin", CERTIFICATE_FILE_EXT, adminCertificate.certificateInPemFormat());
 	}
 
 	public CertificateData getAdminCertificateData() {
@@ -202,7 +202,7 @@ public CertificateData getAdminCertificateData() {
 	* by <a href="https://www.rfc-editor.org/rfc/rfc1421.txt">RFC 1421</a>
 	*/
 	public File getAdminKey(String privateKeyPassword) {
-		return createTempFile("admin", KEY_FILE_EXTENSION, adminCertificate.privateKeyInPemFormat(privateKeyPassword));
+		return createTempFile("admin", KEY_FILE_EXT, adminCertificate.privateKeyInPemFormat(privateKeyPassword));
 	}
 
 	public String[] getAdminDNs() {

src/integrationTest/java/org/opensearch/test/framework/matcher/GetSettingsResponseContainsIndicesMatcher.java

@@ -38,7 +38,11 @@ protected boolean matchesSafely(GetSettingsResponse response, Description mismat
 			if (!indexToSettings.containsKey(index)) {
 				mismatchDescription
 						.appendText("Response contains settings of indices: ")
+<<<<<<< HEAD
 						.appendValue(indexToSettings.keySet().toArray());
+=======
+						.appendValue(indexToSettings.keySet());
+>>>>>>> main
 				return false;
 			}
 		}