Merge branch 'main' into feature/extensions

peternied · Aug 2, 2023 · df3dba3 · df3dba3
2 parents 1268dee + 5384272
commit df3dba3
Show file tree

Hide file tree

Showing 60 changed files with 1,472 additions and 1,347 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -22,7 +22,9 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v1.1.4
+        uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          failure_labels: backport-failed
diff --git a/build.gradle b/build.gradle
@@ -25,7 +25,7 @@ buildscript {
         opensearch_build = version_tokens[0] + '.0'
 
         common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
-        kafka_version  = '3.5.0'
+        kafka_version  = '3.5.1'
         apache_cxf_version = '4.0.2'
         open_saml_version = '4.3.0'
         one_login_java_saml = '2.9.0'
@@ -64,7 +64,7 @@ plugins {
     id 'com.diffplug.spotless' version '6.19.0'
     id 'checkstyle'
     id 'com.netflix.nebula.ospackage' version "11.3.0"
-    id "org.gradle.test-retry" version "1.5.2"
+    id "org.gradle.test-retry" version "1.5.4"
     id 'eclipse'
     id "com.github.spotbugs" version "5.0.14"
     id "com.google.osdetector" version "1.7.3"
@@ -496,7 +496,7 @@ dependencies {
     implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}"
     implementation "io.jsonwebtoken:jjwt-jackson:${jjwt_version}"
     // JSON flattener
-    implementation ("com.github.wnameless.json:json-base:2.4.0") {
+    implementation ("com.github.wnameless.json:json-base:2.4.1") {
         exclude group: "org.glassfish", module: "jakarta.json"
         exclude group: "com.google.code.gson", module: "gson"
         exclude group: "org.json", module: "json"
@@ -524,7 +524,7 @@ dependencies {
     runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2'
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
     runtimeOnly 'commons-codec:commons-codec:1.16.0'
-    runtimeOnly 'org.cryptacular:cryptacular:1.2.4'
+    runtimeOnly 'org.cryptacular:cryptacular:1.2.5'
     runtimeOnly 'com.google.errorprone:error_prone_annotations:2.20.0'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
@@ -559,7 +559,7 @@ dependencies {
     runtimeOnly 'com.google.j2objc:j2objc-annotations:2.8'
     runtimeOnly 'com.google.code.findbugs:jsr305:3.0.2'
     runtimeOnly 'org.lz4:lz4-java:1.8.0'
-    runtimeOnly 'io.dropwizard.metrics:metrics-core:3.1.2'
+    runtimeOnly 'io.dropwizard.metrics:metrics-core:4.2.19'
     runtimeOnly 'org.slf4j:slf4j-api:1.7.30'
     runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
     runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.1'
@@ -630,7 +630,7 @@ dependencies {
     integrationTestImplementation 'junit:junit:4.13.2'
     integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
     integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.11.0'
+    integrationTestImplementation 'commons-io:commons-io:2.13.0'
     integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'

diff --git a/config/roles.yml b/config/roles.yml
@@ -108,6 +108,18 @@ knn_full_access:
     - 'cluster:admin/knn_update_model_graveyard_action'
     - 'cluster:admin/knn_warmup_action'
 
+# Allow users to execute read only ip2geo datasource action
+ip2geo_datasource_read_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/geospatial/datasource/get'
+
+# Allow users to use all ip2geo datasource action
+ip2geo_datasource_full_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/geospatial/datasource/*'
+
 # Allows users to read Notebooks
 notebooks_read_access:
   reserved: true

diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java
@@ -76,10 +76,10 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.component.Lifecycle.State;
-import org.opensearch.common.component.LifecycleComponent;
-import org.opensearch.common.component.LifecycleListener;
 import org.opensearch.common.inject.Inject;
+import org.opensearch.common.lifecycle.Lifecycle;
+import org.opensearch.common.lifecycle.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.network.NetworkModule;
@@ -93,6 +93,7 @@
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
@@ -104,7 +105,6 @@
 import org.opensearch.index.cache.query.QueryCache;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.SystemIndexDescriptor;
-import org.opensearch.indices.breaker.CircuitBreakerService;
 import org.opensearch.plugins.ClusterPlugin;
 import org.opensearch.plugins.MapperPlugin;
 import org.opensearch.repositories.RepositoriesService;
@@ -213,7 +213,7 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin
     private volatile ConfigurationRepository cr;
     private volatile AdminDNs adminDns;
     private volatile ClusterService cs;
-    private static volatile DiscoveryNode localNode;
+    private volatile AtomicReference<DiscoveryNode> localNode = new AtomicReference<>();
     private volatile AuditLog auditLog;
     private volatile BackendRegistry backendRegistry;
     private volatile SslExceptionHandler sslExceptionHandler;
@@ -781,7 +781,7 @@ public <T extends TransportResponse> void sendRequest(
                             TransportRequestOptions options,
                             TransportResponseHandler<T> handler
                         ) {
-                            si.sendRequestDecorate(sender, connection, action, request, options, handler);
+                            si.sendRequestDecorate(sender, connection, action, request, options, handler, localNode.get());
                         }
                     };
                 }
@@ -1810,7 +1810,7 @@ public void onNodeStarted(DiscoveryNode localNode) {
         if (!SSLConfig.isSslOnlyMode() && !client && !disabled) {
             cr.initOnNodeStart();
         }
-        this.localNode = localNode;
+        this.localNode.set(localNode);
         final Set<ModuleInfo> securityModules = ReflectionHelper.getModulesLoaded();
         log.info("{} OpenSearch Security modules loaded so far: {}", securityModules.size(), securityModules);
     }
@@ -1890,14 +1890,6 @@ private static String handleKeyword(final String field) {
         return field;
     }
 
-    public static DiscoveryNode getLocalNode() {
-        return localNode;
-    }
-
-    public static void setLocalNode(DiscoveryNode node) {
-        localNode = node;
-    }
-
     public static class GuiceHolder implements LifecycleComponent {
 
         private static RepositoriesService repositoriesService;
@@ -1950,7 +1942,7 @@ public static ExtensionsManager getExtensionsManager() {
         public void close() {}
 
         @Override
-        public State lifecycleState() {
+        public Lifecycle.State lifecycleState() {
             return null;
         }
 

diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java
@@ -46,6 +46,7 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
@@ -473,6 +474,24 @@ public void close() throws IOException {
         }
     }
 
+    private class DlsFlsStoredFields extends StoredFields {
+        private final StoredFields in;
+
+        public DlsFlsStoredFields(StoredFields storedFields) {
+            this.in = storedFields;
+        }
+
+        @Override
+        public void document(final int docID, StoredFieldVisitor visitor) throws IOException {
+            visitor = getDlsFlsVisitor(visitor);
+            try {
+                in.document(docID, visitor);
+            } finally {
+                finishVisitor(visitor);
+            }
+        }
+    }
+
     @Override
     protected StoredFieldsReader doGetSequentialStoredFieldsReader(final StoredFieldsReader reader) {
         return new DlsFlsStoredFieldsReader(reader);
@@ -1284,6 +1303,12 @@ public TermState termState() throws IOException {
 
     }
 
+    @Override
+    public StoredFields storedFields() throws IOException {
+        ensureOpen();
+        return new DlsFlsStoredFields(in.storedFields());
+    }
+
     private String getRuntimeActionName() {
         return (String) threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_ACTION_NAME);
     }