Merge remote-tracking branch 'upstream/main' into fix_dissect_bug

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- [Admission control] Add Resource usage collector service and resource usage tracker  ([#9890](https://github.com/opensearch-project/OpenSearch/pull/9890))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -93,12 +94,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.zookeeper:zookeeper` from 3.9.0 to 3.9.1 ([#10506](https://github.com/opensearch-project/OpenSearch/pull/10506))
 - Bump `de.thetaphi:forbiddenapis` from 3.5.1 to 3.6 ([#10508](https://github.com/opensearch-project/OpenSearch/pull/10508))
 - Bump `commons-io:commons-io` from 2.13.0 to 2.14.0 ([#10294](https://github.com/opensearch-project/OpenSearch/pull/10294))
+- Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#10639](https://github.com/opensearch-project/OpenSearch/pull/10639))
+- Bump `com.google.http-client:google-http-client` from 1.43.2 to 1.43.3 ([#10635](https://github.com/opensearch-project/OpenSearch/pull/10635))
+- Bump `com.squareup.okio:okio` from 3.5.0 to 3.6.0 ([#10637](https://github.com/opensearch-project/OpenSearch/pull/10637))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
 - [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
+- [BUG] Fix java.lang.SecurityException in repository-gcs plugin ([#10642](https://github.com/opensearch-project/OpenSearch/pull/10642))
 
 ### Deprecated
 
@@ -113,4 +118,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x

plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java

@@ -10,13 +10,11 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.authc.AuthenticationToken;
 import org.apache.shiro.authc.UsernamePasswordToken;
 import org.opensearch.common.Randomness;
 import org.opensearch.identity.IdentityService;
 import org.opensearch.identity.Subject;
-import org.opensearch.identity.noop.NoopSubject;
 import org.opensearch.identity.tokens.AuthToken;
 import org.opensearch.identity.tokens.BasicAuthToken;
 import org.opensearch.identity.tokens.OnBehalfOfClaims;
@@ -88,20 +86,6 @@ public AuthToken issueServiceAccountToken(String audience) {
         return token;
     }
 
-    @Override
-    public Subject authenticateToken(AuthToken authToken) {
-        return new NoopSubject();
-    }
-
-    public boolean validateToken(AuthToken token) {
-        if (token instanceof BasicAuthToken) {
-            final BasicAuthToken basicAuthToken = (BasicAuthToken) token;
-            return basicAuthToken.getUser().equals(SecurityUtils.getSubject().toString())
-                && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken));
-        }
-        return false;
-    }
-
     public String getTokenInfo(AuthToken token) {
         if (token instanceof BasicAuthToken) {
             final BasicAuthToken basicAuthToken = (BasicAuthToken) token;

plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java

@@ -100,11 +100,6 @@ public void testShouldFailGetTokenInfo() {
         assertThrows(UnsupportedAuthenticationToken.class, () -> shiroAuthTokenHandler.getTokenInfo(bearerAuthToken));
     }
 
-    public void testShouldFailValidateToken() {
-        final BearerAuthToken bearerAuthToken = new BearerAuthToken("header.payload.signature");
-        assertFalse(shiroAuthTokenHandler.validateToken(bearerAuthToken));
-    }
-
     public void testShoudPassMapLookupWithToken() {
         final BasicAuthToken authToken = new BasicAuthToken("Basic dGVzdDp0ZTpzdA==");
         shiroAuthTokenHandler.getShiroTokenPasswordMap().put(authToken, "te:st");

plugins/repository-gcs/build.gradle

@@ -75,8 +75,8 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api 'com.google.guava:failureaccess:1.0.1'
 
-  api 'com.google.http-client:google-http-client:1.43.2'
-  api 'com.google.http-client:google-http-client-appengine:1.43.2'
+  api 'com.google.http-client:google-http-client:1.43.3'
+  api 'com.google.http-client:google-http-client-appengine:1.43.3'
   api 'com.google.http-client:google-http-client-gson:1.43.3'
   api 'com.google.http-client:google-http-client-jackson2:1.43.3'
 

plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1

@@ -0,0 +1 @@
+a758b82e55a2f5f681e289c5ed384d3dbda6f3cd

plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1

@@ -0,0 +1 @@
+09d6cbdde6ea3469a67601a811b4e83de3e68a79

plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java

@@ -228,7 +228,7 @@ StorageOptions createStorageOptions(
             }
             storageOptionsBuilder.setCredentials(serviceAccountCredentials);
         }
-        return storageOptionsBuilder.build();
+        return SocketAccess.doPrivilegedException(() -> storageOptionsBuilder.build());
     }
 
     /**

plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/SocketAccess.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.repositories.gcs;
 
+import org.apache.logging.log4j.core.util.Throwables;
 import org.opensearch.SpecialPermission;
 import org.opensearch.common.CheckedRunnable;
 
@@ -71,4 +72,16 @@ public static void doPrivilegedVoidIOException(CheckedRunnable<IOException> acti
             throw (IOException) e.getCause();
         }
     }
+
+    public static <T> T doPrivilegedException(PrivilegedExceptionAction<T> operation) {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            Throwables.rethrow(e.getCause());
+            assert false : "always throws";
+            return null;
+        }
+    }
+
 }

plugins/repository-hdfs/build.gradle

@@ -84,7 +84,7 @@ dependencies {
   api 'net.minidev:json-smart:2.5.0'
   api "io.netty:netty-all:${versions.netty}"
   implementation "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
-  implementation 'org.codehaus.woodstox:stax2-api:4.2.1'
+  implementation 'org.codehaus.woodstox:stax2-api:4.2.2'
 
   hdfsFixture project(':test:fixtures:hdfs-fixture')
   // Set the keytab files in the classpath so that we can access them from test code without the security manager

plugins/repository-hdfs/licenses/stax2-api-4.2.2.jar.sha1

@@ -0,0 +1 @@
+b0d746cadea928e5264f2ea294ea9a1bf815bbde

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java

@@ -56,6 +56,7 @@
 import org.opensearch.monitor.os.OsStats;
 import org.opensearch.monitor.process.ProcessStats;
 import org.opensearch.node.AdaptiveSelectionStats;
+import org.opensearch.node.NodesResourceUsageStats;
 import org.opensearch.script.ScriptCacheStats;
 import org.opensearch.script.ScriptStats;
 import org.opensearch.search.backpressure.stats.SearchBackpressureStats;
@@ -142,6 +143,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private SearchPipelineStats searchPipelineStats;
 
+    @Nullable
+    private NodesResourceUsageStats resourceUsageStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -198,6 +202,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             searchPipelineStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+            resourceUsageStats = in.readOptionalWriteable(NodesResourceUsageStats::new);
+        } else {
+            resourceUsageStats = null;
+        }
     }
 
     public NodeStats(
@@ -216,6 +225,7 @@ public NodeStats(
         @Nullable DiscoveryStats discoveryStats,
         @Nullable IngestStats ingestStats,
         @Nullable AdaptiveSelectionStats adaptiveSelectionStats,
+        @Nullable NodesResourceUsageStats resourceUsageStats,
         @Nullable ScriptCacheStats scriptCacheStats,
         @Nullable IndexingPressureStats indexingPressureStats,
         @Nullable ShardIndexingPressureStats shardIndexingPressureStats,
@@ -241,6 +251,7 @@ public NodeStats(
         this.discoveryStats = discoveryStats;
         this.ingestStats = ingestStats;
         this.adaptiveSelectionStats = adaptiveSelectionStats;
+        this.resourceUsageStats = resourceUsageStats;
         this.scriptCacheStats = scriptCacheStats;
         this.indexingPressureStats = indexingPressureStats;
         this.shardIndexingPressureStats = shardIndexingPressureStats;
@@ -344,6 +355,11 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() {
         return adaptiveSelectionStats;
     }
 
+    @Nullable
+    public NodesResourceUsageStats getResourceUsageStats() {
+        return resourceUsageStats;
+    }
+
     @Nullable
     public ScriptCacheStats getScriptCacheStats() {
         return scriptCacheStats;
@@ -430,6 +446,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_9_0)) {
             out.writeOptionalWriteable(searchPipelineStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) { // make it 2.12 when we backport
+            out.writeOptionalWriteable(resourceUsageStats);
+        }
     }
 
     @Override
@@ -520,7 +539,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getSearchPipelineStats() != null) {
             getSearchPipelineStats().toXContent(builder, params);
         }
-
+        if (getResourceUsageStats() != null) {
+            getResourceUsageStats().toXContent(builder, params);
+        }
         return builder;
     }
 }

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java

@@ -213,7 +213,8 @@ public enum Metric {
         WEIGHTED_ROUTING_STATS("weighted_routing"),
         FILE_CACHE_STATS("file_cache"),
         TASK_CANCELLATION("task_cancellation"),
-        SEARCH_PIPELINE("search_pipeline");
+        SEARCH_PIPELINE("search_pipeline"),
+        RESOURCE_USAGE_STATS("resource_usage_stats");
 
         private String metricName;
 

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java

@@ -124,7 +124,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics),
-            NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics)
+            NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics),
+            NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics)
         );
     }
 

server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -168,6 +168,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List<ShardStats> shardsStats = new ArrayList<>();

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -130,6 +130,7 @@
 import org.opensearch.node.Node.DiscoverySettings;
 import org.opensearch.node.NodeRoleSettings;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.node.resource.tracker.ResourceTrackerSettings;
 import org.opensearch.persistent.PersistentTasksClusterService;
 import org.opensearch.persistent.decider.EnableAssignmentDecider;
 import org.opensearch.plugins.PluginsService;
@@ -655,6 +656,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SegmentReplicationPressureService.MAX_REPLICATION_LIMIT_STALE_REPLICA_SETTING,
                 SegmentReplicationPressureService.MAX_ALLOWED_STALE_SHARDS,
 
+                // Settings related to resource trackers
+                ResourceTrackerSettings.GLOBAL_CPU_USAGE_AC_WINDOW_DURATION_SETTING,
+                ResourceTrackerSettings.GLOBAL_JVM_USAGE_AC_WINDOW_DURATION_SETTING,
+
                 // Settings related to Searchable Snapshots
                 Node.NODE_SEARCH_CACHE_SIZE_SETTING,
                 FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING,

server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java

@@ -50,9 +50,4 @@ public String asAuthHeaderValue() {
             }
         };
     }
-
-    @Override
-    public Subject authenticateToken(AuthToken authToken) {
-        return null;
-    }
 }

server/src/main/java/org/opensearch/identity/tokens/TokenManager.java

@@ -30,11 +30,4 @@ public interface TokenManager {
      * @return a new auth token
      */
     public AuthToken issueServiceAccountToken(final String audience);
-
-    /**
-     * Authenticates a provided authToken
-     * @param authToken: The authToken to authenticate
-     * @return The authenticated subject
-     */
-    public Subject authenticateToken(AuthToken authToken);
 }

server/src/main/java/org/opensearch/index/remote/RemoteSegmentStats.java

@@ -95,8 +95,7 @@ public RemoteSegmentStats(StreamInput in) throws IOException {
         totalRefreshBytesLag = in.readLong();
         totalUploadTime = in.readLong();
         totalDownloadTime = in.readLong();
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             totalRejections = in.readVLong();
         }
     }
@@ -260,8 +259,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(totalRefreshBytesLag);
         out.writeLong(totalUploadTime);
         out.writeLong(totalDownloadTime);
-        // TODO: change to V_2_12_0 on main after backport to 2.x
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeVLong(totalRejections);
         }
     }

server/src/main/java/org/opensearch/node/Node.java

@@ -167,6 +167,7 @@
 import org.opensearch.monitor.fs.FsProbe;
 import org.opensearch.monitor.jvm.JvmInfo;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.node.resource.tracker.NodeResourceUsageTracker;
 import org.opensearch.persistent.PersistentTasksClusterService;
 import org.opensearch.persistent.PersistentTasksExecutor;
 import org.opensearch.persistent.PersistentTasksExecutorRegistry;
@@ -805,7 +806,6 @@ protected Node(
                 remoteStoreStatsTrackerFactory,
                 recoverySettings
             );
-
             final AliasValidator aliasValidator = new AliasValidator();
 
             final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService, systemIndices);
@@ -1070,6 +1070,16 @@ protected Node(
                 transportService.getTaskManager(),
                 taskCancellationMonitoringSettings
             );
+            final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker(
+                threadPool,
+                settings,
+                clusterService.getClusterSettings()
+            );
+            final ResourceUsageCollectorService resourceUsageCollectorService = new ResourceUsageCollectorService(
+                nodeResourceUsageTracker,
+                clusterService,
+                threadPool
+            );
             this.nodeService = new NodeService(
                 settings,
                 threadPool,
@@ -1091,7 +1101,8 @@ protected Node(
                 searchBackpressureService,
                 searchPipelineService,
                 fileCache,
-                taskCancellationMonitoringService
+                taskCancellationMonitoringService,
+                resourceUsageCollectorService
             );
 
             final SearchService searchService = newSearchService(
@@ -1212,6 +1223,8 @@ protected Node(
                 b.bind(RerouteService.class).toInstance(rerouteService);
                 b.bind(ShardLimitValidator.class).toInstance(shardLimitValidator);
                 b.bind(FsHealthService.class).toInstance(fsHealthService);
+                b.bind(NodeResourceUsageTracker.class).toInstance(nodeResourceUsageTracker);
+                b.bind(ResourceUsageCollectorService.class).toInstance(resourceUsageCollectorService);
                 b.bind(SystemIndices.class).toInstance(systemIndices);
                 b.bind(IdentityService.class).toInstance(identityService);
                 b.bind(Tracer.class).toInstance(tracer);
@@ -1328,6 +1341,8 @@ public Node start() throws NodeValidationException {
         injector.getInstance(RepositoriesService.class).start();
         injector.getInstance(SearchService.class).start();
         injector.getInstance(FsHealthService.class).start();
+        injector.getInstance(NodeResourceUsageTracker.class).start();
+        injector.getInstance(ResourceUsageCollectorService.class).start();
         nodeService.getMonitorService().start();
         nodeService.getSearchBackpressureService().start();
         nodeService.getTaskCancellationMonitoringService().start();
@@ -1490,6 +1505,8 @@ private Node stop() {
         injector.getInstance(ClusterService.class).stop();
         injector.getInstance(NodeConnectionsService.class).stop();
         injector.getInstance(FsHealthService.class).stop();
+        injector.getInstance(NodeResourceUsageTracker.class).stop();
+        injector.getInstance(ResourceUsageCollectorService.class).stop();
         nodeService.getMonitorService().stop();
         nodeService.getSearchBackpressureService().stop();
         injector.getInstance(GatewayService.class).stop();
@@ -1553,6 +1570,10 @@ public synchronized void close() throws IOException {
         toClose.add(nodeService.getSearchBackpressureService());
         toClose.add(() -> stopWatch.stop().start("fsHealth"));
         toClose.add(injector.getInstance(FsHealthService.class));
+        toClose.add(() -> stopWatch.stop().start("resource_usage_tracker"));
+        toClose.add(injector.getInstance(NodeResourceUsageTracker.class));
+        toClose.add(() -> stopWatch.stop().start("resource_usage_collector"));
+        toClose.add(injector.getInstance(ResourceUsageCollectorService.class));
         toClose.add(() -> stopWatch.stop().start("gateway"));
         toClose.add(injector.getInstance(GatewayService.class));
         toClose.add(() -> stopWatch.stop().start("search"));