Merge remote-tracking branch 'upstream/main'

Signed-off-by: Oliver Lockwood <oliver.lockwood@cantab.net>

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
 - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
+- Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
 - Allow setting query parameters on requests ([#13776](https://github.com/opensearch-project/OpenSearch/issues/13776))
 
 ### Dependencies
@@ -23,6 +24,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `mockito` from 5.11.0 to 5.12.0 ([#13665](https://github.com/opensearch-project/OpenSearch/pull/13665))
 - Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#13752](https://github.com/opensearch-project/OpenSearch/pull/13752))
 - Bump `ch.qos.logback:logback-core` from 1.5.3 to 1.5.6 ([#13756](https://github.com/opensearch-project/OpenSearch/pull/13756))
+- Bump `netty` from 4.1.109.Final to 4.1.110.Final ([#13802](https://github.com/opensearch-project/OpenSearch/pull/13802))
+- Bump `jackson` from 2.17.0 to 2.17.1 ([#13817](https://github.com/opensearch-project/OpenSearch/pull/13817))
+- Bump `reactor` from 3.5.15 to 3.5.17 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `reactor-netty` from 1.1.17 to 1.1.19 ([#13825](https://github.com/opensearch-project/OpenSearch/pull/13825))
+- Bump `commons-cli:commons-cli` from 1.7.0 to 1.8.0 ([#13840](https://github.com/opensearch-project/OpenSearch/pull/13840))
+- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.0 to 5.2.1 ([#13839](https://github.com/opensearch-project/OpenSearch/pull/13839))
 
 ### Changed
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))

buildSrc/version.properties

@@ -7,8 +7,8 @@ bundled_jdk = 21.0.3+9
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.17.0
-jackson_databind  = 2.17.0
+jackson           = 2.17.1
+jackson_databind  = 2.17.1
 snakeyaml         = 2.1
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -27,12 +27,12 @@ google_http_client = 1.44.1
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.109.Final
+netty             = 4.1.110.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.17
-reactor           = 3.5.15
+reactor_netty     = 1.1.19
+reactor           = 3.5.17
 
 # client dependencies
 httpclient5       = 5.2.1

client/sniffer/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.1.jar.sha1

@@ -0,0 +1 @@
+fca7ef6192c9ad05d07bc50da991bf937a84af3a

distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.1.jar.sha1

@@ -0,0 +1 @@
+0524dcbcccdde7d45a679dfc333e4763feb09079

libs/core/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/x-content/licenses/jackson-core-2.17.1.jar.sha1

@@ -0,0 +1 @@
+5e52a11644cd59a28ef79f02bddc2cc3bab45edb

libs/x-content/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1

@@ -0,0 +1 @@
+ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71

libs/x-content/licenses/jackson-dataformat-smile-2.17.1.jar.sha1

@@ -0,0 +1 @@
+89683ac4f0a0c2c4f69ea56b90480ed40266dac8

libs/x-content/licenses/jackson-dataformat-yaml-2.17.1.jar.sha1

@@ -0,0 +1 @@
+b4c7b8a9ea3f398116a75c146b982b22afebc4ee

modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java

@@ -119,6 +119,8 @@ public class TieredSpilloverCache<K, V> implements ICache<K, V> {
                 .setValueType(builder.cacheConfig.getValueType())
                 .setSettings(builder.cacheConfig.getSettings())
                 .setWeigher(builder.cacheConfig.getWeigher())
+                .setKeySerializer(builder.cacheConfig.getKeySerializer())
+                .setValueSerializer(builder.cacheConfig.getValueSerializer())
                 .setDimensionNames(builder.cacheConfig.getDimensionNames())
                 .setStatsTrackingEnabled(false)
                 .build(),

modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java

@@ -141,6 +141,10 @@ public MockDiskCacheFactory(long delay, int maxSize, boolean statsTrackingEnable
         @Override
         @SuppressWarnings({ "unchecked" })
         public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType, Map<String, Factory> cacheFactories) {
+            // As we can't directly IT with the tiered cache and ehcache, check that we receive non-null serializers, as an ehcache disk
+            // cache would require.
+            assert config.getKeySerializer() != null;
+            assert config.getValueSerializer() != null;
             return new Builder<K, V>().setKeySerializer((Serializer<K, byte[]>) config.getKeySerializer())
                 .setValueSerializer((Serializer<V, byte[]>) config.getValueSerializer())
                 .setMaxSize(maxSize)

modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java

@@ -16,6 +16,7 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.policy.CachedQueryResult;
+import org.opensearch.common.cache.serializer.Serializer;
 import org.opensearch.common.cache.settings.CacheSettings;
 import org.opensearch.common.cache.stats.ImmutableCacheStats;
 import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
@@ -32,6 +33,8 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -166,6 +169,8 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception
                 .setKeyType(String.class)
                 .setWeigher((k, v) -> keyValueSize)
                 .setRemovalListener(removalListener)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .setSettings(settings)
                 .setDimensionNames(dimensionNames)
                 .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken
@@ -318,6 +323,8 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             .setKeyType(String.class)
             .setWeigher((k, v) -> keyValueSize)
             .setRemovalListener(removalListener)
+            .setKeySerializer(new StringSerializer())
+            .setValueSerializer(new StringSerializer())
             .setDimensionNames(dimensionNames)
             .setSettings(
                 Settings.builder()
@@ -830,6 +837,8 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
             .setKeyType(String.class)
             .setWeigher((k, v) -> 150)
             .setRemovalListener(removalListener)
+            .setKeySerializer(new StringSerializer())
+            .setValueSerializer(new StringSerializer())
             .setSettings(
                 Settings.builder()
                     .put(
@@ -1014,6 +1023,8 @@ public void testTookTimePolicyFromFactory() throws Exception {
                 .setKeyType(String.class)
                 .setWeigher((k, v) -> keyValueSize)
                 .setRemovalListener(removalListener)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
                 .setSettings(settings)
                 .setMaxSizeInBytes(onHeapCacheSize * keyValueSize)
                 .setDimensionNames(dimensionNames)
@@ -1415,6 +1426,8 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
             .setSettings(settings)
             .setDimensionNames(dimensionNames)
             .setRemovalListener(removalListener)
+            .setKeySerializer(new StringSerializer())
+            .setValueSerializer(new StringSerializer())
             .setSettings(
                 Settings.builder()
                     .put(
@@ -1479,4 +1492,26 @@ private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache<?, ?> t
         }
         return snapshot;
     }
+
+    // Duplicated here from EhcacheDiskCacheTests.java, we can't add a dependency on that plugin
+    static class StringSerializer implements Serializer<String, byte[]> {
+        private final Charset charset = StandardCharsets.UTF_8;
+
+        @Override
+        public byte[] serialize(String object) {
+            return object.getBytes(charset);
+        }
+
+        @Override
+        public String deserialize(byte[] bytes) {
+            if (bytes == null) {
+                return null;
+            }
+            return new String(bytes, charset);
+        }
+
+        public boolean equals(String object, byte[] bytes) {
+            return object.equals(deserialize(bytes));
+        }
+    }
 }

modules/ingest-geoip/licenses/jackson-annotations-2.17.1.jar.sha1

@@ -0,0 +1 @@
+fca7ef6192c9ad05d07bc50da991bf937a84af3a

modules/ingest-geoip/licenses/jackson-databind-2.17.1.jar.sha1

@@ -0,0 +1 @@
+0524dcbcccdde7d45a679dfc333e4763feb09079

modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -47,6 +47,7 @@
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.repositories.fs.FsRepository;
+import org.opensearch.test.junit.annotations.TestIssueLogging;
 import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
 import org.junit.Before;
@@ -63,6 +64,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
 
+@TestIssueLogging(value = "_root:TRACE", issueUrl = "https://github.com/opensearch-project/OpenSearch/issues/9117")
 public class RepositoryURLClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
 
     public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

modules/transport-netty4/build.gradle

@@ -235,11 +235,14 @@ thirdPartyAudit {
     'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
     'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
     'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',

modules/transport-netty4/licenses/netty-buffer-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+3d918a9ee057d995c362902b54634fc307132aac

modules/transport-netty4/licenses/netty-codec-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+f1fa43b03e93ab88e805b6a4e3e83780c80b47d2

modules/transport-netty4/licenses/netty-codec-http-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+9d05cd927209ea25bbf342962c00b8e5a828c2a4

modules/transport-netty4/licenses/netty-codec-http2-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+e0849843eb5b1c036b12551baca98a9f7ff847a0

modules/transport-netty4/licenses/netty-common-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+ec361e7e025c029be50c55c8480080cabcbc01e7

modules/transport-netty4/licenses/netty-handler-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+168db749c22652ee7fed1ebf7ec46ce856d75e51

modules/transport-netty4/licenses/netty-resolver-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+66c15921104cda0159b34e316541bc765dfaf3c0

modules/transport-netty4/licenses/netty-transport-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+b91f04c39ac14d6a29d07184ef305953ee6e0348

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.110.Final.jar.sha1

@@ -0,0 +1 @@
+a7096e7c0a25a983647909d7513f5d4943d589c0

plugins/cache-ehcache/build.gradle

@@ -24,6 +24,7 @@ versions << [
 
 dependencies {
   api "org.ehcache:ehcache:${versions.ehcache}"
+  api "org.slf4j:slf4j-api:${versions.slf4j}"
 }
 
 thirdPartyAudit {
@@ -78,10 +79,9 @@ thirdPartyAudit {
     'org.osgi.framework.BundleActivator',
     'org.osgi.framework.BundleContext',
     'org.osgi.framework.ServiceReference',
-    'org.slf4j.Logger',
-    'org.slf4j.LoggerFactory',
-    'org.slf4j.Marker',
-    'org.slf4j.event.Level'
+    'org.slf4j.impl.StaticLoggerBinder',
+    'org.slf4j.impl.StaticMDCBinder',
+    'org.slf4j.impl.StaticMarkerBinder'
   )
 }
 
@@ -90,13 +90,3 @@ tasks.named("bundlePlugin").configure {
     into 'config'
   }
 }
-
-test {
-  // TODO: Adding permission in plugin-security.policy doesn't seem to work.
-  systemProperty 'tests.security.manager', 'false'
-}
-
-internalClusterTest {
-  // TODO: Remove this later once we have a way.
-  systemProperty 'tests.security.manager', 'false'
-}

plugins/cache-ehcache/licenses/slf4j-api-1.7.36.jar.sha1

@@ -0,0 +1 @@
+6c62681a2f655b49963a5983b8b0950a6120ae14

plugins/cache-ehcache/licenses/slf4j-api-LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2004-2022 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free  of charge, to any person obtaining
+a  copy  of this  software  and  associated  documentation files  (the
+"Software"), to  deal in  the Software without  restriction, including
+without limitation  the rights to  use, copy, modify,  merge, publish,
+distribute,  sublicense, and/or sell  copies of  the Software,  and to
+permit persons to whom the Software  is furnished to do so, subject to
+the following conditions:
+
+The  above  copyright  notice  and  this permission  notice  shall  be
+included in all copies or substantial portions of the Software.
+
+THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.