diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42f8da8c72..8280db0fb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,29 +79,34 @@ jobs: steps: - uses: actions/checkout@v2 - - run: ./gradlew clean assemble - - - id: security-plugin-version - uses: madhead/read-java-properties@66cc8c88f5c6f6069ebfb42586024dd6ffe2e451 - with: - file: gradle.properties - property: security-plugin.version - - uses: actions/setup-java@v1 with: java-version: 11 - - run: ./gradlew clean assemble - - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-SNAPSHOT.jar + - run: | + security_plugin_version=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}') + security_plugin_version_no_snapshot=$(echo $security_plugin_version | sed 's/-SNAPSHOT//g') + security_plugin_version_only_number=$(echo $security_plugin_version_no_snapshot | cut -d- -f1) + test_qualifier=alpha2 + + echo "SECURITY_PLUGIN_VERSION=$security_plugin_version" >> $GITHUB_ENV + echo "SECURITY_PLUGIN_VERSION_NO_SNAPSHOT=$security_plugin_version_no_snapshot" >> $GITHUB_ENV + echo "SECURITY_PLUGIN_VERSION_ONLY_NUMBER=$security_plugin_version_only_number" >> $GITHUB_ENV + echo "TEST_QUALIFIER=$test_qualifier" >> $GITHUB_ENV + + - run: | + echo ${{ env.SECURITY_PLUGIN_VERSION }} + echo ${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }} + echo ${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }} + echo ${{ env.TEST_QUALIFIER }} + + - run: ./gradlew clean assemble && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION }}.jar - - run: ./gradlew clean assemble -Dbuild.snapshot=false - - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}.jar + - run: ./gradlew clean assemble -Dbuild.snapshot=false && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }}.jar - - run: ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=alpha1 - - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-alpha1.jar + - run: ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}.jar - - run: ./gradlew clean assemble -Dbuild.version_qualifier=alpha1 - - run: test -s ./build/opensearch-security-${{ steps.security-plugin-version.outputs.value }}-alpha1-SNAPSHOT.jar + - run: ./gradlew clean assemble -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}-SNAPSHOT.jar - name: List files in the build directory if there was an error run: ls -al ./build/ diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index 1f90e7745b..3cab575dc0 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -9,12 +9,6 @@ jobs: steps: - uses: actions/checkout@v2 - - id: opensearch-version - uses: madhead/read-java-properties@66cc8c88f5c6f6069ebfb42586024dd6ffe2e451 - with: - file: gradle.properties - property: opensearch-core.version - - name: Set up JDK uses: actions/setup-java@v1 with: @@ -25,7 +19,8 @@ jobs: - name: Download OpenSearch Core run: | - wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ steps.opensearch-version.outputs.value }}/latest/linux/x64/builds/opensearch/dist/opensearch-min-${{ steps.opensearch-version.outputs.value }}-linux-x64.tar.gz + opensearch_version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}' | sed 's/-SNAPSHOT//g'` + wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$opensearch_version/latest/linux/x64/builds/opensearch/dist/opensearch-min-$opensearch_version-linux-x64.tar.gz tar -xzf opensearch-*.tar.gz rm -f opensearch-*.tar.gz diff --git a/build.gradle b/build.gradle index 10818763c7..b546847a87 100644 --- a/build.gradle +++ b/build.gradle @@ -55,9 +55,19 @@ repositories { } ext { - default_opensearch_version = property("opensearch-core.version") + "-SNAPSHOT" - opensearch_version = System.getProperty("opensearch.version", default_opensearch_version) - buildVersionQualifier = System.getProperty("build.version_qualifier") + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "2.0.0-alpha1-SNAPSHOT") + buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1") + // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + opensearch_build_nosnapshot = opensearch_build + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } } configurations.all { @@ -124,19 +134,8 @@ dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" } -ext { - securityPluginVersion = property('security-plugin.version') - isSnapshot = "true" == System.getProperty("build.snapshot", "true") -} - group = 'org.opensearch' -version = securityPluginVersion -if (buildVersionQualifier) { - version += "-${buildVersionQualifier}" -} -if (isSnapshot) { - version += "-SNAPSHOT" -} +version = opensearch_build description = 'OpenSearch Security' @@ -326,16 +325,13 @@ task bundleSecurityAdminStandaloneTarGz(dependsOn: jar, type: Tar) { } task createPluginDescriptor() { - if (opensearch_version.contains("-SNAPSHOT")) { - opensearch_version=opensearch_version.substring(0, opensearch_version.length() - 9) - } List descriptorProperties = [ "description=Provide access control related features for OpenSearch", "version=${version}", "name=opensearch-security", "classname=org.opensearch.security.OpenSearchSecurityPlugin", "java.version=${java.targetCompatibility}", - "opensearch.version=${opensearch_version}", + "opensearch.version=${version_tokens[0]}", ] new File("plugin-descriptor.properties").text = descriptorProperties.join ("\n") diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index e4cbe65dcc..0000000000 --- a/gradle.properties +++ /dev/null @@ -1,4 +0,0 @@ -# Sets the version of the Security plugin -security-plugin.version=2.0.0.0 -# Sets the version of OpenSearch this plugin should be built with -opensearch-core.version=2.0.0 diff --git a/src/main/java/org/opensearch/security/compliance/ComplianceIndexingOperationListenerImpl.java b/src/main/java/org/opensearch/security/compliance/ComplianceIndexingOperationListenerImpl.java index 7ae1bdfbcf..7142a05c2e 100644 --- a/src/main/java/org/opensearch/security/compliance/ComplianceIndexingOperationListenerImpl.java +++ b/src/main/java/org/opensearch/security/compliance/ComplianceIndexingOperationListenerImpl.java @@ -149,12 +149,12 @@ public void postIndex(ShardId shardId, Index index, IndexResult result) { if(previousContent == null) { //no previous content if(!result.isCreated()) { - log.warn("No previous content and not created (its an update but do not find orig source) for {}/{}/{}/{}", index.startTime(), shardId, index.type(), index.id()); + log.warn("No previous content and not created (its an update but do not find orig source) for {}/{}/{}", index.startTime(), shardId, index.id()); } assert result.isCreated():"No previous content and not created"; } else { if(result.isCreated()) { - log.warn("Previous content and created for {}/{}/{}/{}", index.startTime(), shardId, index.type(), index.id()); + log.warn("Previous content and created for {}/{}/{}", index.startTime(), shardId, index.id()); } assert !result.isCreated():"Previous content and created"; } diff --git a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java index 585d5c750f..03858968f5 100644 --- a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java +++ b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java @@ -120,12 +120,12 @@ public void binaryFieldRead(final FieldInfo fieldInfo, byte[] fieldValue) { } } - public void stringFieldRead(final FieldInfo fieldInfo, final byte[] fieldValue) { + public void stringFieldRead(final FieldInfo fieldInfo, final String fieldValue) { try { if(!recordField(fieldInfo.name, true)) { return; } - fieldRead0(fieldInfo.name, new String(fieldValue, StandardCharsets.UTF_8)); + fieldRead0(fieldInfo.name, fieldValue); } catch (Exception e) { log.error("Unexpected error reading string field '{}' in index '{}'", fieldInfo.name, index.getName()); } diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java index 3d78dd9ac3..716b051fce 100644 --- a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java +++ b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java @@ -421,11 +421,6 @@ public void checkIntegrity() throws IOException { public void close() throws IOException { in.close(); } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } } @Override @@ -521,7 +516,7 @@ public int hashCode() { } @Override - public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException { + public void stringField(final FieldInfo fieldInfo, final String value) throws IOException { fieldReadCallback.stringFieldRead(fieldInfo, value); delegate.stringField(fieldInfo, value); } @@ -610,7 +605,7 @@ public int hashCode() { } @Override - public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException { + public void stringField(final FieldInfo fieldInfo, final String value) throws IOException { delegate.stringField(fieldInfo, value); } @@ -681,7 +676,7 @@ public int hashCode() { } @Override - public void stringField(final FieldInfo fieldInfo, final byte[] value) throws IOException { + public void stringField(final FieldInfo fieldInfo, final String value) throws IOException { final Optional mf = maskedFieldsMap.getMaskedField(fieldInfo.name); if(mf.isPresent()) { @@ -868,11 +863,6 @@ private SortedDocValues wrapSortedDocValues(final String field, final SortedDocV if (mf != null) { return new SortedDocValues() { - @Override - public BytesRef binaryValue() throws IOException { - return mf.mask(sortedDocValues.binaryValue()); - } - @Override public int lookupTerm(BytesRef key) throws IOException { return sortedDocValues.lookupTerm(key); diff --git a/src/test/java/org/opensearch/security/TracingTests.java b/src/test/java/org/opensearch/security/TracingTests.java index a5a626c671..32cdb0bbf4 100644 --- a/src/test/java/org/opensearch/security/TracingTests.java +++ b/src/test/java/org/opensearch/security/TracingTests.java @@ -62,14 +62,10 @@ public void testAdvancedMapping() throws Exception { setup(Settings.EMPTY, new DynamicSecurityConfig(), Settings.EMPTY, true, ClusterConfiguration.DEFAULT); try (Client tc = getClient()) { - tc.admin().indices().create(new CreateIndexRequest("myindex1") - .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex2") - .mapping("mytype2", FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex3") - .mapping("mytype3", FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex4") - .mapping("mytype4", FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex3").mapping(FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex4").mapping(FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet(); } RestHelper rh = nonSslRestHelper(); diff --git a/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java b/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java index f4d79f571a..60f0cde15e 100644 --- a/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java +++ b/src/test/java/org/opensearch/security/auditlog/impl/TracingTests.java @@ -338,14 +338,10 @@ public void testAdvancedMapping() throws Exception { rh.executePutRequest("_opendistro/_security/api/audit/config", AuditTestUtils.createAuditPayload(settings), encodeBasicHeader("admin", "admin")); try (Client tc = getClient()) { - tc.admin().indices().create(new CreateIndexRequest("myindex1") - .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex2") - .mapping("mytype2", FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex3") - .mapping("mytype3", FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex4") - .mapping("mytype4", FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex3").mapping(FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex4").mapping(FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet(); } System.out.println("############ write into mapping 1"); @@ -391,10 +387,8 @@ public void testImmutableIndex() throws Exception { rh.executePutRequest("_opendistro/_security/api/audit/config", AuditTestUtils.createAuditPayload(Settings.EMPTY), encodeBasicHeader("admin", "admin")); try (Client tc = getClient()) { - tc.admin().indices().create(new CreateIndexRequest("myindex1") - .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); - tc.admin().indices().create(new CreateIndexRequest("myindex2") - .mapping("mytype2", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex1").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("myindex2").mapping(FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); } System.out.println("############ immutable 1"); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java index a6b968dbdd..0cb392c66e 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedComplexMappingTest.java @@ -36,7 +36,7 @@ public class CustomFieldMaskedComplexMappingTest extends AbstractDlsFlsTest{ protected void populateData(Client tc) { try { - tc.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("logs").mapping(FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet(); byte[] data = FileHelper.loadFile("dlsfls/logs_bulk_data.json").getBytes(StandardCharsets.UTF_8); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java index 4369db6599..0dbceab386 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsNestedTest.java @@ -44,8 +44,9 @@ protected void populateData(Client tc) { ""; tc.admin().indices().create(new CreateIndexRequest("deals") - .settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build()) - .mapping("mytype", mapping, XContentType.JSON)).actionGet(); + .simpleMapping("amount", "type=integer", "owner", "type=text", "my_nested_object", "type=nested") + .settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build())) + .actionGet(); //tc.index(new IndexRequest("deals").id("3").setRefreshPolicy(RefreshPolicy.IMMEDIATE) // .source("{\"amount\": 7,\"owner\": \"a\", \"my_nested_object\" : {\"name\": \"spock\"}}", XContentType.JSON)).actionGet(); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java index 0de711caec..7c704c2bda 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java @@ -273,7 +273,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { setup(); try (Client client = getClient()) { - client.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", ImmutableMap.of("properties", ImmutableMap.of("termX", ImmutableMap.of("type", "keyword"))))).actionGet(); + client.admin().indices().create(new CreateIndexRequest("logs").simpleMapping("termX", "type=keyword")).actionGet(); for (int i = 0; i < 3; i++) { client.index(new IndexRequest("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("amount", i, "termX", "A", "timestamp", "2022-01-06T09:05:00Z")).actionGet(); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java index 103387d1f1..80d17d3ee0 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsExistsFieldsTest.java @@ -30,12 +30,8 @@ public class FlsExistsFieldsTest extends AbstractDlsFlsTest { protected void populateData(Client tc) { - tc.admin().indices().create(new CreateIndexRequest("data").mapping("doc", - "@timestamp", "type=date", - "host", "type=text,norms=false", - "response", "type=text,norms=false", - "non-existing", "type=text,norms=false" - )) + tc.admin().indices().create(new CreateIndexRequest("data") + .simpleMapping("@timestamp", "type=date", "host", "type=text,norms=false", "response", "type=text,norms=false", "non-existing", "type=text,norms=false")) .actionGet(); for (int i = 0; i < 1; i++) { diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java index b115f5d502..91e0544cb4 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsTest.java @@ -34,8 +34,7 @@ public class FlsFieldsTest extends AbstractDlsFlsTest{ protected void populateData(Client tc) { - tc.admin().indices().create(new CreateIndexRequest("deals") - .mapping("deals", "timestamp","type=date","@timestamp","type=date")).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("deals").simpleMapping("timestamp", "type=date", "@timestamp", "type=date")).actionGet(); try { String doc = FileHelper.loadFile("dlsfls/doc1.json"); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java index 5063eb3f25..1a6db9ec02 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FlsFieldsWcTest.java @@ -34,8 +34,7 @@ public class FlsFieldsWcTest extends AbstractDlsFlsTest{ protected void populateData(Client tc) { - tc.admin().indices().create(new CreateIndexRequest("deals") - .mapping("deals", "timestamp","type=date","@timestamp","type=date")).actionGet(); + tc.admin().indices().create(new CreateIndexRequest("deals").simpleMapping("timestamp", "type=date", "@timestamp", "type=date")).actionGet(); try { String doc = FileHelper.loadFile("dlsfls/doc1.json");