BanyanDB: support using native term searching for keyword in query …

…`findEndpoint` and `getAlarm`. (#12633)

docs/en/changes/changes.md

@@ -69,6 +69,7 @@
 * Add SkyWalking Java Agent self observability dashboard.
 * Add Component ID(5022) for the GoFrame framework.
 * Bump up protobuf java dependencies to 3.25.5.
+* BanyanDB: support using native term searching for `keyword` in query `findEndpoint` and `getAlarm`.
 
 #### UI
 

oap-server-bom/pom.xml

@@ -73,7 +73,7 @@
         <httpcore.version>4.4.13</httpcore.version>
         <httpasyncclient.version>4.1.5</httpasyncclient.version>
         <commons-compress.version>1.21</commons-compress.version>
-        <banyandb-java-client.version>0.7-rc3</banyandb-java-client.version>
+        <banyandb-java-client.version>0.7.0-rc4</banyandb-java-client.version>
         <kafka-clients.version>3.4.0</kafka-clients.version>
         <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
         <consul.client.version>1.5.3</consul.client.version>

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java

@@ -82,6 +82,7 @@ public StorageID id() {
     private long startTime;
     @Column(name = ALARM_MESSAGE, length = 512)
     @ElasticSearch.MatchQuery
+    @BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
     private String alarmMessage;
     @Column(name = RULE_NAME)
     private String ruleName;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java

@@ -59,6 +59,7 @@ public class EndpointTraffic extends Metrics {
     @Column(name = NAME)
     @ElasticSearch.Column(legacyName = "name")
     @ElasticSearch.MatchQuery
+    @BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.URL)
     @BanyanDB.SeriesID(index = 1)
     private String name = Const.EMPTY_STRING;
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java

@@ -177,4 +177,37 @@ enum IndexType {
          */
         int countersNumber() default 1000;
     }
+
+    /**
+     * Match query is designed for BanyanDB match query with specific analyzer. It is a fuzzy query implementation
+     * powered by analyzer.
+     *
+     * @since 10.1.0
+     */
+    @Target({ElementType.FIELD})
+    @Retention(RetentionPolicy.RUNTIME)
+    @interface MatchQuery {
+        AnalyzerType analyzer();
+
+        enum AnalyzerType {
+            /**
+             * Keyword analyzer is a “noop” analyzer which returns the entire input string as a single token.
+             */
+            KEYWORD,
+            /**
+             * Standard analyzer provides grammar based tokenization
+             */
+            STANDARD,
+            /**
+             * Simple analyzer breaks text into tokens at any non-letter character,
+             * such as numbers, spaces, hyphens and apostrophes, discards non-letter characters,
+             * and changes uppercase to lowercase.
+             */
+            SIMPLE,
+            /**
+             * URL analyzer breaks test into tokens at any non-letter and non-digit character
+             */
+            URL
+        }
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java

@@ -61,6 +61,12 @@ public class BanyanDBExtension {
     @Getter
     private final boolean isMeasureField;
 
+    /**
+     * The analyzer policy appointed to fuzzy query, especially for BanyanDB.
+     */
+    @Getter
+    private final BanyanDB.MatchQuery.AnalyzerType analyzer;
+
     /**
      * @return true if this column is a part of sharding key
      */

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java

@@ -99,6 +99,11 @@ public ModelColumn(ColumnName columnName,
         }
         this.indexOnly = indexOnly;
         this.banyanDBExtension = banyanDBExtension;
+
+        if (!this.banyanDBExtension.shouldIndex() && this.banyanDBExtension.getAnalyzer() != null) {
+            throw new IllegalArgumentException(
+                "The column " + columnName + " should be indexed if require MatchQuery.");
+        }
     }
 
     /**

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java

@@ -215,11 +215,15 @@ private void retrieval(final Class<?> clazz,
                     BanyanDB.MeasureField.class);
                 final BanyanDB.TopNAggregation topNAggregation = field.getAnnotation(
                     BanyanDB.TopNAggregation.class);
+                final BanyanDB.MatchQuery analyzer = field.getAnnotation(
+                    BanyanDB.MatchQuery.class);
+                final boolean shouldIndex = (banyanDBNoIndex == null) && !column.storageOnly();
                 BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
                     banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(),
-                    banyanDBNoIndex == null && !column.storageOnly(),
+                    shouldIndex,
                     banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(),
-                    banyanDBMeasureField != null
+                    banyanDBMeasureField != null,
+                    analyzer == null ? null : analyzer.analyzer()
                 );
 
                 if (topNAggregation != null) {

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java

@@ -49,7 +49,7 @@ public void testColumnDefine() {
                                              new SQLDatabaseExtension(),
                                              new ElasticSearchExtension(
                                                  ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
-                                             new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
+                                             new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
         );
         Assertions.assertTrue(column.isStorageOnly());
         Assertions.assertEquals("abc", column.getColumnName().getName());
@@ -58,7 +58,7 @@ public void testColumnDefine() {
                                  false, false, true, 200,
                                  new SQLDatabaseExtension(),
                                  new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
-                                 new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
+                                 new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB. MatchQuery.AnalyzerType.SIMPLE)
         );
         Assertions.assertTrue(column.isStorageOnly());
         Assertions.assertEquals("abc", column.getColumnName().getName());
@@ -68,7 +68,7 @@ public void testColumnDefine() {
                                  false, false, true, 200,
                                  new SQLDatabaseExtension(),
                                  new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
-                                 new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
+                                 new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
         );
         Assertions.assertFalse(column.isStorageOnly());
         Assertions.assertEquals("abc", column.getColumnName().getName());
@@ -82,7 +82,7 @@ public void testConflictDefinition() {
                     new SQLDatabaseExtension(),
                     new ElasticSearchExtension(
                             ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
-                    new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
+                    new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
             );
         });
     }
@@ -95,7 +95,23 @@ public void testConflictDefinitionIndexOnly() {
                     new SQLDatabaseExtension(),
                     new ElasticSearchExtension(
                             ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
-                    new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
+                    new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
+            );
+        });
+    }
+
+    @Test
+    public void testConflictDefinitionStorageOnly() {
+        assertThrows(IllegalArgumentException.class, () -> {
+            new ModelColumn(new ColumnName(c), String.class, String.class,
+                            true, false, false, 200,
+                            new SQLDatabaseExtension(),
+                            new ElasticSearchExtension(
+                                ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
+                            new BanyanDBExtension(
+                                -1, false, BanyanDB.IndexRule.IndexType.INVERTED, false,
+                                BanyanDB.MatchQuery.AnalyzerType.SIMPLE
+                            )
             );
         });
     }

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java

@@ -79,6 +79,7 @@
 import org.apache.skywalking.oap.server.core.config.ConfigService;
 import org.apache.skywalking.oap.server.core.query.enumeration.Step;
 import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
 import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
 import org.apache.skywalking.oap.server.core.storage.model.Model;
@@ -167,7 +168,7 @@ public MeasureModel registerMeasureModel(Model model, BanyanDBStorageConfig conf
                 .collect(Collectors.toList());
 
         if (model.getBanyanDBModelExtension().isStoreIDTag()) {
-            indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID));
+            indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, null));
         }
 
         final Measure.Builder builder = Measure.newBuilder();
@@ -299,10 +300,29 @@ Duration downSamplingDuration(DownSampling downSampling) {
         }
     }
 
-    IndexRule indexRule(String group, String tagName) {
-        return IndexRule.newBuilder()
-                        .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
-                        .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName).build();
+    IndexRule indexRule(String group, String tagName, BanyanDB.MatchQuery.AnalyzerType analyzer) {
+        IndexRule.Builder builder = IndexRule.newBuilder()
+                                             .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
+                                             .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName);
+        if (analyzer != null) {
+            switch (analyzer) {
+                case KEYWORD:
+                    builder.setAnalyzer("keyword");
+                    break;
+                case STANDARD:
+                    builder.setAnalyzer("standard");
+                    break;
+                case SIMPLE:
+                    builder.setAnalyzer("simple");
+                    break;
+                case URL:
+                    builder.setAnalyzer("url");
+                    break;
+                default:
+                    throw new UnsupportedOperationException("unsupported analyzer type: " + analyzer);
+            }
+        }
+        return builder.build();
     }
 
     /**
@@ -341,8 +361,13 @@ List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder, Li
             final TagSpec tagSpec = parseTagSpec(col);
             builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
             String colName = col.getColumnName().getStorageName();
-            if (!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex()) {
-                tagMetadataList.add(new TagMetadata(indexRule(group, tagSpec.getName()), tagSpec));
+            if (col.getBanyanDBExtension().shouldIndex()) {
+                if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
+                    tagMetadataList.add(new TagMetadata(
+                        indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
+                } else {
+                    tagMetadataList.add(new TagMetadata(null, tagSpec));
+                }
             } else {
                 tagMetadataList.add(new TagMetadata(null, tagSpec));
             }
@@ -382,7 +407,17 @@ MeasureMetadata parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder build
             final TagSpec tagSpec = parseTagSpec(col);
             builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
             String colName = col.getColumnName().getStorageName();
-            result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex() ? indexRule(group, tagSpec.getName()) : null, tagSpec));
+
+            if (col.getBanyanDBExtension().shouldIndex()) {
+                if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
+                    result.tag(new TagMetadata(
+                        indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
+                } else {
+                    result.tag(new TagMetadata(null, tagSpec));
+                }
+            } else {
+                result.tag(new TagMetadata(null, tagSpec));
+            }
         }
 
         return result.build();

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java

@@ -24,6 +24,7 @@
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.skywalking.banyandb.model.v1.BanyandbModel;
 import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
 import org.apache.skywalking.banyandb.v1.client.AbstractQuery;
 import org.apache.skywalking.banyandb.v1.client.DataPoint;
@@ -186,6 +187,12 @@ protected void apply(MeasureQuery query) {
                         if (StringUtil.isNotEmpty(serviceId)) {
                             query.and(eq(EndpointTraffic.SERVICE_ID, serviceId));
                         }
+                        if (StringUtil.isNotEmpty(keyword)) {
+                            query.and(match(EndpointTraffic.NAME, keyword,
+                                            BanyandbModel.Condition.MatchOption.newBuilder().setOperator(
+                                                BanyandbModel.Condition.MatchOption.Operator.OPERATOR_AND).build()
+                            ));
+                        }
                         query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC));
                     }
                 });
@@ -194,10 +201,6 @@ protected void apply(MeasureQuery query) {
         for (final DataPoint dataPoint : resp.getDataPoints()) {
             endpoints.add(buildEndpoint(dataPoint, schema));
         }
-
-        if (StringUtil.isNotEmpty(keyword)) {
-            return endpoints.stream().filter(e -> e.getName().contains(keyword)).collect(Collectors.toList());
-        }
         return endpoints;
     }
 

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream;
 
 import com.google.gson.Gson;
+import org.apache.skywalking.banyandb.model.v1.BanyandbModel;
 import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
 import org.apache.skywalking.banyandb.v1.client.AbstractQuery;
 import org.apache.skywalking.banyandb.v1.client.And;
@@ -297,6 +298,14 @@ protected PairQueryCondition<String> eq(String name, String value) {
             return PairQueryCondition.StringQueryCondition.eq(name, value);
         }
 
+        protected PairQueryCondition<String> match(String name, String value) {
+            return PairQueryCondition.StringQueryCondition.match(name, value);
+        }
+
+        protected PairQueryCondition<String> match(String name, String value, BanyandbModel.Condition.MatchOption matchOption) {
+            return PairQueryCondition.StringQueryCondition.match(name, value, matchOption);
+        }
+
         protected PairQueryCondition<List<String>> in(String name, List<String> values) {
             return PairQueryCondition.StringArrayQueryCondition.in(name, values);
         }

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java

@@ -32,6 +32,7 @@
 import org.apache.skywalking.oap.server.core.query.type.Alarms;
 import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO;
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
 import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter;
 import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient;
 
@@ -71,7 +72,9 @@ public void apply(StreamQuery query) {
                         if (Objects.nonNull(scopeId)) {
                             query.and(eq(AlarmRecord.SCOPE, (long) scopeId));
                         }
-
+                        if (StringUtil.isNotEmpty(keyword)) {
+                            query.and(match(AlarmRecord.ALARM_MESSAGE, keyword));
+                        }
                         if (CollectionUtils.isNotEmpty(tags)) {
                             List<String> tagsConditions = new ArrayList<>(tags.size());
                             for (final Tag tag : tags) {

test/e2e-v2/script/env

@@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
 SW_KUBERNETES_COMMIT_SHA=1335f15bf821a40a7cd71448fa805f0be265afcc
 SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c
-SW_BANYANDB_COMMIT=0e734c462571dcf55dbb7761211c07d8b156521e
+SW_BANYANDB_COMMIT=9770bf48f9b88574683f56e2a137555cd68066d9
 SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3
 
 SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468