Merge branch 'master' into dynamic-menu

docs/en/changes/changes.md

@@ -25,6 +25,8 @@
 * Fix ElasticSearch scroller bug.
 * Add component ID for Aerospike(ID=149).
 * Packages with name `recevier` are renamed to `receiver`.
+* `BanyanDBMetricsDAO` handles `storeIDTag` in `multiGet` for `BanyanDBModelExtension`.
+* Fix endpoint grouping-related logic and enhance the performance of PatternTree retrieval. 
 * Support dynamic UI menu query.
 
 #### UI

oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java

@@ -68,14 +68,19 @@ public List<HttpUriPattern> fetchAllPatterns(final String service) {
                                                        .build()
                       );
             final String newVersion = httpUriRecognitionResponse.getVersion();
+            if (log.isDebugEnabled()) {
+                log.debug("Remote response version: {}, local version {}", newVersion, version);
+            }
             if (version.equals(newVersion)) {
-                // Same version, nothing changed.
                 return null;
+            } else {
+                version = newVersion;
             }
+
             return httpUriRecognitionResponse.getPatternsList()
-                                             .stream()
-                                             .map(pattern -> new HttpUriPattern(pattern.getPattern()))
-                                             .collect(Collectors.toList());
+                    .stream()
+                    .map(pattern -> new HttpUriPattern(pattern.getPattern()))
+                    .collect(Collectors.toList());
 
         } catch (Exception e) {
             log.error("fetch all patterns failed from remote server.", e);
@@ -91,15 +96,20 @@ public void feedRawData(final String service, final List<HTTPUri> unrecognizedUR
             }
             final HttpUriRecognitionRequest.Builder builder = HttpUriRecognitionRequest.newBuilder();
             builder.setService(service);
-            unrecognizedURIs.forEach(httpUri -> {
-                builder.getUnrecognizedUrisBuilderList().add(
-                    HttpRawUri.newBuilder().setName(httpUri.getName())
-                );
-            });
+            if (log.isDebugEnabled()) {
+                log.debug("Feeding to remote, service: {}, uri size: {}", service, unrecognizedURIs.size());
+            }
+            List<HttpRawUri> rawUriList = unrecognizedURIs.stream()
+                    .map(httpUri -> HttpRawUri.newBuilder()
+                            .setName(httpUri.getName())
+                            .build())
+                    .collect(Collectors.toList());
+
+            builder.addAllUnrecognizedUris(rawUriList);
             stub.withDeadlineAfter(30, TimeUnit.SECONDS)
-                .feedRawData(builder.build());
+                    .feedRawData(builder.build());
         } catch (Exception e) {
-            log.error("feed matched and unmatched URIs to the remote server.", e);
+            log.error("Failed to feed matched and unmatched URIs to the remote server.", e);
         }
     }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java

@@ -97,7 +97,7 @@ public Tuple2<String, Boolean> format(String serviceName, String endpointName) {
             if (svrHttpUris.size() < maxHttpUrisNumberPerService) {
                 if (formattedName._2()) {
                     // Algorithm side should not return a pattern that has no {var} in it else this
-                    // code may accidentally retreive the size 1 queue created by unformatted endpoint
+                    // code may accidentally retrieve the size 1 queue created by unformatted endpoint
                     // The queue size is 10, which means only cache the first 10 formatted names.
                     final ArrayBlockingQueue<String> formattedURIs = svrHttpUris.computeIfAbsent(
                         formattedName._1(), k -> new ArrayBlockingQueue<>(10));
@@ -140,7 +140,7 @@ private Tuple2<String, Boolean> formatByOpenapi(String serviceName, String endpo
                 log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName);
             }
         }
-        return new Tuple2<>(formatResult.getName(), formatResult.isMatch());
+        return new Tuple2<>(formatResult.getReplacedName(), formatResult.isMatch());
     }
 
     private Tuple2<String, Boolean> formatByQuickUriPattern(String serviceName, String endpointName) {
@@ -155,7 +155,7 @@ private Tuple2<String, Boolean> formatByQuickUriPattern(String serviceName, Stri
                 log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName);
             }
         }
-        return new Tuple2<>(formatResult.getName(), formatResult.isMatch());
+        return new Tuple2<>(formatResult.getReplacedName(), formatResult.isMatch());
     }
 
     public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr,
@@ -172,7 +172,8 @@ public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecogniti
                  .scheduleWithFixedDelay(
                      new RunnableWithExceptionProtection(
                          () -> {
-                             if (aiPipelineExecutionCounter.incrementAndGet() % trainingPeriodHttpUriRecognitionPattern == 0) {
+                             int currentExecutionCounter = aiPipelineExecutionCounter.incrementAndGet();
+                             if (currentExecutionCounter % trainingPeriodHttpUriRecognitionPattern == 0) {
                                  // Send the cached URIs to the recognition server to build new patterns.
                                  cachedHttpUris.forEach((serviceName, httpUris) -> {
                                      final List<HttpUriRecognition.HTTPUri> candidates4UriPatterns = new ArrayList<>(
@@ -182,7 +183,7 @@ public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecogniti
                                              //unrecognized uri
                                              candidates4UriPatterns.add(new HttpUriRecognition.HTTPUri(uri));
                                          } else {
-                                             String candidateUri = null;
+                                             String candidateUri;
                                              while ((candidateUri = candidates.poll()) != null) {
                                                  candidates4UriPatterns.add(
                                                      new HttpUriRecognition.HTTPUri(candidateUri));
@@ -195,7 +196,7 @@ public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecogniti
                                      httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
                                  });
                              }
-                             if (aiPipelineExecutionCounter.incrementAndGet() % syncPeriodHttpUriRecognitionPattern == 0) {
+                             if (currentExecutionCounter % syncPeriodHttpUriRecognitionPattern == 0) {
                                  // Sync with the recognition server per 1 min to get the latest patterns.
                                  try {
                                      metadataQueryService.listServices(null, null).forEach(

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java

@@ -44,7 +44,7 @@ void addGroupedRule(String serviceName, String endpointGroupName, String ruleReg
     public StringFormatGroup.FormatResult format(String service, String endpointName) {
         Map<String, String> endpointNameLookup = directLookup.get(service);
         if (endpointNameLookup != null && endpointNameLookup.get(endpointName) != null) {
-            return new StringFormatGroup.FormatResult(true, endpointNameLookup.get(endpointName), endpointName);
+            return new StringFormatGroup.FormatResult(true, endpointName, endpointNameLookup.get(endpointName));
         }
 
         Map<String, StringFormatGroup> rules = groupedRules.get(service);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/uri/quickmatch/PatternTree.java

@@ -40,28 +40,28 @@ public PatternTree() {
      * @param pattern of URIs
      */
     public void addPattern(String pattern) {
-        final String[] tokens = pattern.split("/");
+        final List<String> tokens = splitByCharacter(pattern);
 
         PatternToken current = null;
         for (final PatternToken patternToken : roots) {
-            if (patternToken.isMatch(tokens[0])) {
+            if (patternToken.isMatch(tokens.get(0))) {
                 current = patternToken;
                 break;
             }
         }
 
         if (current == null) {
-            current = new StringToken(tokens[0]);
+            current = new StringToken(tokens.get(0));
             roots.add(current);
         }
 
-        if (tokens.length == 1) {
+        if (tokens.size() == 1) {
             current.setExpression(pattern);
             return;
         }
 
-        for (int i = 1; i < tokens.length; i++) {
-            final String token = tokens[i];
+        for (int i = 1; i < tokens.size(); i++) {
+            final String token = tokens.get(i);
             PatternToken newToken;
             if (VarToken.VAR_TOKEN.equals(token)) {
                 newToken = new VarToken();
@@ -78,27 +78,38 @@ public void addPattern(String pattern) {
         current.setExpression(pattern);
     }
 
-    List<String> splitByCharacter(String input, char delimiter) {
+    List<String> splitByCharacter(String input) {
         List<String> parts = new ArrayList<>();
+        int length = input.length();
         int start = 0;
 
-        for (int i = 0; i < input.length(); i++) {
-            if (input.charAt(i) == delimiter) {
+        for (int i = 0; i < length; i++) {
+            if (input.charAt(i) == '/') {
+                if (i == 0) {
+                    start = i + 1;
+                    continue;
+                }
                 parts.add(input.substring(start, i));
                 start = i + 1;
             }
         }
 
         // Add the last part if necessary
-        if (start < input.length()) {
+        if (start < length) {
             parts.add(input.substring(start));
         }
 
         return parts;
     }
 
     public StringFormatGroup.FormatResult match(String uri) {
-        final List<String> slices = splitByCharacter(uri, '/');
+        final List<String> slices = splitByCharacter(uri);
+        if (slices.size() == 1) {
+            // Special case handling, since if a URI is just length one
+            // itself will never be a variable, so simply return true and itself
+            // trailing slashes, if ever encountered will be kept as is
+            return new StringFormatGroup.FormatResult(true, uri, uri);
+        }
         List<PatternToken> current = roots;
         PatternToken matchedToken = null;
         for (final String slice : slices) {
@@ -111,14 +122,52 @@ public StringFormatGroup.FormatResult match(String uri) {
                 }
             }
             if (!matched) {
-                return new StringFormatGroup.FormatResult(false, uri, null);
+                return new StringFormatGroup.FormatResult(false, uri, uri);
             }
             current = matchedToken.children();
         }
         if (matchedToken.isLeaf()) {
             return new StringFormatGroup.FormatResult(true, uri, matchedToken.expression());
         } else {
-            return new StringFormatGroup.FormatResult(false, uri, null);
+            return new StringFormatGroup.FormatResult(false, uri, uri);
+        }
+    }
+
+    @SuppressWarnings("unused")
+    // Utility method to visualize the full tree for debugging purposes
+    public String printTree() {
+        StringBuilder sb = new StringBuilder();
+        for (PatternToken root : roots) {
+            sb.append(printNode(root, 0));
         }
+        return sb.toString();
     }
+
+    private String printNode(PatternToken node, int depth) {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("  ".repeat(Math.max(0, depth)));
+
+        sb.append(node.toString()).append("\n");
+
+        // Append expression if not null
+        if (node.expression() != null) {
+            sb.append("  ").append(node.expression()).append("\n");
+        }
+
+        if (node instanceof StringToken) {
+            StringToken stringToken = (StringToken) node;
+            for (PatternToken child : stringToken.children()) {
+                sb.append(printNode(child, depth + 1));
+            }
+        } else if (node instanceof VarToken) {
+            VarToken varToken = (VarToken) node;
+            for (PatternToken child : varToken.children()) {
+                sb.append(printNode(child, depth + 1));
+            }
+        }
+
+        return sb.toString();
+    }
+
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/uri/quickmatch/StringToken.java

@@ -22,10 +22,8 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
-import lombok.ToString;
 
 @EqualsAndHashCode(of = "value")
-@ToString
 public class StringToken implements PatternToken {
     private final String value;
     private final List<PatternToken> children;
@@ -56,4 +54,9 @@ public String expression() {
     public List<PatternToken> children() {
         return children;
     }
+
+    @Override
+    public String toString() {
+        return "StringToken: \"" + value + "\"";
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/uri/quickmatch/VarToken.java

@@ -22,10 +22,8 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
-import lombok.ToString;
 
 @EqualsAndHashCode(of = "")
-@ToString
 public class VarToken implements PatternToken {
     public static final String VAR_TOKEN = "{var}";
     private List<PatternToken> children = new ArrayList<>();
@@ -54,4 +52,9 @@ public String expression() {
     public List<PatternToken> children() {
         return children;
     }
+
+    @Override
+    public String toString() {
+        return "VarToken: \"" + VAR_TOKEN + "\"";
+    }
 }

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/EndpointGroupingRuleReaderTest.java

@@ -36,8 +36,14 @@ public void testReadingRule() {
         Assertions.assertTrue(formatResult.isMatch());
         Assertions.assertEquals("/prod/{var}", formatResult.getReplacedName());
 
+        // This will always match, since after slicing length is 1, which goes into special handling
         formatResult = rule.format("serviceA", "/prod/");
+        Assertions.assertTrue(formatResult.isMatch());
+        Assertions.assertEquals("/prod/", formatResult.getReplacedName());
+
+        formatResult = rule.format("serviceA", "/prod/123/456");
         Assertions.assertFalse(formatResult.isMatch());
+        Assertions.assertEquals("/prod/123/456", formatResult.getReplacedName());
 
         formatResult = rule.format("serviceB", "/prod/123");
         Assertions.assertFalse(formatResult.isMatch());