Merge branch 'main' into tuples-join-console-lib

opensearch-project · Sep 19, 2024 · 94927bd · 94927bd
2 parents c157cd0 + d2dce08
commit 94927bd
Show file tree

Hide file tree

Showing 102 changed files with 2,239 additions and 1,395 deletions.
diff --git a/.github/workflows/rfs_pr_e2e_test.yml b/.github/workflows/rfs_pr_e2e_test.yml
@@ -0,0 +1,40 @@
+name: Jenkins
+
+on:
+  push:
+    branches-ignore:
+      - 'backport/**'
+      - 'dependabot/**'
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+env:
+  python-version: '3.11'
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  rfs-e2e-aws-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Determine Github repository and branch
+        id: determine-repo-vars
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
+            branch_name="${{ github.event.pull_request.head.ref }}"
+            pr_repo_url="https://github.com/${{ github.event.pull_request.head.repo.full_name }}.git"
+          else
+            branch_name="${{ github.ref_name }}"
+            pr_repo_url="https://github.com/${{ github.repository }}.git"
+          fi
+          echo "Running jenkins test on repo: $pr_repo_url and branch: $branch_name"
+          echo "branch_name=$branch_name" >> $GITHUB_OUTPUT
+          echo "pr_repo_url=$pr_repo_url" >> $GITHUB_OUTPUT
+      - name: Jenkins Job Trigger and Monitor
+        uses: lewijacn/jenkins-trigger@1.0.4
+        with:
+          jenkins_url: "https://migrations.ci.opensearch.org"
+          job_name: "rfs-default-e2e-test"
+          api_token: "${{ secrets.JENKINS_MIGRATIONS_GENERIC_WEBHOOK_TOKEN }}"
+          job_params: "GIT_REPO_URL=${{ steps.determine-repo-vars.outputs.pr_repo_url }},GIT_BRANCH=${{ steps.determine-repo-vars.outputs.branch_name }}"
diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle
@@ -18,7 +18,8 @@ dependencies {
     implementation group: 'org.slf4j', name: 'slf4j-api'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl'
 
-    testImplementation testFixtures(project(path: ':RFS'))
+    testImplementation testFixtures(project(':RFS'))
+    testImplementation testFixtures(project(':testHelperFixtures'))
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core'
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl'
     testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api'

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java
@@ -77,7 +77,7 @@ public static void main(String[] args) throws Exception {
                 result = meta.evaluate(evaluateArgs).execute(context);
                 break;
         }
-        log.info(result.toString());
+        log.atInfo().setMessage("{}").addArgument(result::asCliOutput).log();
         System.exit(result.getExitCode());
     }
 
@@ -94,18 +94,20 @@ public Migrate migrate(MigrateOrEvaluateArgs arguments) {
     }
 
     private static void printTopLevelHelp(JCommander commander) {
-        log.info("Usage: [options] [command] [commandOptions]");
-        log.info("Options:");
+        var sb = new StringBuilder();
+        sb.append("Usage: [options] [command] [commandOptions]");
+        sb.append("Options:");
         for (var parameter : commander.getParameters()) {
-            log.info("  " + parameter.getNames());
-            log.info("    " + parameter.getDescription());
+            sb.append("  " + parameter.getNames());
+            sb.append("    " + parameter.getDescription());
         }
 
-        log.info("Commands:");
+        sb.append("Commands:");
         for (var command : commander.getCommands().entrySet()) {
-            log.info("  " + command.getKey());
+            sb.append("  " + command.getKey());
         }
-        log.info("\nUse --help with a specific command for more information.");
+        sb.append("\nUse --help with a specific command for more information.");
+        log.info(sb.toString());
     }
 
     private static void printCommandUsage(JCommander jCommander) {

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/cli/ClusterReaderExtractor.java b/MetadataMigration/src/main/java/org/opensearch/migrations/cli/ClusterReaderExtractor.java
@@ -3,6 +3,7 @@
 import java.nio.file.Path;
 
 import org.opensearch.migrations.MigrateOrEvaluateArgs;
+import org.opensearch.migrations.Version;
 import org.opensearch.migrations.cluster.ClusterProviderRegistry;
 import org.opensearch.migrations.cluster.ClusterReader;
 
@@ -11,22 +12,23 @@
 import com.rfs.common.S3Repo;
 import com.rfs.common.S3Uri;
 import com.rfs.common.SourceRepo;
+import com.rfs.common.http.ConnectionContext;
 import lombok.AllArgsConstructor;
 
 @AllArgsConstructor
 public class ClusterReaderExtractor {
-    final MigrateOrEvaluateArgs arguments;
+    private final MigrateOrEvaluateArgs arguments;
 
-    public ClusterReader extractClusterReader() { 
-        if (arguments.fileSystemRepoPath != null && arguments.s3RepoUri != null && arguments.sourceArgs.host != null) {
+    public ClusterReader extractClusterReader() {
+        if (arguments.fileSystemRepoPath == null && arguments.s3RepoUri == null && arguments.sourceArgs.host == null) {
             throw new ParameterException("No details on the source cluster found, please supply a connection details or a snapshot");
         }
         if ((arguments.s3RepoUri != null) && (arguments.s3Region == null || arguments.s3LocalDirPath == null)) {
             throw new ParameterException("If an s3 repo is being used, s3-region and s3-local-dir-path must be set");
         }
 
         if (arguments.sourceArgs != null && arguments.sourceArgs.host != null) {
-            return ClusterProviderRegistry.getRemoteReader(arguments.sourceArgs.toConnectionContext());
+            return getRemoteReader(arguments.sourceArgs.toConnectionContext());
         }
 
         SourceRepo repo = null;
@@ -38,6 +40,14 @@ public ClusterReader extractClusterReader() {
             throw new ParameterException("Unable to find valid resource provider");
         }
 
-        return ClusterProviderRegistry.getSnapshotReader(arguments.sourceVersion, repo);
+        return getSnapshotReader(arguments.sourceVersion, repo);
+    }
+
+    ClusterReader getRemoteReader(ConnectionContext connection) {
+        return ClusterProviderRegistry.getRemoteReader(connection);
+    }
+
+    ClusterReader getSnapshotReader(Version sourceVersion, SourceRepo repo) {
+        return ClusterProviderRegistry.getSnapshotReader(sourceVersion, repo);
     }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Clusters.java b/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Clusters.java
@@ -14,17 +14,17 @@ public class Clusters {
     private ClusterReader source;
     private ClusterWriter target;
 
-    public String toString() {
+    public String asCliOutput() {
         var sb = new StringBuilder();
         sb.append("Clusters:" + System.lineSeparator());
         if (getSource() != null) {
-            sb.append("   Source:" + System.lineSeparator());
-            sb.append("      " + getSource() + System.lineSeparator());
+            sb.append(Format.indentToLevel(1) + "Source:" + System.lineSeparator());
+            sb.append(Format.indentToLevel(2) + getSource() + System.lineSeparator());
             sb.append(System.lineSeparator());
         }
         if (getTarget() != null) {
-            sb.append("   Target:" + System.lineSeparator());
-            sb.append("      " + getTarget() + System.lineSeparator());
+            sb.append(Format.indentToLevel(1) + "Target:" + System.lineSeparator());
+            sb.append(Format.indentToLevel(2) + getTarget() + System.lineSeparator());
             sb.append(System.lineSeparator());
         }
         return sb.toString();

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Format.java b/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Format.java
@@ -0,0 +1,15 @@
+package org.opensearch.migrations.cli;
+
+import lombok.experimental.UtilityClass;
+
+/** Shared formatting for command line interface components */
+@UtilityClass
+public class Format {
+
+    private static final String INDENT = "   ";
+
+    /** Indents to a given level for printing to the console */
+    public static String indentToLevel(final int level) {
+        return INDENT.repeat(level);
+    }
+}
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Items.java b/MetadataMigration/src/main/java/org/opensearch/migrations/cli/Items.java
@@ -5,44 +5,50 @@
 
 import lombok.Builder;
 import lombok.Data;
+import lombok.NonNull;
 
 /**
  * Either items that are candidates for migration or have been migrated;
  */
 @Builder
 @Data
 public class Items {
-    public boolean dryRun;
-    public List<String> indexTemplates;
-    public List<String> componentTemplates;
-    public List<String> indexes;
-    public List<String> aliases;
+    static final String NONE_FOUND_MARKER = "<NONE FOUND>";
+    private final boolean dryRun;
+    @NonNull
+    private final List<String> indexTemplates;
+    @NonNull
+    private final List<String> componentTemplates;
+    @NonNull
+    private final List<String> indexes;
+    @NonNull
+    private final List<String> aliases;
 
-    public String toString() {
+    public String asCliOutput() {
         var sb = new StringBuilder();
         if (isDryRun()) {
             sb.append("Migration Candidates:" + System.lineSeparator());
         } else {
             sb.append("Migrated Items:" + System.lineSeparator());
         }
-        sb.append("   Index Templates:" + System.lineSeparator());
-        sb.append("      " + getPrintableList(getIndexTemplates()) + System.lineSeparator());
+        sb.append(Format.indentToLevel(1) + "Index Templates:" + System.lineSeparator());
+        sb.append(Format.indentToLevel(2) + getPrintableList(getIndexTemplates()) + System.lineSeparator());
         sb.append(System.lineSeparator());
-        sb.append("   Component Templates:" + System.lineSeparator());
-        sb.append("      " + getPrintableList(getComponentTemplates()) + System.lineSeparator());
+        sb.append(Format.indentToLevel(1) + "Component Templates:" + System.lineSeparator());
+        sb.append(Format.indentToLevel(2) +getPrintableList(getComponentTemplates()) + System.lineSeparator());
         sb.append(System.lineSeparator());
-        sb.append("   Indexes:" + System.lineSeparator());
-        sb.append("      " + getPrintableList(getIndexes()) + System.lineSeparator());
+        sb.append(Format.indentToLevel(1) + "Indexes:" + System.lineSeparator());
+        sb.append(Format.indentToLevel(2) + getPrintableList(getIndexes()) + System.lineSeparator());
         sb.append(System.lineSeparator());
-        sb.append("   Aliases:" + System.lineSeparator());
-        sb.append("      " + getPrintableList(getAliases()) + System.lineSeparator());
+        sb.append(Format.indentToLevel(1) + "Aliases:" + System.lineSeparator());
+        sb.append(Format.indentToLevel(2) +getPrintableList(getAliases()) + System.lineSeparator());
         sb.append(System.lineSeparator());
         return sb.toString();
     }
 
     private String getPrintableList(List<String> list) {
         if (list == null || list.isEmpty()) {
-            return "<NONE FOUND>";
+            return NONE_FOUND_MARKER;
         }
         return list.stream().sorted().collect(Collectors.joining(", "));
     }

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Configure.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Configure.java
@@ -6,7 +6,8 @@
 public class Configure {
 
     public ConfigureResult execute() {
-        log.atError().setMessage("configure is not supported").log();
-        return new ConfigureResult(9999);
+        var message = "configure is not supported";
+        log.atError().setMessage(message).log();
+        return new ConfigureResult(9999, message);
     }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/ConfigureResult.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/ConfigureResult.java
@@ -2,9 +2,18 @@
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.ToString;
 
 @AllArgsConstructor
+@ToString
 public class ConfigureResult implements Result {
     @Getter
     private final int exitCode;
+
+    @Getter
+    private final String errorMessage;
+
+    public String asCliOutput() {
+        return this.toString();
+    }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/EvaluateResult.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/EvaluateResult.java
@@ -1,7 +1,5 @@
 package org.opensearch.migrations.commands;
 
-import org.apache.logging.log4j.util.Strings;
-
 import org.opensearch.migrations.cli.Clusters;
 import org.opensearch.migrations.cli.Items;
 
@@ -15,23 +13,4 @@ public class EvaluateResult implements MigrationItemResult {
     private final Items items;
     private final String errorMessage;
     private final int exitCode;
-
-    public String toString() {
-        var sb = new StringBuilder();
-        if (getClusters() != null) {
-            sb.append(getClusters() + System.lineSeparator());
-        }
-        if (getItems() != null) {
-            sb.append(getItems() + System.lineSeparator());
-        }
-        sb.append("Results:" + System.lineSeparator());
-        if (Strings.isNotBlank(getErrorMessage())) {
-            sb.append("   Issue(s) detected" + System.lineSeparator());
-            sb.append("Issues:" + System.lineSeparator());
-            sb.append("   " + getErrorMessage() + System.lineSeparator());
-        } else {
-            sb.append("   " + getExitCode() + " issue(s) detected" + System.lineSeparator());
-        }
-        return sb.toString();
-    }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigrateResult.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigrateResult.java
@@ -1,7 +1,5 @@
 package org.opensearch.migrations.commands;
 
-import org.apache.logging.log4j.util.Strings;
-
 import org.opensearch.migrations.cli.Clusters;
 import org.opensearch.migrations.cli.Items;
 
@@ -15,23 +13,4 @@ public class MigrateResult implements MigrationItemResult {
     private final Items items;
     private final String errorMessage;
     private final int exitCode;
-
-    public String toString() {
-        var sb = new StringBuilder();
-        if (getClusters() != null) {
-            sb.append(getClusters() + System.lineSeparator());
-        }
-        if (getItems() != null) {
-            sb.append(getItems() + System.lineSeparator());
-        }
-        sb.append("Results:" + System.lineSeparator());
-        if (Strings.isNotBlank(getErrorMessage())) {
-            sb.append("   Issue(s) detected" + System.lineSeparator());
-            sb.append("Issues:" + System.lineSeparator());
-            sb.append("   " + getErrorMessage() + System.lineSeparator());
-        } else {
-            sb.append("   " + getExitCode() + " issue(s) detected" + System.lineSeparator());
-        }
-        return sb.toString();
-    }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigrationItemResult.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigrationItemResult.java
@@ -1,10 +1,32 @@
 package org.opensearch.migrations.commands;
 
+import org.apache.logging.log4j.util.Strings;
+
 import org.opensearch.migrations.cli.Clusters;
+import org.opensearch.migrations.cli.Format;
 import org.opensearch.migrations.cli.Items;
 
 /** All shared cli result information */
 public interface MigrationItemResult extends Result {
     Clusters getClusters();
     Items getItems();
+
+    default String asCliOutput() {
+        var sb = new StringBuilder();
+        if (getClusters() != null) {
+            sb.append(getClusters().asCliOutput() + System.lineSeparator());
+        }
+        if (getItems() != null) {
+            sb.append(getItems().asCliOutput() + System.lineSeparator());
+        }
+        sb.append("Results:" + System.lineSeparator());
+        if (Strings.isNotBlank(getErrorMessage())) {
+            sb.append(Format.indentToLevel(1) + "Issue(s) detected" + System.lineSeparator());
+            sb.append("Issues:" + System.lineSeparator());
+            sb.append(Format.indentToLevel(1) + getErrorMessage() + System.lineSeparator());
+        } else {
+            sb.append(Format.indentToLevel(1) + getExitCode() + " issue(s) detected" + System.lineSeparator());
+        }
+        return sb.toString();
+    }
 }
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java
@@ -3,4 +3,7 @@
 /** All shared cli result information */
 public interface Result {
     int getExitCode();
+    String getErrorMessage();
+    /** Render this result as a string for displaying on the command line  */
+    String asCliOutput();
 }