Backport rename folders commit (#184)

* Update folder names (#166)

Signed-off-by: sruti1312 <srutiparthiban@gmail.com>

* Update PA directories from plugins to root (#168)

Signed-off-by: sruti1312 <srutiparthiban@gmail.com>

* Update configFilePath location (#172)

Signed-off-by: sruti1312 <srutiparthiban@gmail.com>

* Disable plugin if configFilePath is not present (#173)

Signed-off-by: sruti1312 <srutiparthiban@gmail.com>

.gitignore

@@ -6,7 +6,6 @@
 /.settings/
 /build
 /.project
-/bin/
 /coverage-error.log
 gradle.properties
 /src/test/resources/tmp/file_gc/

INSTALL.md

@@ -39,7 +39,7 @@ This document walks you through the process of building and deploying the RCA fr
  This will not currently enforce client  authentication. This feature, including full support for TLS is
  planned for an upcoming release.
 
-    a. Open pa_config/performance-analyzer.properties
+    a. Open $OPENSEARCH-HOME/config/opensearch-performance-analyzer/performance-analyzer.properties
 
     b. Modify the certificate-file-path, private-key-file-path, and https-enabled entries
 

README.md

@@ -49,7 +49,7 @@ __Metrics__: Metrics are typically served as continuous datastreams to downstrea
 
 ### Components
 
-__Framework__: The RCA runtime operates on an `AnalysisGraph`. You can extend this class and override the `construct` method to build your own RCAs. You should specify the path to the class in the `analysis-graph-implementor` section of `pa_config/rca.conf`.  The `addLeaf` and `addAllUpstreams` helper methods are useful when you define the dependencies between nodes of the graph.
+__Framework__: The RCA runtime operates on an `AnalysisGraph`. You can extend this class and override the `construct` method to build your own RCAs. You should specify the path to the class in the `analysis-graph-implementor` section of `config/opensearch-performance-analyzer/rca.conf`.  The `addLeaf` and `addAllUpstreams` helper methods are useful when you define the dependencies between nodes of the graph.
 
 __Scheduler__: The scheduler invokes the `operate` method on each graph node in topological order as defined in the `AnalysisGraph`. Nodes with no dependencies can be executed in parallel. Use flow-units to share data between RCA nodes instead of shared objects to avoid data races and performance bottlenecks.
 

pa_bin/performance-analyzer-agent → bin/performance-analyzer-agent

@@ -28,12 +28,19 @@ fi
 echo "Using JAVA_HOME: $JAVA_HOME"
 export JAVA_HOME=$JAVA_HOME
 
+if [[ -z "$OPENSEARCH_PATH_CONF" ]]; then
+        echo "OPENSEARCH_PATH_CONF needs to be set."
+        exit 1
+fi
+
+echo "Using OPENSEARCH_PATH_CONF: $OPENSEARCH_PATH_CONF"
+
 if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then
-    export JAVA_OPTS=-Dopensearch.path.home=$OPENSEARCH_HOME\ -Dlog4j.configurationFile=$OPENSEARCH_HOME/performance-analyzer-rca/pa_config/log4j2.xml\ -XX:+ExitOnOutOfMemoryError
+    export JAVA_OPTS=-Dopensearch.path.home=$OPENSEARCH_HOME\ -Dopensearch.path.conf=$OPENSEARCH_PATH_CONF\ -Dlog4j.configurationFile=$OPENSEARCH_HOME/performance-analyzer-rca/config/log4j2.xml\ -XX:+ExitOnOutOfMemoryError
     exec $OPENSEARCH_HOME/performance-analyzer-rca/bin/performance-analyzer-rca
 else
     echo 'Starting deamon'
-    export JAVA_OPTS=-Dopensearch.path.home=$OPENSEARCH_HOME\ -Dlog4j.configurationFile=$OPENSEARCH_HOME/performance-analyzer-rca/pa_config/log4j2.xml\ -XX:+ExitOnOutOfMemoryError
+    export JAVA_OPTS=-Dopensearch.path.home=$OPENSEARCH_HOME\ -Dopensearch.path.conf=$OPENSEARCH_PATH_CONF\ -Dlog4j.configurationFile=$OPENSEARCH_HOME/performance-analyzer-rca/config/log4j2.xml\ -XX:+ExitOnOutOfMemoryError
     exec $OPENSEARCH_HOME/performance-analyzer-rca/bin/performance-analyzer-rca &
 
     pid=$!

build.gradle

@@ -35,10 +35,10 @@ application {
                                  '-XX:MaxRAM=400m']
 }
 
-// Include pa_config folder in the distribution.
+// Include PA related folder in the distribution.
 applicationDistribution.from(".") {
-    include 'pa_config/*'
-    include 'pa_bin/*'
+    include 'config/*'
+    include 'bin/*'
 }
 
 distributions {
@@ -206,6 +206,8 @@ test {
             excludeTestsMatching 'org.opensearch.performanceanalyzer.rca.integTests.*'
             // TODO: Fix this test as it causes OutOfMemoryError: Java heap space error and runs forever
             excludeTestsMatching 'org.opensearch.performanceanalyzer.reader.OSMetricsSnapshotTests'
+            // TODO: Fix flaky test: https://github.com/opensearch-project/performance-analyzer-rca/issues/167
+            excludeTestsMatching 'org.opensearch.performanceanalyzer.PerformanceAnalyzerWebServerTest'
         }
     }
     else {

pa_config/opensearch_security.policy → config/opensearch_security.policy

@@ -13,4 +13,3 @@ grant codeBase "jrt:/jdk.attach" {
 grant codeBase "jrt:/jdk.internal.jvmstat" {
     permission java.security.AllPermission;
 };
-

pa_config/supervisord.conf → config/supervisord.conf

@@ -28,5 +28,5 @@ serverurl=/usr/share/supervisord.sock
 files = /etc/supervisor/conf.d/*.conf
 
 [program:performance_analyzer]
-command=/usr/share/opensearch/performance-analyzer-rca/pa_bin/performance-analyzer-agent /usr/share/opensearch
+command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch
 user=1000

docker/Dockerfile

@@ -53,8 +53,6 @@ RUN chown -R opensearch:0 . && \
 
 RUN unzip config/performance-analyzer-rca-1.3.1.0-SNAPSHOT.zip
 
-RUN cp -r performance-analyzer-rca/* plugins/opensearch-performance-analyzer/
-RUN chmod 755 /usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent
 RUN chmod -R 755 /dev/shm
 ################################################################################
 # Build stage 1 (the actual OpenSearch image):
@@ -90,6 +88,7 @@ RUN yum update -y && \
 COPY CENTOS_LICENSING.txt /root
 COPY --from=prep_open_search_files --chown=1000:0 /opt/jdk-11.0.1 /opt/jdk-11.0.1
 ENV OPENSEARCH_JAVA_HOME /opt/jdk-11.0.1
+ENV OPENSEARCH_PATH_CONF /usr/share/opensearch/config
 
 # Replace OpenJDK's built-in CA certificate keystore with the one from the OS
 # vendor. The latter is superior in several ways.
@@ -118,7 +117,7 @@ RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
     chmod 0775 /usr/local/bin/docker-entrypoint.sh
 
 # Bind to all interfaces so that the docker container is accessible from the host machine
-RUN sed -i "s|#webservice-bind-host =|webservice-bind-host = 0.0.0.0|g" /usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties
+RUN sed -i "s|#webservice-bind-host =|webservice-bind-host = 0.0.0.0|g" /usr/share/opensearch/config/opensearch-performance-analyzer/performance-analyzer.properties
 
 EXPOSE 9200 9300 9600 9650
 

docker/docker-entrypoint.sh

@@ -91,8 +91,8 @@ fi
 if [[ -d "/usr/share/opensearch/plugins/opensearch-performance-analyzer" ]]; then
     CLK_TCK=`/usr/bin/getconf CLK_TCK`
     DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n"
-    OPENSEARCH_JAVA_OPTS="-Djava.security.policy=file:///usr/share/opensearch/performance-analyzer-rca/pa_config/opensearch_security.policy -Dclk.tck=$CLK_TCK -Djdk.attach.allowAttachSelf=true $OPENSEARCH_JAVA_OPTS"
-    /usr/bin/supervisord -c /usr/share/opensearch/performance-analyzer-rca/pa_config/supervisord.conf
+    OPENSEARCH_JAVA_OPTS="-Djava.security.policy=file:///usr/share/opensearch/performance-analyzer-rca/config/opensearch_security.policy -Dclk.tck=$CLK_TCK -Djdk.attach.allowAttachSelf=true $OPENSEARCH_JAVA_OPTS"
+    /usr/bin/supervisord -c /usr/share/opensearch/performance-analyzer-rca/config/supervisord.conf
 fi
 
 run_as_other_user_if_needed /usr/share/opensearch/bin/opensearch "${open_search_opts[@]}"

src/main/java/org/opensearch/performanceanalyzer/collectors/StatsCollector.java

@@ -22,7 +22,6 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.config.PluginSettings;
-import org.opensearch.performanceanalyzer.core.Util;
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.rca.Version;
 import org.opensearch.performanceanalyzer.rca.formatter.StatsCollectorFormatter;
@@ -146,13 +145,15 @@ private static Map<String, String> loadMetadata(String fileLocation) {
 
             try (InputStream input =
                     new FileInputStream(
-                            Util.PLUGIN_LOCATION
-                                    + PluginSettings.CONFIG_FILES_PATH
-                                    + fileLocation); ) {
+                            PluginSettings.instance().getConfigFolderPath() + fileLocation); ) {
                 // load properties file
                 props.load(input);
             } catch (Exception ex) {
-                GENERAL_LOG.error("Error in loading metadata for fileLocation: {}", fileLocation);
+                GENERAL_LOG.error(
+                        "Error in loading metadata for folderLocation: {}, fileLocation: {}",
+                        PluginSettings.instance().getConfigFolderPath(),
+                        fileLocation,
+                        ex);
             }
 
             props.forEach((key, value) -> retVal.put((String) key, (String) value));

src/main/java/org/opensearch/performanceanalyzer/config/PluginSettings.java

@@ -21,9 +21,7 @@ public class PluginSettings {
     private static final Logger LOG = LogManager.getLogger(PluginSettings.class);
 
     private static PluginSettings instance;
-    public static final String CONFIG_FILES_PATH = "pa_config/";
-    private static final String DEFAULT_CONFIG_FILE_PATH =
-            Util.PLUGIN_LOCATION + "pa_config/performance-analyzer.properties";
+    public static final String CONFIG_FILES_PATH = "config/";
     private static final String METRICS_LOCATION_KEY = "metrics-location";
     private static final String METRICS_LOCATION_DEFAULT = "/dev/shm/performanceanalyzer/";
     private static final String DELETION_INTERVAL_KEY = "metrics-deletion-interval";
@@ -57,7 +55,8 @@ public class PluginSettings {
 
     private boolean httpsEnabled;
     private Properties settings;
-    private final String configFilePath;
+    private String configFolderPath;
+    private String configFilePath;
 
     /**
      * Determines how many minutes worth of metricsdb files will be retained if batch metrics is
@@ -72,6 +71,10 @@ public class PluginSettings {
         Util.invokePrivilegedAndLogError(PluginSettings::createInstance);
     }
 
+    public String getConfigFolderPath() {
+        return configFolderPath;
+    }
+
     public String getMetricsLocation() {
         return metricsLocation;
     }
@@ -163,9 +166,12 @@ private PluginSettings(String cfPath) {
         rpcPort = RPC_DEFAULT_PORT;
         webServicePort = WEBSERVICE_DEFAULT_PORT;
         if (cfPath == null || cfPath.isEmpty()) {
-            this.configFilePath = DEFAULT_CONFIG_FILE_PATH;
+            LOG.error("Config file path is not set. Disabling plugin.");
+            ConfigStatus.INSTANCE.setConfigurationInvalid();
         } else {
-            this.configFilePath = cfPath;
+            this.configFolderPath =
+                    cfPath + File.separator + "opensearch-performance-analyzer" + File.separator;
+            this.configFilePath = configFolderPath + "performance-analyzer.properties";
         }
 
         settings = new Properties();
@@ -209,7 +215,7 @@ public static PluginSettings instance() {
     }
 
     private static void createInstance() {
-        String cfPath = System.getProperty("configFilePath");
+        String cfPath = System.getProperty("opensearch.path.conf");
         instance = new PluginSettings(cfPath);
     }
 

src/main/java/org/opensearch/performanceanalyzer/rca/framework/util/RcaConsts.java

@@ -22,7 +22,7 @@ public class RcaConsts {
     private static final String RCA_CONF_IDLE_MASTER_FILENAME = "rca_idle_master.conf";
     private static final String THRESHOLDS_DIR_NAME = "thresholds";
     public static final String CONFIG_DIR_PATH =
-            Paths.get(Util.READER_LOCATION, "pa_config").toString();
+            Paths.get(Util.READER_LOCATION, "config").toString();
     public static final String RCA_CONF_PATH =
             Paths.get(CONFIG_DIR_PATH, RCA_CONF_FILENAME).toString();
     public static final String RCA_CONF_MASTER_PATH =