Adding Support to run tests with Security (#33)

Signed-off-by: Aditya Jindal <aditjind@amazon.com>

.ci/run-opensearch.sh

@@ -37,7 +37,7 @@ NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
 http_port=9200
 for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
   node_name=${opensearch_node_name}$i
-  node_url=${external_opensearch_url/9200/${http_port}}$i
+  node_url=${external_opensearch_url/9200/${http_port}}
   if [[ "$i" == "0" ]]; then node_name=$opensearch_node_name; fi
   environment+=($(cat <<-END
     --env node.name=$node_name
@@ -58,12 +58,17 @@ END
   echo 'cluster is' $CLUSTER
   docker build \
     --file=.ci/$CLUSTER/Dockerfile \
-    --build-arg SECURE_INTEGRATION=false \
-    --tag=$CLUSTER \
+    --build-arg SECURE_INTEGRATION=$SECURE_INTEGRATION \
+    --tag=$CLUSTER-secure-$SECURE_INTEGRATION \
     .
 
   echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
   set -x
+  healthcmd="curl -vvv -s --fail http://localhost:9200/_cluster/health || exit 1"
+  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    healthcmd="curl -vvv -s --insecure -u admin:admin --fail https://localhost:9200/_cluster/health || exit 1"
+  fi
+
   docker run \
     --name "$node_name" \
     --network "$network_name" \
@@ -74,13 +79,13 @@ END
     --ulimit nofile=65536:65536 \
     --ulimit memlock=-1:-1 \
     --detach="$local_detach" \
-    --health-cmd="curl -vvv -s --fail $opensearch_url/_cluster/health || exit 1" \
+    --health-cmd="$(echo $healthcmd)" \
     --health-interval=2s \
     --health-retries=20 \
     --health-timeout=2s \
     --rm \
     -d \
-    $CLUSTER;
+    $CLUSTER-secure-$SECURE_INTEGRATION;
 
   set +x
   if wait_for_container "$opensearch_node_name" "$network_name"; then

.ci/run-tests

@@ -3,8 +3,13 @@
 # Version 1.1
 # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
 
-export OPENSEARCH_URL_EXTENSION="${OPENSEARCH_URL_EXTENSION:=http}"
 export CLUSTER="${1:=opensearch}"
+export SECURE_INTEGRATION="${2:-false}"
+if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    export OPENSEARCH_URL_EXTENSION="https"
+else
+    export OPENSEARCH_URL_EXTENSION="http"
+fi
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh

.github/workflows/test-integrations.yml

@@ -1,36 +1,35 @@
 name: Integration tests
 
-on:
-  pull_request:
-    paths-ignore:
-      - 'README.md'
-    branches:
-      - "*"
-  push:
-    paths-ignore:
-      - 'README.md'
-    branches:
-      - "*"
+on: [push, pull_request]
+
+env:
+  GITHUB_ACTIONS: true
 
 jobs:
-  assemble:
-    name: Assemble
+  tests:
+    name: Test
     runs-on: ubuntu-latest
     env:
       SECURE_INTEGRATION: false
     strategy:
-      fail-fast: false
       matrix:
         cluster-type: ['opendistro', 'opensearch']
+        secured: ["true", "false"]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Increase system limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
       - name: Install latest nightly
         uses: actions-rs/toolchain@v1
         with:
             toolchain: nightly
             override: true
             components: rustfmt, clippy
-      - run: "./.ci/run-tests ${{ matrix.cluster-type }} "
-        name: Integ ${{ matrix.cluster-type }}
+      - run: "./.ci/run-tests ${{ matrix.cluster-type }} ${{ matrix.secured }}"
+        name: Integ ${{ matrix.cluster-type }} secured=${{ matrix.secured }}

yaml_test_runner/src/main.rs

@@ -162,7 +162,7 @@ fn branch_suite_and_version_from_opensearch(
 
     let suite = TestSuite::Free;
 
-    let mut response = client.get(url).send()?;
+    let mut response = client.get(url).basic_auth("admin", Some("admin")).send()?;
     let json: Value = response.json()?;
     let branch = json["version"]["build_hash"].as_str().unwrap().to_string();