diff --git a/spec/plans/report.fmf b/spec/plans/report.fmf index a289df540b..a847850d69 100644 --- a/spec/plans/report.fmf +++ b/spec/plans/report.fmf @@ -123,17 +123,21 @@ description: Provide test results and fmf data per each plan, and send it to a Report Portal instance via its API with token, url and project name given. + + Note that all options can be passed environment variable + in the format TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} + to enable execution purely via metadata and/or + environment variables (e.g. Testing Farm) + example: - | # Optionally set environment variables according to TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=${url-to-RP-instance} export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=${token-from-RP-profile} - # Bolean options are activated with value of 1: + # Boolean options are activated with value of 1: TMT_PLUGIN_REPORT_REPORTPORTAL_SUITE_PER_PLAN=1 - - NOTE: All options are supported in the format demonstrated above to enable execution purely via metadata and/or environment variables (e.g. Testing Farm) - - | # Enable ReportPortal report from the command line depending on the use case: diff --git a/tests/report/reportportal/test.sh b/tests/report/reportportal/test.sh index dc0e859b17..d1c1a99c17 100755 --- a/tests/report/reportportal/test.sh +++ b/tests/report/reportportal/test.sh @@ -12,6 +12,7 @@ TEST_PREFIX='/test' declare -A test=([1,'uuid']="" [1,'name']='/bad' [1,'status']='FAILED' [2,'uuid']="" [2,'name']='/good' [2,'status']='PASSED' [3,'uuid']="" [3,'name']='/weird' [3,'status']='FAILED') +DIV="|" ## @@ -21,7 +22,7 @@ declare -A test=([1,'uuid']="" [1,'name']='/bad' [1,'status']='FAILED' # << $launch_name # >> $launch_uuid, $launch_id # -function foo_launch(){ +function identify_launch(){ rlLog "Verify and get launch data" rlAssertGrep "launch: $launch_name" $rlRun_LOG @@ -31,23 +32,24 @@ function foo_launch(){ launch_uuid=$(rlRun "grep -A2 'launch:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") fi - rlAssertNotEquals "Assert the launch uuid is not empty" "$launch_uuid" "" + rlAssertNotEquals "Assert the launch UUID is not empty" "$launch_uuid" "" launch_id=$(rlRun "grep 'url:' $rlRun_LOG | awk '{print \$NF}' | xargs basename") - rlAssertNotEquals "Assert the launch id is not empty" "$launch_id" "" + rlAssertNotEquals "Assert the launch ID is not empty" "$launch_id" "" } + ## # Read and verify reported suite name and uuid from $rlRun_LOG # # GLOBALS: # << $suite_name -# >> $suite_uuid, $suite_id -function foo_suite(){ +# >> $suite_uuid +function identify_suite(){ rlLog "Verify and get suite data" rlAssertGrep "suite: $suite_name" $rlRun_LOG - suite_uuid=$(rlRun "grep -A1 'launch:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the launch uuid is not empty" "$suite_uuid" "" + suite_uuid=$(rlRun "grep -A1 'suite:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") + rlAssertNotEquals "Assert the suite UUID is not empty" "$suite_uuid" "" } ## @@ -55,7 +57,7 @@ function foo_suite(){ # # GLOBALS: # >> $test_uuid[1..3], $test_fullname[1..3] -function foo_tests(){ +function identify_tests(){ rlLog "Verify and get test data" for i in {1..3}; do @@ -63,12 +65,30 @@ function foo_tests(){ rlAssertGrep "test: ${test_fullname[$i]}" $rlRun_LOG test_uuid[$i]=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the test$i uuid is not empty" "{$test_uuid[$i]}" "" + rlAssertNotEquals "Assert the test$i UUID is not empty" "${test_uuid[$i]}" "" test[$i,'uuid']=${test_uuid[$i]} done } +## Proceed GET request via ReportPortal REST API and verify the response +# +# ARGUMENT: +# request_url +function rest_api(){ + + rlLog "REST API request (GET $1)" + response=$(curl --write-out "$DIV%{http_code}" --silent -X GET "$1" -H "accept: */*" -H "Authorization: bearer $TOKEN") + + response_code=${response##*"$DIV"} + response=${response%"$DIV"*} + if [[ $response_code -ge 300 ]]; then + rlFail "Request responded with an error: $response" + fi + + echo "$response" +} + rlJournalStart @@ -81,24 +101,27 @@ rlJournalStart fi rlPhaseEnd + echo -e "\n\n\n:: PART 1\n" rlPhaseStartTest "Core Functionality" launch_name=$PLAN_PREFIX launch_status=$PLAN_STATUS - rlRun -s "tmt run --id $run --verbose --all" 2 + # TMT RUN + rlLogInfo "A run with default setup" + rlRun -s "tmt run --id $run --verbose --all" 2 rlAssertGrep "url: http.*redhat.com.ui/#${PROJECT}/launches/all/[0-9]{4}" $rlRun_LOG -Eq - foo_launch # >> $launch_uuid, $launch_id - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] + identify_launch # >> $launch_uuid, $launch_id + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] rlPhaseEnd rlPhaseStartTest "Core Functionality - DEFAULT SETUP" - # LAUNCH - via API launch-controller /v1/{projectName}/launch/uuid/{launchId} - rlLog "REST API - Get info about the launch" - response=$(curl -X GET "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] launch-controller | uuid + rlLogInfo "Get info about the launch" + response=$(rest_api "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid") rlAssertEquals "Assert the URL ID of launch is correct" "$(echo $response | jq -r '.id')" "$launch_id" rlAssertEquals "Assert the name of launch is correct" "$(echo $response | jq -r '.name')" "$launch_name" rlAssertEquals "Assert the status of launch is correct" "$(echo $response | jq -r '.status')" "$launch_status" @@ -112,6 +135,7 @@ rlJournalStart rlAssertEquals "Assert the description of launch is correct" "$(echo $response | jq -r '.description')" "$plan_summary, $ARTIFACTS" fi fi + echo "" # Check all the launch attributes rl_message="Test attributes of the launch (context)" @@ -126,6 +150,7 @@ rlJournalStart rm tmp_attributes* for i in {1..3}; do + echo "" test_name[$i]=${test[$i,'name']} test_name=${test_name[$i]} test_fullname=${test_fullname[$i]} @@ -133,11 +158,9 @@ rlJournalStart test_status[$i]=${test[$i,'status']} test_status=${test_status[$i]} - echo "" - - # TEST ITEMS - via API test-item-controller /v1/{projectName}/item/uuid/{itemId} - rlLog "REST API - Get info about the test item $test_name" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/uuid/$test_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] test-item-controller | uuid + rlLogInfo "Get info about the test item $test_name" + response=$(rest_api "$URL/api/v1/$PROJECT/item/uuid/$test_uuid") test_id=$(echo $response | jq -r '.id') rlAssertNotEquals "Assert the test id is not empty" "$test_id" "" rlAssertEquals "Assert the name is correct" "$(echo $response | jq -r '.name')" "$test_fullname" @@ -152,7 +175,7 @@ rlJournalStart # Check all the common test attributes/parameters [[ $jq_element == attributes ]] && fmf_label="context" [[ $jq_element == parameters ]] && fmf_label="environment" - rlLog "Check the $jq_element for test $test_name ($fmf_label)" + rlLogInfo "Check the $jq_element for test $test_name ($fmf_label)" echo "$response" | jq -r ".$jq_element" > tmp_attributes.json || rlFail "$jq_element listing into tmp_attributes.json" length=$(yq -r ".$fmf_label | length" plan.fmf) for ((item_index=0; item_index<$length; item_index++ )); do @@ -180,9 +203,9 @@ rlJournalStart rm tmp_attributes* done - # Check the logs - via API log-controller /v1/{projectName}/log/nested/{parentId} - rlLog "REST API - Get all logs from the test $test_name" - response=$(curl -X GET "$URL/api/v1/$PROJECT/log/nested/$test_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] log-controller | parent_id + rlLogInfo "Get all logs from the test $test_name" + response=$(rest_api "$URL/api/v1/$PROJECT/log/nested/$test_id") length=$(echo $response | jq -r ".content | length") level=("INFO" "ERROR") for ((content_index=0; content_index<$length; content_index++ )); do @@ -196,21 +219,23 @@ rlJournalStart rlPhaseEnd + echo -e "\n\n\n:: PART 2\n" + # Testing launch-per-plan mapping with launch-test structure rlPhaseStartTest "Extended Functionality - LAUNCH-PER-PLAN" launch_name=${PLAN_PREFIX}/launch-per-plan + # TMT RUN + rlLogInfo "A run that creates a launch per each plan and test items directly within" rlRun -s "tmt run --verbose --all report --how reportportal --launch-per-plan --launch '$launch_name' " 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id + identify_launch # >> $launch_uuid, $launch_id rlAssertNotGrep "suite:" $rlRun_LOG - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] - - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=0; content_index<$length; content_index++ )); do rlAssertEquals "Assert the item is no suite" "$(echo $response | jq -r .content[$content_index].hasChildren)" "false" @@ -218,26 +243,28 @@ rlJournalStart rlPhaseEnd - + # Testing suite-per-plan mapping with launch-suite-test structure rlPhaseStartTest "Extended Functionality - SUITE-PER-PLAN" launch_name=${PLAN_PREFIX}/suite-per-plan plan_summary=$(yq -r '.summary' plan.fmf) launch_description="Testing the integration of tmt and Report Portal via its API with suite-per-plan mapping" launch_status=$PLAN_STATUS suite_name=$PLAN_PREFIX + + # TMT RUN + rlLogInfo "A run that creates a launch with a suite per each plan and test items within" rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name' --launch-description '$launch_description'" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - foo_suite # >> $suite_uuid, $suite_id - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] + identify_launch # >> $launch_uuid, $launch_id + identify_suite # >> $suite_uuid, $suite_id + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] + echo "" - # LAUNCH - via API launch-controller /v1/{projectName}/launch/uuid/{launchId} - rlLog "REST API - Get info about the launch" - response=$(curl -X GET "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] launch-controller | uuid + rlLogInfo "Get info about the launch" + response=$(rest_api "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid") rlAssertEquals "Assert the URL ID of launch is correct" "$(echo $response | jq -r '.id')" "$launch_id" rlAssertEquals "Assert the name of launch is correct" "$(echo $response | jq -r '.name')" "$launch_name" rlAssertEquals "Assert the status of launch is correct" "$(echo $response | jq -r '.status')" "$launch_status" - xx="$(echo $response | jq -r .description)" - echo $xx rlAssertEquals "Assert the description of launch is correct" "$(echo $response | jq -r .description)" "$launch_description" # Check all the launch attributes @@ -245,20 +272,21 @@ rlJournalStart echo "$response" | jq -r ".attributes" > tmp_attributes.json && rlPass "$rl_message" || rlFail "$rl_message" length=$(yq -r ".context | length" plan.fmf) for ((item_index=0; item_index<$length; item_index++ )); do + echo "" key=$(yq -r ".context | keys | .[$item_index]" plan.fmf) value=$(yq -r ".context.$key" plan.fmf) rlAssertGrep "$key" tmp_attributes.json -A1 > tmp_attributes_selection rlAssertGrep "$value" tmp_attributes_selection done rm tmp_attributes* + echo "" - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" - + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=0; content_index<$length; content_index++ )); do + echo "" if [[ $content_index -eq 0 ]]; then rlAssertEquals "Assert the item is a suite" "$(echo $response | jq -r .content[$content_index].hasChildren)" "true" rlAssertEquals "Assert the name of suite item ${suite_name}" "$(echo $response | jq -r .content[$content_index].name)" "${suite_name}" @@ -267,63 +295,71 @@ rlJournalStart i=$content_index rlAssertEquals "Assert the item is no suite" "$(echo $response | jq -r .content[$content_index].hasChildren)" "false" rlAssertEquals "Assert the name of test item ${test_name[$i]}" "$(echo $response | jq -r .content[$content_index].name)" "${test_fullname[$i]}" - rlAssertEquals "Assert the uuid of test item ${test_name[$i]}" "$(echo $response | jq -r .content[$content_index].uuid)" "${test_uuid[$i]}" + rlAssertEquals "Assert the UUID of test item ${test_name[$i]}" "$(echo $response | jq -r .content[$content_index].uuid)" "${test_uuid[$i]}" fi done rlPhaseEnd + + # Testing the test history is aggregated correctly with/out unique parameters and case IDs rlPhaseStartTest "Extended Functionality - HISTORY AGGREGATION" launch_name=${PLAN_PREFIX}/history-aggregation + # TMT RUN [0] + rlLogInfo "Initial run that creates a launch for history" rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_1'" 2 "" 1>/dev/null - for i in {1..3}; do + echo "" test_fullname=${TEST_PREFIX}${test[$i,'name']} test_uuid=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the test$i uuid is not empty" "{$test_uuid}" "" + rlAssertNotEquals "Assert the test$i UUID is not empty" "{$test_uuid}" "" - # TEST ITEM - via API test-item-controller /v1/{projectName}/item/uuid/{itemId} - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/uuid/$test_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] test-item-controller | uuid + rlLogInfo "Get info about the test item $i" + response=$(rest_api "$URL/api/v1/$PROJECT/item/uuid/$test_uuid") rlAssertEquals "Assert the name is correct" "$(echo $response | jq -r '.name')" "$test_fullname" launch1_test_id[$i]=$(echo $response | jq -r '.id') rlAssertNotEquals "Assert the test id is not empty" "$launch1_test_id[$i]" "" done - echo "" - rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_2'" 2 "" 1>/dev/null + # TMT RUN [1] + rlLogInfo "A run that creates a launch with filtered environment variables (by default)" + rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_2'" 2 "" 1>/dev/null for i in {1..3}; do + echo "" test_name=${test[$i,'name']} test_fullname=${TEST_PREFIX}${test_name} test_uuid=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the test$i uuid is not empty" "{$test_uuid}" "" + rlAssertNotEquals "Assert the test$i UUID is not empty" "{$test_uuid}" "" - # TEST ITEM - via API test-item-controller /v1/{projectName}/item/uuid/{itemId} - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/uuid/$test_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] test-item-controller | uuid + rlLogInfo "Get info about the test item $i" + response=$(rest_api "$URL/api/v1/$PROJECT/item/uuid/$test_uuid") rlAssertEquals "Assert the name is correct" "$(echo $response | jq -r '.name')" "$test_fullname" launch2_test_id[$i]=$(echo $response | jq -r '.id') rlAssertNotEquals "Assert the test id is not empty" "$launch2_test_id[$i]" "" - - rlLog "Verify the history is aggregated" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/history?filter.eq.id=${launch2_test_id[$i]}&historyDepth=2" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + # REST API | [GET] test-item-controller | history + rlLogInfo "Verify the history is aggregated" + response=$(rest_api "$URL/api/v1/$PROJECT/item/history?filter.eq.id=${launch2_test_id[$i]}&historyDepth=2") rlAssertEquals "Assert the previous item in history" "$(echo $response | jq -r .content[0].resources[1].id)" "${launch1_test_id[$i]}" done - - echo "" - rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_3' --exclude-variables ''" 2 "" 1>/dev/null + # TMT RUN [2] + rlLogInfo "A run that creates a launch without filtering the environment variables that break history aggregation" + rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_3' --exclude-variables ''" 2 "" 1>/dev/null for i in {1..3}; do + echo "" test_name=${test[$i,'name']} test_fullname=${TEST_PREFIX}${test_name} test_uuid=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the test$i uuid is not empty" "{$test_uuid}" "" + rlAssertNotEquals "Assert the test$i UUID is not empty" "{$test_uuid}" "" - # TEST ITEM - via API test-item-controller /v1/{projectName}/item/uuid/{itemId} - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/uuid/$test_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] test-item-controller | uuid + response=$(rest_api "$URL/api/v1/$PROJECT/item/uuid/$test_uuid") rlAssertEquals "Assert the name is correct" "$(echo $response | jq -r '.name')" "$test_fullname" launch3_test_id[$i]=$(echo $response | jq -r '.id') rlAssertNotEquals "Assert the test id is not empty" "$launch3_test_id[$i]" "" @@ -334,9 +370,10 @@ rlJournalStart rm tmp_attributes* # history is not aggregated unless test case id is defined for given test (only test_2) - [[ $i -eq 2 ]] && rlLog "Verify the history is aggregated" || rlLog "Verify the history is not aggregated" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item/history?filter.eq.id=${launch3_test_id[$i]}&historyDepth=2" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + [[ $i -eq 2 ]] && rlLogInfo "Verify the history should be aggregated" || rlLogInfo "Verify the history should not be aggregated" + + # REST API | [GET] test-item-controller | history + response=$(rest_api "$URL/api/v1/$PROJECT/item/history?filter.eq.id=${launch3_test_id[$i]}&historyDepth=2") [[ $i -eq 2 ]] && rlAssertEquals "Assert the previous item is in history" "$(echo $response | jq -r .content[0].resources[1].id)" "${launch2_test_id[$i]}" \ || rlAssertNotEquals "Assert the previous item is not in history" "$(echo $response | jq -r .content[0].resources[1].id)" "${launch2_test_id[$i]}" done @@ -344,33 +381,36 @@ rlJournalStart rlPhaseEnd - # Testing integration with ReportPortal build-in RERUN feature with Retry items + # Testing integration with ReportPortal built-in RERUN feature with Retry items rlPhaseStartTest "Extended Functionality - NAME-BASED RERUN" launch_name=${PLAN_PREFIX}/name-based-rerun suite_name=$PLAN_PREFIX + # TMT RUN [0] + rlLogInfo "Initial run that creates a launch" rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null for i in {1..3}; do core_test_uuid[$i]=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ") - rlAssertNotEquals "Assert the test$i uuid is not empty" "{$core_test_uuid[$i]}" "" + rlAssertNotEquals "Assert the test$i UUID is not empty" "{$core_test_uuid[$i]}" "" done - echo "" + + # TMT RE-RUN [1] + rlLogInfo "Create a new run that is reported as rerun within the last same-named launch" rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name' --launch-rerun" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - foo_suite # >> $suite_uuid, $suite_id - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] + identify_launch # >> $launch_uuid, $launch_id + identify_suite # >> $suite_uuid + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] rlAssertGrep "suite: $suite_name" $rlRun_LOG - # LAUNCH - via API launch-controller /v1/{projectName}/launch/uuid/{launchId} - rlLog "REST API - Get info about the launch" - response=$(curl -X GET "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") + # REST API | [GET] launch-controller | uuid + rlLogInfo "Get info about the launch" + response=$(rest_api "$URL/api/v1/$PROJECT/launch/uuid/$launch_uuid") rlAssertEquals "Assert the launch is rerun" "$(echo $response | jq -r '.rerun')" "true" - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=1; content_index<$length; content_index++ )); do @@ -386,16 +426,20 @@ rlJournalStart launch_name=${PLAN_PREFIX}/UUID-based-rerun suite_name=$PLAN_PREFIX + # TMT RUN [0] + rlLogInfo "Initial run that creates a launch" rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] echo "" - rlRun -s "tmt run --verbose --last --all report --how reportportal --suite-per-plan --launch '$launch_name' --again" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - # ITEMS - via API test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .error)" "null" + # TMT RE-RUN [1] + rlLogInfo "Execute and report the same run again to append new test logs" + rlRun -s "tmt run --verbose --last --all execute --again report --how reportportal --suite-per-plan --launch '$launch_name' --again" 2 "" 1>/dev/null + identify_launch # >> $launch_uuid, $launch_id + + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=1; content_index<$length; content_index++ )); do @@ -404,13 +448,10 @@ rlJournalStart test_id[$i]="$(echo $response | jq -r .content[$content_index].id)" rlAssertNotEquals "Assert the test$i id is not empty" "${test_id[$i]}" "" + # REST API | [GET] log-controller | parent_id + rlLogInfo "Get all logs from the test$i" + response_log=$(rest_api "$URL/api/v1/$PROJECT/log/nested/${test_id[$i]}") test_name=${test[$i,'name']} - # LOGS - via API log-controller /v1/{projectName}/log/nested/{parentId} - rlLog "REST API - Get all logs from the test$i" - response_log=$(curl -X GET "$URL/api/v1/$PROJECT/log/nested/${test_id[$i]}" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response_log | jq -r .error)" "null" - echo "$response_log" | jq - length_log=$(echo $response_log | jq -r ".content | length") if [[ $i -eq 2 ]]; then level=("INFO" "INFO") @@ -425,121 +466,124 @@ rlJournalStart fi done done - rlPhaseEnd + # Uploading empty report with IDLE states and updating it within the same tmt run rlPhaseStartTest "Extended Functionality - IDLE REPORT" launch_name=${PLAN_PREFIX}/idle_report suite_name=$PLAN_PREFIX - rlRun -s "tmt run discover report --verbose --how reportportal --suite-per-plan --launch '$launch_name' --defect-type 'idle'" 3 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - foo_tests # >> $test_uuid[1..3], $test_fullname[1..3] - - # ITEMS - via API test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .error)" "null" + # TMT RUN [0] + rlLogInfo "Initial run that only creates an empty launch (with empty suite and empty test items within) with defect type 'Idle' (pre-defined in the project within 'To Investigate' category)" + rlRun -s "tmt run discover report --verbose --how reportportal --suite-per-plan --launch '$launch_name' --defect-type 'IDLE'" 3 "" 1>/dev/null + identify_launch # >> $launch_uuid, $launch_id + identify_tests # >> $test_uuid[1..3], $test_fullname[1..3] + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=1; content_index<$length; content_index++ )); do - rlAssertEquals "Assert the defect type was defined" "$(echo $response | jq -r .content[$content_index].statistics.defects.to_investigate.total)" "1" - + i=$content_index + rlAssertEquals "Assert the defect type of test[$i] was defined" "$(echo $response | jq -r .content[$content_index].statistics.defects.to_investigate.total)" "1" + rlAssertNotEquals "Assert the defect type of test[$i]is not the default one" "$(echo $response | jq -r '.content['$content_index'].statistics.defects.to_investigate | keys[0]')" "ti001" done - echo "" - rlRun -s "tmt run --last --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name' --again" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - # ITEMS - via API test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .error)" "null" + # TMT RE-RUN [1] + rlLogInfo "Execute the same run and update the results, redefine the defect type to default value" + rlRun -s "tmt run --last --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name' --again" 2 "" 1>/dev/null + identify_launch # >> $launch_uuid, $launch_id + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") length=$(echo $response | jq -r ".content | length") for ((content_index=1; content_index<$length; content_index++ )); do i=$content_index - rlAssertEquals "Assert the test item has correct UUID" "$(echo $response | jq -r .content[$content_index].uuid)" "${test_uuid[$i]}" + rlAssertEquals "Assert the test[$i] item has correct UUID" "$(echo $response | jq -r .content[$content_index].uuid)" "${test_uuid[$i]}" test_id[$i]="$(echo $response | jq -r .content[$content_index].id)" - rlAssertNotEquals "Assert the test$i id is not empty" "${test_id[$i]}" "" - + rlAssertNotEquals "Assert the test[$i] id is not empty" "${test_id[$i]}" "" + if [[ "$(echo $response | jq -r .content[$content_index].status)" == "FAILED" ]]; then + rlAssertEquals "Assert the defect type of test[$i] is the default one" "$(echo $response | jq -r '.content['$content_index'].statistics.defects.to_investigate | keys[0]')" "ti001" + fi done - rlPhaseEnd + # Uploading new suites and new tests to an existing launch rlPhaseStartTest "Extended Functionality - UPLOAD TO LAUNCH" launch_name=${PLAN_PREFIX}/upload-to-launch suite_name=$PLAN_PREFIX + # TMT RUN [0] + rlLogInfo "Initial run that creates a launch (with suite item and 3 test items within)" rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id + identify_launch # >> $launch_uuid, $launch_id init_launch_uuid=$launch_uuid - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items (1)" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items (1)" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") rlAssertEquals "Assert launch contains suite and 3 test items" "$(echo $response | jq -r .page.totalElements)" "4" - echo "" + + # TMT RUN [1] + rlLogInfo "Additional run for an upload (of a suite item with 3 test items) to the launch" rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --upload-to-launch '$launch_id'" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - rlAssertEquals "Assert the launch uuid is same" "$init_launch_uuid" "$launch_uuid" + identify_launch # >> $launch_uuid, $launch_id + rlAssertEquals "Assert the launch UUID is the same as the initial one" "$init_launch_uuid" "$launch_uuid" - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items (2)" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items (2)" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") rlAssertEquals "Assert launch contains another suite and 3 test items" "$(echo $response | jq -r .page.totalElements)" "8" - echo "" + + # TMT RUN [2] + rlLogInfo "Additional run for an upload (of 3 test items) to the launch" rlRun -s "tmt run --all report --verbose --how reportportal --launch-per-plan --upload-to-launch '$launch_id'" 2 "" 1>/dev/null - foo_launch # >> $launch_uuid, $launch_id - rlAssertEquals "Assert the response has no error message" "$init_launch_uuid" "$launch_uuid" + identify_launch # >> $launch_uuid, $launch_id + rlAssertEquals "Assert the launch UUID is the same as the initial one" "$init_launch_uuid" "$launch_uuid" - # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - rlLog "REST API - Get info about all launch items (3)" - response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - rlAssertEquals "Assert the response has no error message" "$(echo $response | jq -r .message | grep 'Error')" "" + # REST API | [GET] test-item-controller | launch_id + rlLogInfo "Get info about all launch items (3)" + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id") rlAssertEquals "Assert launch contains another 3 test items" "$(echo $response | jq -r .page.totalElements)" "11" - rlPhaseEnd - # Uploading new suites and new tests to an existing suite + # Uploading new tests to an existing suite rlPhaseStartTest "Extended Functionality - UPLOAD TO SUITE" launch_name=${PLAN_PREFIX}/upload-to-suite suite_name=$PLAN_PREFIX - rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" - #TODO get suite uuid - foo_launch # >> $launch_uuid, $launch_id - foo_suite # >> $suite_uuid + # TMT RUN [0] + rlLogInfo "Initial run that creates a suite item (with 3 test items)" + rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null + identify_launch # >> $launch_uuid, $launch_id + identify_suite # >> $suite_uuid init_launch_uuid=$launch_uuid - # response=$(curl -X GET "$URL/api/v1/$PROJECT/item/$suite_uuid" -H "accept: */*" -H "Authorization: bearer $TOKEN") - # rlAssertNotEquals "Assert the response has no error message" "$(echo $response | jq -r .errorCode | grep '4043')" "" - - # echo $response | jq - # suite_id=$(echo $response | jq -r .id) - # echo $suite_id - - # #TODO verify new tests created in given suite - # # TEST ITEMS - test-item-controller /v1/{projectName}/item + launch_id - # rlLog "REST API - Get info about all launch items (1)" - # response=$(curl -X GET "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id&filter.eq.parentId=$suite_id" -H "accept: */*" -H "Authorization: bearer $TOKEN") - # rlAssertNotEquals "Assert the response has no error message" "$(echo $response | jq -r .errorCode | grep '4043')" "" - - # echo $response | jq - - # rlAssertEquals "Assert launch contains suite and 3 test items" "$(echo $response | jq -r .page.totalElements)" "3" - - # echo "" + # REST API | [GET] test-item-controller | uuid + rlLogInfo "Obtain 'suite id' for an additional upload." + response=$(rest_api "$URL/api/v1/$PROJECT/item/uuid/$suite_uuid") + rlAssertEquals "Assert the UUID of launch is correct" "$(echo $response | jq -r '.launchId')" "$launch_id" + suite_id=$(echo $response | jq -r .id) + rlAssertNotEquals "Assert the suite ID is not empty" "$suite_id" "" + echo "" - # rlRun -s "tmt run --all report --verbose --how reportportal --upload-to-suite '$suite_id'" 2 "" 1>/dev/null + # TMT RUN [1] + rlLogInfo "Additional run for an upload (of 3 test items) to the suite" + rlRun -s "tmt run --all report --verbose --how reportportal --upload-to-suite '$suite_id'" 2 "" 1>/dev/null + identify_launch + rlAssertEquals "Assert the launch UUID is the same as the initial one" "$init_launch_uuid" "$launch_uuid" + # REST API | [GET] test-item-controller | launch_id, parent_id + rlLogInfo "Get info about all the suite items." + response=$(rest_api "$URL/api/v1/$PROJECT/item?filter.eq.launchId=$launch_id&filter.eq.parentId=$suite_id") + rlAssertEquals "Assert suite contains suite 6 test items" "$(echo $response | jq -r .page.totalElements)" "6" rlPhaseEnd diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index 045be63de8..673ed5ce29 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -62,18 +62,18 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="LAUNCH_NAME", default=_str_env_to_default('launch', None), help=""" - Set the launch name, otherwise name of the plan is used by default. - Should be defined with suite-per-plan option or it will be named after the first plan. - """) + Set the launch name, otherwise name of the plan is used by default. + Should be defined with 'suite-per-plan' option or it will be named after the first plan. + """) launch_description: Optional[str] = field( option="--launch-description", metavar="DESCRIPTION", default=_str_env_to_default('launch_description', None), help=""" - Pass the description for ReportPortal launch with '--suite-per-plan' option + Pass the description for ReportPortal launch with 'suite-per-plan' option or append the original (plan summary) with additional info. - Appends test description with upload-to-launch/suite options. + Appends test description with 'upload-to-launch/suite' options. """) launch_per_plan: bool = field( @@ -88,8 +88,8 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): is_flag=True, help=""" Mapping suite per plan, creating one launch and continuous uploading suites into it. - Recommended to use with '--launch' and '--launch-description' options. - Can be used with '--upload-to-launch' option to avoid creating a new launch. + Recommended to use with 'launch' and 'launch-description' options. + Can be used with 'upload-to-launch' option for an additional upload of new suites. """) upload_to_launch: Optional[str] = field( @@ -97,10 +97,10 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="LAUNCH_ID", default=_str_env_to_default('upload_to_launch', None), help=""" - Pass the launch ID for an additional test/suite upload to an existing launch. - ID can be found in the launch URL. - To upload specific info into description see also launch-description. - """) + Pass the launch ID for an additional test/suite upload to an existing launch. ID can be + found in the launch URL. Keep the launch structure with options 'launch/suite-per-plan'. + To upload specific info into description see also 'launch-description'. + """) upload_to_suite: Optional[str] = field( option="--upload-to-suite", @@ -109,7 +109,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): help=""" Pass the suite ID for an additional test upload to a suite within an existing launch. ID can be found in the suite URL. - To upload specific info into description see also launch-description. + To upload specific info into description see also 'launch-description'. """) launch_rerun: bool = field( @@ -204,13 +204,20 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin[ReportReportPortalData]): Other reported fmf data are summary, id, web link and contact per test. - There are supported two ways of mapping plans into ReportPortal + Two types of data structures are supported for reporting to ReportPortal: + + * 'launch-per-plan' mapping (default) that results in launch-test structure. + * 'suite-per-plan' mapping that results in launch-suite-test structure. - * launch-per-plan (default) with reported structure 'launch > test', - resulting in one or more launches. - * suite-per-plan with reported structure 'launch > suite > test' - resulting in one launch only, and one or more suites within. - It is recommended to define launch name and launch description in addition. + Supported report use cases: + + * Report a new run in launch-suite-test or launch-test structure + * Report an additional rerun with 'launch-rerun' option and same launch name (->Retry items) + or by reusing the run and reporting with 'again' option (->append logs) + * To see plan progress, discover and report an empty (IDLE) run + and reuse the run for execution and updating the report with 'again' option + * Report contents of a new run to an existing launch via the URL ID in three ways: + tests to launch, suites to launch and tests to suite. """ _data_class = ReportReportPortalData @@ -379,7 +386,7 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: launch_rerun = self.data.launch_rerun envar_pattern = self.data.exclude_variables or "$^" - defect_type = self.data.defect_type + defect_type = self.data.defect_type or "" attributes = [ {'key': key, 'value': value[0]} @@ -572,12 +579,6 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: self.handle_response(response) launch_time = test_time - # TODO: Resolve the problem with reporting original defect type (idle) - # after additional report of results - # Temporary solution idea: - # if again_additional_tests and status failed, - # get test_id, report passed and then again failed - if create_suite: # Finish the test suite response = session.put( @@ -589,6 +590,8 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: self.handle_response(response) is_the_last_plan = self.step.plan == self.step.plan.my_run.plans[-1] + if is_the_last_plan: + self.data.defect_type = None if ((launch_per_plan or (suite_per_plan and is_the_last_plan)) and not additional_upload):