diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 217b51eb4..094538890 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -33,6 +33,25 @@ jobs: fi integration-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - group: scan + filter: "^TestScan" + timeout: 90 + - group: containers + filter: "^Test.*(Container|IacRealtime|SecretsRealtime).*" + timeout: 60 + - group: scm + filter: "^Test.*(UserCount|Azure|Bitbucket|Github|Gitlab|PR).*" + timeout: 45 + - group: git-hooks + filter: "^Test.*(PreCommit|PreReceive).*" + timeout: 45 + - group: core + filter: "^Test(Auth|Config|Project|Result|Root|Tenant|Feature|Chat|Log|Import|Predicate|Telemetry|Learn|Rate|Bfl|Asca|Oss).*" + timeout: 60 steps: - name: Checkout the repository uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 @@ -49,8 +68,9 @@ jobs: run: | pip install pre-commit pre-commit install - - name: Go Integration test + - name: Go Integration test - ${{ matrix.group }} shell: bash + timeout-minutes: ${{ matrix.timeout }} env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} @@ -96,27 +116,178 @@ jobs: PR_BITBUCKET_ID: 1 run: | sudo chmod +x ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh - ./internal/commands/.scripts/integration_up.sh + ./internal/commands/.scripts/integration_up.sh "${{ matrix.filter }}" ./internal/commands/.scripts/integration_down.sh - - name: Coverage report + - name: Upload coverage artifact uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() + with: + name: coverage-${{ matrix.group }} + path: | + cover.out + coverage.html + + - name: Upload test output + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() + with: + name: test-output-${{ matrix.group }} + path: test_output.log + + # Merge coverage from all parallel test groups + integration-coverage: + runs-on: ubuntu-latest + needs: integration-tests + if: always() + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Download all coverage artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4 with: - name: ${{ runner.os }}-coverage-latest - path: coverage.html + pattern: coverage-* + merge-multiple: false + + - name: Download all test outputs + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4 + with: + pattern: test-output-* + merge-multiple: false + + - name: Generate test summary + shell: bash + run: | + echo "## 🧪 Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Group | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped | Pass Rate |" >> $GITHUB_STEP_SUMMARY + echo "|-------|-------|-----------|-----------|------------|-----------|" >> $GITHUB_STEP_SUMMARY + + GRAND_TOTAL=0 + GRAND_PASSED=0 + GRAND_FAILED=0 + GRAND_SKIPPED=0 + + for group in scan containers scm git-hooks core; do + LOG_FILE="test-output-${group}/test_output.log" + if [ -f "$LOG_FILE" ]; then + PASSED=$(grep -c "^--- PASS:" "$LOG_FILE" 2>/dev/null || echo "0") + FAILED=$(grep -c "^--- FAIL:" "$LOG_FILE" 2>/dev/null || echo "0") + SKIPPED=$(grep -c "^--- SKIP:" "$LOG_FILE" 2>/dev/null || echo "0") + TOTAL=$((PASSED + FAILED + SKIPPED)) + + if [ "$TOTAL" -gt 0 ]; then + RATE=$(awk "BEGIN {printf \"%.1f\", ($PASSED/$TOTAL)*100}") + else + RATE="0.0" + fi + + GRAND_TOTAL=$((GRAND_TOTAL + TOTAL)) + GRAND_PASSED=$((GRAND_PASSED + PASSED)) + GRAND_FAILED=$((GRAND_FAILED + FAILED)) + GRAND_SKIPPED=$((GRAND_SKIPPED + SKIPPED)) + + echo "| **${group}** | ${TOTAL} | ${PASSED} | ${FAILED} | ${SKIPPED} | ${RATE}% |" >> $GITHUB_STEP_SUMMARY + else + echo "| **${group}** | - | - | - | - | N/A |" >> $GITHUB_STEP_SUMMARY + fi + done + + # Calculate grand total pass rate + if [ "$GRAND_TOTAL" -gt 0 ]; then + GRAND_RATE=$(awk "BEGIN {printf \"%.1f\", ($GRAND_PASSED/$GRAND_TOTAL)*100}") + else + GRAND_RATE="0.0" + fi + + echo "|-------|-------|-----------|-----------|------------|-----------|" >> $GITHUB_STEP_SUMMARY + echo "| **TOTAL** | **${GRAND_TOTAL}** | **${GRAND_PASSED}** | **${GRAND_FAILED}** | **${GRAND_SKIPPED}** | **${GRAND_RATE}%** |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # List failed tests if any + if [ "$GRAND_FAILED" -gt 0 ]; then + echo "### ❌ Failed Tests" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + for group in scan containers scm git-hooks core; do + LOG_FILE="test-output-${group}/test_output.log" + if [ -f "$LOG_FILE" ]; then + FAILED_TESTS=$(grep "^--- FAIL:" "$LOG_FILE" | awk '{print $3}' | head -10) + if [ -n "$FAILED_TESTS" ]; then + echo "**${group}:**" >> $GITHUB_STEP_SUMMARY + echo "$FAILED_TESTS" | while read test; do + echo "- \`${test}\`" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + fi + fi + done + fi + + - name: Merge coverage reports + shell: bash + run: | + # Debug: Show downloaded artifact structure + echo "=== Downloaded artifacts structure ===" + ls -la + find . -name "cover.out" -type f -exec ls -la {} \; + + # Find all cover.out files and merge them + find . -name "cover.out" -type f > coverage_files.txt + + echo "=== Coverage files found ===" + cat coverage_files.txt + echo "=== File count: $(wc -l < coverage_files.txt) ===" + + if [ -s coverage_files.txt ]; then + # Show individual coverage before merge + echo "=== Individual coverage percentages ===" + while read -r file; do + COV=$(go tool cover -func "$file" | grep total | awk '{print $3}') + echo "$file: $COV" + done < coverage_files.txt + + # Merge all coverage files + gocovmerge $(cat coverage_files.txt | tr '\n' ' ') > merged-cover.out + go tool cover -html=merged-cover.out -o merged-coverage.html + + echo "=== Merged coverage ===" + go tool cover -func merged-cover.out | grep total + else + echo "ERROR: No coverage files found!" + fi + + - name: Upload merged coverage report + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: ${{ runner.os }}-coverage-merged + path: | + merged-cover.out + merged-coverage.html - name: Check if total coverage is greater then 75 shell: bash run: | - CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') - EXPECTED_CODE_COV=75 - var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') - if [ "$var" -eq 1 ];then - echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" - exit 1 + if [ -f merged-cover.out ]; then + CODE_COV=$(go tool cover -func merged-cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + EXPECTED_CODE_COV=75 + var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') + if [ "$var" -eq 1 ];then + echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" + exit 1 + else + echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" + exit 0 + fi else - echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" - exit 0 + echo "No coverage file found - tests may have failed" + exit 1 fi lint: name: lint diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 62708467b..674607fc4 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -1,3 +1,6 @@ +# Accept optional test filter as first argument (regex pattern for -run flag) +TEST_FILTER=${1:-""} + # Start the Squid proxy in a Docker container docker run \ --name squid \ @@ -12,12 +15,21 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz -# Step 1: Check if the failedTests file exists +# Build the test filter argument if provided +RUN_ARG="" +if [ -n "$TEST_FILTER" ]; then + RUN_ARG="-run $TEST_FILTER" + echo "Running tests matching filter: $TEST_FILTER" +fi + +# Initialize variables FAILED_TESTS_FILE="failedTests" +rerun_status=0 -# Step 2: Create the failedTests file -echo "Creating $FAILED_TESTS_FILE..." +# Step 1: Create tracking files (ensure they exist for CI artifact upload) +echo "Creating tracking files..." touch "$FAILED_TESTS_FILE" +touch test_output.log # Step 3: Run all tests and write failed test names to failedTests file echo "Running all tests..." @@ -25,6 +37,7 @@ go test \ -tags integration \ -v \ -timeout 210m \ + $RUN_ARG \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log @@ -46,7 +59,7 @@ fi if [ ! -s "$FAILED_TESTS_FILE" ]; then # If the file is empty, all tests passed echo "All tests passed." - rm -f "$FAILED_TESTS_FILE" test_output.log + rm -f "$FAILED_TESTS_FILE" else # If the file is not empty, rerun the failed tests echo "Rerunning failed tests..." @@ -79,12 +92,94 @@ else fi fi -# Step 7: Run the cleandata package to delete projects +# Step 7: Generate test summary table +echo "" +echo "==============================================" +echo " TEST EXECUTION SUMMARY " +echo "==============================================" + +# Parse test results from log (use tr to remove any newlines/carriage returns) +TOTAL_PASSED=$(grep -c "^--- PASS:" test_output.log 2>/dev/null | tr -d '\r\n' || echo "0") +TOTAL_FAILED=$(grep -c "^--- FAIL:" test_output.log 2>/dev/null | tr -d '\r\n' || echo "0") +TOTAL_SKIPPED=$(grep -c "^--- SKIP:" test_output.log 2>/dev/null | tr -d '\r\n' || echo "0") + +# Ensure values are valid integers (default to 0 if empty or invalid) +TOTAL_PASSED=${TOTAL_PASSED:-0} +TOTAL_FAILED=${TOTAL_FAILED:-0} +TOTAL_SKIPPED=${TOTAL_SKIPPED:-0} + +# Remove any non-numeric characters +TOTAL_PASSED=$(echo "$TOTAL_PASSED" | tr -cd '0-9') +TOTAL_FAILED=$(echo "$TOTAL_FAILED" | tr -cd '0-9') +TOTAL_SKIPPED=$(echo "$TOTAL_SKIPPED" | tr -cd '0-9') + +# Default to 0 if empty after cleaning +TOTAL_PASSED=${TOTAL_PASSED:-0} +TOTAL_FAILED=${TOTAL_FAILED:-0} +TOTAL_SKIPPED=${TOTAL_SKIPPED:-0} + +TOTAL_TESTS=$((TOTAL_PASSED + TOTAL_FAILED + TOTAL_SKIPPED)) + +# Calculate pass rate +if [ "$TOTAL_TESTS" -gt 0 ]; then + PASS_RATE=$(awk "BEGIN {printf \"%.1f\", ($TOTAL_PASSED/$TOTAL_TESTS)*100}") +else + PASS_RATE="0.0" +fi + +# Extract duration from test output (look for the integration test package line) +DURATION=$(grep -E "(ok|FAIL)\s+github.com/checkmarx/ast-cli/test/integration\s+" test_output.log | awk '{for(i=1;i<=NF;i++) if($i ~ /^[0-9]+\.[0-9]+s$/) print $i}' | head -1) +if [ -z "$DURATION" ]; then + DURATION="N/A" +fi + +# Get test filter info (truncate if too long for table) +if [ -n "$TEST_FILTER" ]; then + FILTER_INFO=$(echo "$TEST_FILTER" | cut -c1-18) + if [ ${#TEST_FILTER} -gt 18 ]; then + FILTER_INFO="${FILTER_INFO}..." + fi +else + FILTER_INFO="All tests" +fi + +# Print summary table (no ANSI colors for CI compatibility) +printf "\n" +printf "+---------------------+---------------------+\n" +printf "| %-19s | %-19s |\n" "Metric" "Value" +printf "+---------------------+---------------------+\n" +printf "| %-19s | %-19s |\n" "Test Filter" "$FILTER_INFO" +printf "| %-19s | %-19s |\n" "Total Tests" "$TOTAL_TESTS" +printf "| %-19s | %-19s |\n" "Passed" "$TOTAL_PASSED" +printf "| %-19s | %-19s |\n" "Failed" "$TOTAL_FAILED" +printf "| %-19s | %-19s |\n" "Skipped" "$TOTAL_SKIPPED" +printf "| %-19s | %-19s |\n" "Pass Rate" "${PASS_RATE}%" +printf "| %-19s | %-19s |\n" "Duration" "$DURATION" +printf "+---------------------+---------------------+\n" + +# Print failed test names if any +if [ "$TOTAL_FAILED" -gt 0 ]; then + echo "" + echo "Failed Tests:" + echo "-------------" + grep "^--- FAIL:" test_output.log | awk '{print " [X] " $3}' | head -20 + FAIL_COUNT=$(grep -c "^--- FAIL:" test_output.log 2>/dev/null | tr -cd '0-9') + FAIL_COUNT=${FAIL_COUNT:-0} + if [ "$FAIL_COUNT" -gt 20 ]; then + echo " ... and $((FAIL_COUNT - 20)) more" + fi +fi + +echo "" +echo "==============================================" + +# Step 8: Run the cleandata package to delete projects echo "Running cleandata to clean up projects..." go test -v github.com/checkmarx/ast-cli/test/cleandata -# Step 8: Final cleanup and exit -rm -f "$FAILED_TESTS_FILE" test_output.log +# Step 9: Final cleanup and exit +# Note: Keep test_output.log for CI artifact upload +rm -f "$FAILED_TESTS_FILE" if [ $status -ne 0 ] || [ $rerun_status -eq 1 ]; then exit 1