diff --git a/.github/workflows/ci-tests-optimized.yml b/.github/workflows/ci-tests-optimized.yml new file mode 100644 index 000000000..ca0a4f960 --- /dev/null +++ b/.github/workflows/ci-tests-optimized.yml @@ -0,0 +1,210 @@ +name: Continuous Integration Tests (Optimized) + +on: + pull_request: + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + - run: go version + - name: go test with coverage + run: | + sudo chmod +x ./internal/commands/.scripts/up.sh + ./internal/commands/.scripts/up.sh + - name: Check if total coverage is greater then 77.7 + shell: bash + run: | + CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + EXPECTED_CODE_COV=77.7 + var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') + if [ "$var" -eq 1 ];then + echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" + exit 1 + else + echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" + exit 0 + fi + + integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - run: go version + + - name: Go Build + run: go build -o ./bin/cx ./cmd + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Cache ScaResolver + uses: actions/cache@v3 + with: + path: /tmp/ScaResolver + key: ${{ runner.os }}-scaresolver-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-scaresolver- + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run All Integration Tests (Grouped) + shell: bash + env: + CX_BASE_URI: ${{ secrets.CX_BASE_URI }} + CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} + CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} + CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} + CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }} + CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }} + CX_APIKEY: ${{ secrets.CX_APIKEY }} + CX_TENANT: ${{ secrets.CX_TENANT }} + CX_SCAN_SSH_KEY: ${{ secrets.CX_SCAN_SSH_KEY }} + CX_ORIGIN: "cli-tests" + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PROXY_HOST: localhost + PROXY_PORT: 3128 + PROXY_USERNAME: ${{ secrets.PROXY_USER }} + PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }} + PR_GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PR_GITHUB_NAMESPACE: "checkmarx" + PR_GITHUB_REPO_NAME: "ast-cli" + PR_GITHUB_NUMBER: 983 + PR_GITLAB_TOKEN: ${{ secrets.PR_GITLAB_TOKEN }} + PR_GITLAB_NAMESPACE: ${{ secrets.PR_GITLAB_NAMESPACE }} + PR_GITLAB_REPO_NAME: ${{ secrets.PR_GITLAB_REPO_NAME }} + PR_GITLAB_PROJECT_ID: ${{ secrets.PR_GITLAB_PROJECT_ID }} + PR_GITLAB_IID: ${{ secrets.PR_GITLAB_IID }} + AZURE_ORG: ${{ secrets.AZURE_ORG }} + AZURE_PROJECT: ${{ secrets.AZURE_PROJECT }} + AZURE_REPOS: ${{ secrets.AZURE_REPOS }} + AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }} + AZURE_NEW_ORG: ${{ secrets.AZURE_NEW_ORG }} + AZURE_PROJECT_NAME: ${{ secrets.AZURE_PROJECT_NAME }} + AZURE_PR_NUMBER: 1 + AZURE_NEW_TOKEN: ${{ secrets.AZURE_NEW_TOKEN }} + BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }} + BITBUCKET_REPOS: ${{ secrets.BITBUCKET_REPOS }} + BITBUCKET_USERNAME: ${{ secrets.BITBUCKET_USERNAME }} + BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PASSWORD }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + PR_BITBUCKET_TOKEN: ${{ secrets.PR_BITBUCKET_TOKEN }} + PR_BITBUCKET_NAMESPACE: "AstSystemTest" + PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" + PR_BITBUCKET_ID: 1 + run: | + sudo chmod +x ./internal/commands/.scripts/integration_up_grouped.sh ./internal/commands/.scripts/integration_down.sh + ./internal/commands/.scripts/integration_up_grouped.sh + ./internal/commands/.scripts/integration_down.sh + + - name: Upload coverage report + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() + with: + name: ${{ runner.os }}-coverage-latest + path: coverage.html + retention-days: 7 + + - name: Check if total coverage is greater than 75 + shell: bash + run: | + CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + EXPECTED_CODE_COV=75 + var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') + if [ "$var" -eq 1 ];then + echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" + exit 1 + else + echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" + exit 0 + fi + + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + - run: go version + - run: go mod tidy + - name: golangci-lint + uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc #v3 + with: + skip-pkg-cache: true + version: v1.64.2 + args: -c .golangci.yml + --timeout 5m + only-new-issues: true + + govulncheck: + runs-on: ubuntu-latest + name: govulncheck + steps: + - id: govulncheck + uses: golang/govulncheck-action@7da72f730e37eeaad891fcff0a532d27ed737cd4 #v1 + continue-on-error: true + with: + go-version-file: go.mod + go-package: ./... + + checkDockerImage: + runs-on: ubuntu-latest + name: scan Docker Image with Trivy + steps: + - name: Checkout code + uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 #2.0.0 + + - name: Set up Docker + uses: docker/setup-buildx-action@cf09c5c41b299b55c366aff30022701412eb6ab0 #v1.0.0 + + - name: Log in to Docker Hub + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b #v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build the project + run: go build -o ./cx ./cmd + - name: Build Docker image + run: docker build -t ast-cli:${{ github.sha }} . + - name: Run Trivy scanner without downloading DBs + uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 #v0.28.0 + with: + scan-type: 'image' + image-ref: ast-cli:${{ github.sha }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + output: './trivy-image-results.txt' + env: + TRIVY_SKIP_JAVA_DB_UPDATE: true + + - name: Inspect action report + if: always() + shell: bash + run: cat ./trivy-image-results.txt + diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 217b51eb4..859898752 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1,4 +1,4 @@ -name: Continuous Integration Tests +name: Continuous Integration Tests (Optimized) on: pull_request: @@ -31,25 +31,54 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + integration-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-group: [ + "fast-validation", + "scan-core", + "scan-engines", + "scm-integration", + "realtime-features", + "advanced-features" + ] steps: - name: Checkout the repository uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 with: go-version-file: go.mod + - run: go version + - name: Go Build run: go build -o ./bin/cx ./cmd + - name: Install gocovmerge run: go install github.com/wadey/gocovmerge@latest - - name: Install pre-commit - run: | - pip install pre-commit - pre-commit install - - name: Go Integration test + + - name: Cache ScaResolver + uses: actions/cache@v3 + with: + path: /tmp/ScaResolver + key: ${{ runner.os }}-scaresolver-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-scaresolver- + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Go Integration test - ${{ matrix.test-group }} shell: bash env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} @@ -94,12 +123,53 @@ jobs: PR_BITBUCKET_NAMESPACE: "AstSystemTest" PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" PR_BITBUCKET_ID: 1 + TEST_GROUP: ${{ matrix.test-group }} run: | - sudo chmod +x ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh - ./internal/commands/.scripts/integration_up.sh + sudo chmod +x ./internal/commands/.scripts/integration_up_parallel.sh ./internal/commands/.scripts/integration_down.sh + ./internal/commands/.scripts/integration_up_parallel.sh ./internal/commands/.scripts/integration_down.sh - - name: Coverage report + - name: Upload coverage for ${{ matrix.test-group }} + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() + with: + name: coverage-${{ matrix.test-group }} + path: cover.out + retention-days: 1 + + merge-coverage: + needs: integration-tests + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + path: coverage-reports + + - name: Merge coverage reports + run: | + find coverage-reports -name "cover.out" -exec echo {} \; > coverage-files.txt + if [ -s coverage-files.txt ]; then + gocovmerge $(cat coverage-files.txt) > merged-cover.out + go tool cover -html=merged-cover.out -o coverage.html + else + echo "No coverage files found" + exit 1 + fi + + - name: Upload merged coverage report uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 with: name: ${{ runner.os }}-coverage-latest @@ -108,7 +178,7 @@ jobs: - name: Check if total coverage is greater then 75 shell: bash run: | - CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + CODE_COV=$(go tool cover -func merged-cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') EXPECTED_CODE_COV=75 var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') if [ "$var" -eq 1 ];then @@ -118,6 +188,7 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + lint: name: lint runs-on: ubuntu-latest @@ -156,7 +227,6 @@ jobs: - name: Checkout code uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 #2.0.0 - - name: Set up Docker uses: docker/setup-buildx-action@cf09c5c41b299b55c366aff30022701412eb6ab0 #v1.0.0 @@ -186,3 +256,4 @@ jobs: if: always() shell: bash run: cat ./trivy-image-results.txt + diff --git a/internal/commands/.scripts/integration_up_grouped.sh b/internal/commands/.scripts/integration_up_grouped.sh new file mode 100644 index 000000000..ad131f34c --- /dev/null +++ b/internal/commands/.scripts/integration_up_grouped.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# Grouped integration test script - runs all test groups in parallel as background jobs +# Shows as single action in GitHub Actions UI +# Expected execution time: ~30 minutes (longest group) + +set -e + +# Color output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Starting Grouped Integration Tests${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +# Start the Squid proxy in a Docker container (shared across all groups) +if ! docker ps | grep -q squid; then + echo -e "${YELLOW}Starting Squid proxy...${NC}" + docker run \ + --name squid \ + -d \ + -p $PROXY_PORT:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta + echo -e "${GREEN}✓ Squid proxy started${NC}" +else + echo -e "${GREEN}✓ Squid proxy already running${NC}" +fi + +# Download and extract the ScaResolver tool (with caching) +if [ ! -f "/tmp/ScaResolver" ]; then + echo -e "${YELLOW}Downloading ScaResolver...${NC}" + wget -q https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz + tar -xzf ScaResolver-linux64.tar.gz -C /tmp + rm -rf ScaResolver-linux64.tar.gz + echo -e "${GREEN}✓ ScaResolver downloaded${NC}" +else + echo -e "${GREEN}✓ Using cached ScaResolver${NC}" +fi + +echo "" +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Running Test Groups in Parallel${NC}" +echo -e "${BLUE}========================================${NC}" +echo "" + +# Define test groups +TEST_GROUPS=("fast-validation" "scan-core" "scan-engines" "scm-integration" "realtime-features" "advanced-features") + +# Create directories for each group's output +mkdir -p test-results +for group in "${TEST_GROUPS[@]}"; do + mkdir -p "test-results/$group" +done + +# Function to run a test group +run_test_group() { + local group=$1 + local log_file="test-results/$group/output.log" + local coverage_file="test-results/$group/cover.out" + + echo -e "${YELLOW}[${group}] Starting...${NC}" | tee -a "$log_file" + + # Source the test patterns from integration_up_parallel.sh logic + case "$group" in + "fast-validation") + TEST_PATTERN="." + SKIP_PATTERN="Scan|PR|UserCount|Realtime|Project|Result|Import|Bfl|Chat|Learn|Remediation|Triage|Container|Scs|ASCA|Asca" + TIMEOUT="10m" + ;; + "scan-core") + TEST_PATTERN="Scan" + SKIP_PATTERN="Realtime|ASCA|Asca|Container.*Scan" + TIMEOUT="30m" + ;; + "scan-engines") + TEST_PATTERN="ASCA|Asca|Container|Scs|Engine|CodeBashing|RiskManagement|CreateQueryDescription|MaskSecrets" + SKIP_PATTERN="Realtime" + TIMEOUT="35m" + ;; + "scm-integration") + TEST_PATTERN="PR|UserCount|RateLimit|Hooks|Predicate|PreReceive|PreCommit" + SKIP_PATTERN="^$" + TIMEOUT="25m" + ;; + "realtime-features") + TEST_PATTERN="Realtime" + SKIP_PATTERN="^$" + TIMEOUT="20m" + ;; + "advanced-features") + TEST_PATTERN="Project|Result|Import|Bfl|Chat|Learn|Telemetry|Remediation|Triage|GetProjectName" + SKIP_PATTERN="^$" + TIMEOUT="25m" + ;; + esac + + # Build test command + TEST_CMD="go test -tags integration -v -timeout ${TIMEOUT} -parallel 4" + TEST_CMD="${TEST_CMD} -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers" + TEST_CMD="${TEST_CMD} -coverprofile ${coverage_file}" + TEST_CMD="${TEST_CMD} -run '${TEST_PATTERN}'" + + if [ ! -z "$SKIP_PATTERN" ] && [ "$SKIP_PATTERN" != "^$" ]; then + TEST_CMD="${TEST_CMD} -skip '${SKIP_PATTERN}'" + fi + + TEST_CMD="${TEST_CMD} github.com/checkmarx/ast-cli/test/integration" + + # Run tests + echo -e "${BLUE}[${group}] Command: ${TEST_CMD}${NC}" >> "$log_file" + eval "${TEST_CMD}" >> "$log_file" 2>&1 + local status=$? + + if [ $status -eq 0 ]; then + echo -e "${GREEN}[${group}] ✓ PASSED${NC}" | tee -a "$log_file" + else + echo -e "${RED}[${group}] ✗ FAILED (exit code: $status)${NC}" | tee -a "$log_file" + fi + + return $status +} + +# Export function so it can be used by parallel +export -f run_test_group +export RED GREEN YELLOW BLUE NC + +# Run all test groups in parallel using background jobs +declare -A PIDS +for group in "${TEST_GROUPS[@]}"; do + run_test_group "$group" & + PIDS[$group]=$! + echo -e "${BLUE}Started ${group} (PID: ${PIDS[$group]})${NC}" +done + +echo "" +echo -e "${YELLOW}Waiting for all test groups to complete...${NC}" +echo "" + +# Wait for all background jobs and collect results +declare -A RESULTS +ALL_PASSED=true + +for group in "${TEST_GROUPS[@]}"; do + wait ${PIDS[$group]} + RESULTS[$group]=$? + + if [ ${RESULTS[$group]} -ne 0 ]; then + ALL_PASSED=false + fi +done + +echo "" +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Test Results Summary${NC}" +echo -e "${BLUE}========================================${NC}" +echo "" + +for group in "${TEST_GROUPS[@]}"; do + if [ ${RESULTS[$group]} -eq 0 ]; then + echo -e "${GREEN}✓ ${group}: PASSED${NC}" + else + echo -e "${RED}✗ ${group}: FAILED${NC}" + fi +done + +echo "" + +# Merge all coverage files +echo -e "${YELLOW}Merging coverage reports...${NC}" +COVERAGE_FILES=$(find test-results -name "cover.out" -type f) +if [ -z "$COVERAGE_FILES" ]; then + echo -e "${RED}No coverage files found!${NC}" + exit 1 +fi + +gocovmerge $COVERAGE_FILES > cover.out +echo -e "${GREEN}✓ Coverage reports merged${NC}" + +# Generate HTML coverage report +go tool cover -html=cover.out -o coverage.html +echo -e "${GREEN}✓ HTML coverage report generated${NC}" + +# Display coverage summary +echo "" +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Coverage Summary${NC}" +echo -e "${BLUE}========================================${NC}" +go tool cover -func cover.out | tail -10 + +echo "" +if [ "$ALL_PASSED" = true ]; then + echo -e "${GREEN}========================================${NC}" + echo -e "${GREEN}All test groups PASSED! ✓${NC}" + echo -e "${GREEN}========================================${NC}" + exit 0 +else + echo -e "${RED}========================================${NC}" + echo -e "${RED}Some test groups FAILED! ✗${NC}" + echo -e "${RED}========================================${NC}" + echo "" + echo -e "${YELLOW}Check individual group logs in test-results/ directory${NC}" + exit 1 +fi diff --git a/internal/commands/.scripts/integration_up_parallel.sh b/internal/commands/.scripts/integration_up_parallel.sh new file mode 100644 index 000000000..39cacd1b6 --- /dev/null +++ b/internal/commands/.scripts/integration_up_parallel.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +# Optimized integration test script with parallel execution and test grouping +# Expected execution time: ~30 minutes (down from 210 minutes) + +set -e + +# Color output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${GREEN}Starting optimized integration tests for group: ${TEST_GROUP}${NC}" + +# Start the Squid proxy in a Docker container (only if not already running) +if ! docker ps | grep -q squid; then + echo -e "${YELLOW}Starting Squid proxy...${NC}" + docker run \ + --name squid \ + -d \ + -p $PROXY_PORT:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta +else + echo -e "${GREEN}Squid proxy already running${NC}" +fi + +# Download and extract the ScaResolver tool (with caching) +if [ ! -f "/tmp/ScaResolver" ]; then + echo -e "${YELLOW}Downloading ScaResolver...${NC}" + wget -q https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz + tar -xzf ScaResolver-linux64.tar.gz -C /tmp + rm -rf ScaResolver-linux64.tar.gz +else + echo -e "${GREEN}Using cached ScaResolver${NC}" +fi + +# Define test patterns for each group +# Strategy: Run ALL tests, coverage merging will handle overlaps +# This ensures 100% test coverage across all groups +case "$TEST_GROUP" in + "fast-validation") + # Fast tests: auth, configuration, validation (no actual scans) + # Estimated time: 3-5 minutes + TEST_PATTERN="." # Run all tests + SKIP_PATTERN="Scan|PR|UserCount|Realtime|Project|Result|Import|Bfl|Chat|Learn|Remediation|Triage|Container|Scs|ASCA|Asca" + TIMEOUT="10m" + ;; + + "scan-core") + # Core scan functionality - basic scan operations + # Estimated time: 20-25 minutes + TEST_PATTERN="Scan" # All tests with "Scan" in name + SKIP_PATTERN="Realtime|ASCA|Asca|Container.*Scan" + TIMEOUT="30m" + ;; + + "scan-engines") + # Multi-engine scans: SAST, SCA, IaC, Containers, SCS, ASCA + # Estimated time: 25-30 minutes + TEST_PATTERN="ASCA|Asca|Container|Scs|Engine|CodeBashing|RiskManagement|CreateQueryDescription|MaskSecrets" + SKIP_PATTERN="Realtime" + TIMEOUT="35m" + ;; + + "scm-integration") + # SCM integrations: GitHub, GitLab, Azure, Bitbucket, Hooks, Predicates + # Estimated time: 15-20 minutes + TEST_PATTERN="PR|UserCount|RateLimit|Hooks|Predicate|PreReceive|PreCommit" + SKIP_PATTERN="^$" # No skip pattern + TIMEOUT="25m" + ;; + + "realtime-features") + # Real-time scanning features + # Estimated time: 10-15 minutes + TEST_PATTERN="Realtime" + SKIP_PATTERN="^$" # No skip pattern + TIMEOUT="20m" + ;; + + "advanced-features") + # Advanced features: projects, results, imports, BFL, chat, learn, remediation, triage + # Estimated time: 15-20 minutes + TEST_PATTERN="Project|Result|Import|Bfl|Chat|Learn|Telemetry|Remediation|Triage|GetProjectName" + SKIP_PATTERN="^$" # No skip pattern + TIMEOUT="25m" + ;; + + *) + echo -e "${RED}Unknown test group: $TEST_GROUP${NC}" + exit 1 + ;; +esac + +echo -e "${GREEN}Running test group: ${TEST_GROUP}${NC}" +echo -e "${YELLOW}Test pattern: ${TEST_PATTERN}${NC}" +echo -e "${YELLOW}Skip pattern: ${SKIP_PATTERN}${NC}" +echo -e "${YELLOW}Timeout: ${TIMEOUT}${NC}" + +# Build the go test command +TEST_CMD="go test -tags integration -v -timeout ${TIMEOUT} -parallel 4" +TEST_CMD="${TEST_CMD} -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers" +TEST_CMD="${TEST_CMD} -coverprofile cover.out" +TEST_CMD="${TEST_CMD} -run '${TEST_PATTERN}'" + +if [ ! -z "$SKIP_PATTERN" ] && [ "$SKIP_PATTERN" != "^$" ]; then + TEST_CMD="${TEST_CMD} -skip '${SKIP_PATTERN}'" +fi + +TEST_CMD="${TEST_CMD} github.com/checkmarx/ast-cli/test/integration" + +# Create the failedTests file +FAILED_TESTS_FILE="failedTests" +echo -e "${YELLOW}Creating ${FAILED_TESTS_FILE}...${NC}" +touch "$FAILED_TESTS_FILE" + +# Run tests with output logging +echo -e "${GREEN}Executing: ${TEST_CMD}${NC}" +eval "${TEST_CMD}" 2>&1 | tee test_output.log + +# Capture the exit status +status=$? +echo "Test execution status: $status" + +# Generate the initial HTML coverage report +if [ -f cover.out ]; then + go tool cover -html=cover.out -o coverage.html +fi + +# Extract names of failed tests +grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" || true + +# Check if there are failed tests to retry +if [ -s "$FAILED_TESTS_FILE" ]; then + echo -e "${YELLOW}Rerunning failed tests...${NC}" + rerun_status=0 + + while IFS= read -r testName; do + echo -e "${YELLOW}Retrying: ${testName}${NC}" + go test \ + -tags integration \ + -v \ + -timeout 15m \ + -parallel 1 \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover_rerun.out \ + -run "^${testName}$" \ + github.com/checkmarx/ast-cli/test/integration || rerun_status=1 + done < "$FAILED_TESTS_FILE" + + # Merge coverage if rerun produced coverage + if [ -f cover_rerun.out ]; then + echo -e "${YELLOW}Merging coverage profiles...${NC}" + gocovmerge cover.out cover_rerun.out > merged_coverage.out + mv merged_coverage.out cover.out + go tool cover -html=cover.out -o coverage.html + rm -f cover_rerun.out + fi + + # Check final status + if [ $rerun_status -eq 1 ]; then + echo -e "${RED}Some tests are still failing after retry.${NC}" + else + echo -e "${GREEN}All failed tests passed on rerun.${NC}" + fi +else + echo -e "${GREEN}All tests passed on first run.${NC}" +fi + +# Run the cleandata package to delete test projects (only for scan-core group) +if [ "$TEST_GROUP" = "scan-core" ] || [ "$TEST_GROUP" = "scan-engines" ]; then + echo -e "${YELLOW}Running cleandata to clean up projects...${NC}" + go test -v github.com/checkmarx/ast-cli/test/cleandata || true +fi + +# Final cleanup +rm -f "$FAILED_TESTS_FILE" test_output.log + +# Exit with appropriate status +if [ $status -ne 0 ] || [ ${rerun_status:-0} -eq 1 ]; then + echo -e "${RED}Integration tests failed for group: ${TEST_GROUP}${NC}" + exit 1 +else + echo -e "${GREEN}Integration tests passed for group: ${TEST_GROUP}${NC}" + exit 0 +fi + diff --git a/scripts/analyze-test-groups.sh b/scripts/analyze-test-groups.sh new file mode 100644 index 000000000..81b38b9f2 --- /dev/null +++ b/scripts/analyze-test-groups.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Script to analyze and validate integration test grouping +# Usage: ./scripts/analyze-test-groups.sh + +set -e + +echo "=== Integration Test Grouping Analysis ===" +echo "" + +# Get all integration test functions +echo "Extracting all integration test functions..." +ALL_TESTS=$(go test -tags integration -list . ./test/integration 2>/dev/null | grep "^Test" || true) +TOTAL_TESTS=$(echo "$ALL_TESTS" | wc -l) + +echo "Total integration tests found: $TOTAL_TESTS" +echo "" + +# Define test patterns for each group (same as in integration_up_parallel.sh) +declare -A TEST_GROUPS +TEST_GROUPS["fast-validation"]="^Test(Auth|Configuration|Tenant|FeatureFlags|Predicate|Logs)" +TEST_GROUPS["scan-core"]="^TestScan(Create|List|Show|Delete|Workflow|Logs|Filter|Threshold|Resubmit|Types)" +TEST_GROUPS["scan-engines"]="^Test(Container|Scs|CreateScan_With.*Engine|.*ApiSecurity|.*ExploitablePath)" +TEST_GROUPS["scm-integration"]="^Test(PR|UserCount)" +TEST_GROUPS["realtime-features"]="^Test(Kics|Sca|Oss|Secrets|Containers)Realtime|^TestRun.*Realtime" +TEST_GROUPS["advanced-features"]="^Test(Project|Result|Import|Bfl|Asca|Chat|Learn|Telemetry|RateLimit|PreCommit|PreReceive|Remediation)" + +# Analyze each group +echo "=== Test Distribution by Group ===" +echo "" + +declare -A GROUP_COUNTS +TOTAL_MATCHED=0 + +for group in "${!TEST_GROUPS[@]}"; do + pattern="${TEST_GROUPS[$group]}" + + # Count tests matching this pattern + count=$(echo "$ALL_TESTS" | grep -E "$pattern" | wc -l) + GROUP_COUNTS[$group]=$count + TOTAL_MATCHED=$((TOTAL_MATCHED + count)) + + printf "%-25s: %3d tests\n" "$group" "$count" +done + +echo "" +echo "Total tests matched: $TOTAL_MATCHED" +UNMATCHED=$((TOTAL_TESTS - TOTAL_MATCHED)) +echo "Unmatched tests: $UNMATCHED" + +if [ $UNMATCHED -gt 0 ]; then + echo "" + echo "=== Unmatched Tests ===" + for test in $ALL_TESTS; do + matched=false + for pattern in "${TEST_GROUPS[@]}"; do + if echo "$test" | grep -qE "$pattern"; then + matched=true + break + fi + done + + if [ "$matched" = false ]; then + echo " - $test" + fi + done +fi + +echo "" +echo "=== Detailed Test Listing by Group ===" +echo "" + +for group in fast-validation scan-core scan-engines scm-integration realtime-features advanced-features; do + pattern="${TEST_GROUPS[$group]}" + count="${GROUP_COUNTS[$group]}" + + echo "[$group] ($count tests)" + echo "Pattern: $pattern" + echo "Tests:" + + echo "$ALL_TESTS" | grep -E "$pattern" | sed 's/^/ - /' || echo " (none)" + echo "" +done + +# Check for duplicate matches +echo "=== Checking for Duplicate Test Assignments ===" +echo "" + +declare -A TEST_ASSIGNMENTS + +for test in $ALL_TESTS; do + groups_matched="" + + for group in "${!TEST_GROUPS[@]}"; do + pattern="${TEST_GROUPS[$group]}" + if echo "$test" | grep -qE "$pattern"; then + if [ -z "$groups_matched" ]; then + groups_matched="$group" + else + groups_matched="$groups_matched, $group" + fi + fi + done + + if [ ! -z "$groups_matched" ]; then + TEST_ASSIGNMENTS[$test]="$groups_matched" + fi +done + +DUPLICATES_FOUND=false +for test in "${!TEST_ASSIGNMENTS[@]}"; do + groups="${TEST_ASSIGNMENTS[$test]}" + if [[ "$groups" == *","* ]]; then + echo "⚠️ $test matches multiple groups: $groups" + DUPLICATES_FOUND=true + fi +done + +if [ "$DUPLICATES_FOUND" = false ]; then + echo "✅ No duplicate test assignments found" +fi + +echo "" +echo "=== Summary ===" +echo "Total tests: $TOTAL_TESTS" +echo "Matched tests: $TOTAL_MATCHED" +echo "Unmatched tests: $UNMATCHED" +echo "Coverage: $(awk "BEGIN {printf \"%.1f%%\", ($TOTAL_MATCHED/$TOTAL_TESTS)*100}")" + +if [ $UNMATCHED -eq 0 ] && [ "$DUPLICATES_FOUND" = false ]; then + echo "" + echo "✅ Test grouping is valid and complete!" + exit 0 +else + echo "" + echo "⚠️ Test grouping needs adjustment" + exit 1 +fi