NCO-58: Flesh out functional testing #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Functional Tests | |
| permissions: | |
| contents: read | |
| packages: read | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| version_tag: | |
| description: "Optional release tag (e.g. 1.0.0 or 1.0.0-rc1)" | |
| required: false | |
| type: string | |
| cbdc_version: | |
| description: "Optional cbdinocluster version (e.g. v0.0.88)" | |
| required: false | |
| type: string | |
| env: | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 | |
| DOTNET_NOLOGO: 1 | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| functional-tests: | |
| name: Functional Tests (ubuntu-22.04) | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout (tag) | |
| if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version_tag != '' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: refs/tags/${{ github.event.inputs.version_tag }} | |
| - name: Checkout (PR head) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout (default) | |
| if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.version_tag == '' || github.event.inputs.version_tag == null) }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup .NET SDK | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| cache: false | |
| - name: Install cbdinocluster | |
| id: install-cbdc | |
| run: | | |
| mkdir -p "$HOME/bin" | |
| CBDC_VER=${{ github.event.inputs.cbdc_version }} | |
| if [ -z "$CBDC_VER" ]; then | |
| CBDC_VER=v0.0.88 | |
| fi | |
| echo "Installing cbdinocluster $CBDC_VER" | |
| wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/${CBDC_VER}/cbdinocluster-linux-amd64 | |
| chmod +x $HOME/bin/cbdinocluster | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Initialize cbdinocluster | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cbdinocluster -v init --auto | |
| - name: Start Couchbase Analytics cluster | |
| id: start-cluster | |
| env: | |
| INPUT_TAG: ${{ github.event.inputs.version_tag }} | |
| run: | | |
| TAG=${INPUT_TAG} | |
| EA_VERSION=2.2.0-1166 | |
| export EA_VERSION | |
| cat > cluster.yml <<'YAML' | |
| columnar: true | |
| nodes: | |
| - count: 2 | |
| version: ${EA_VERSION} | |
| docker: | |
| passive-load-balancer: true | |
| use-dino-certs: true | |
| jwt: true | |
| YAML | |
| # Substitute shell vars (e.g. EA_VERSION) into YAML. | |
| # envsubst preserves YAML structure better than eval+echo. | |
| envsubst < cluster.yml > cluster.resolved.yml | |
| echo "Resolved cluster definition:" && cat cluster.resolved.yml | |
| CBDINO_CLUSTER_ID=$(cbdinocluster -v alloc --def="$(cat cluster.resolved.yml)") | |
| # Connection string through the load balancer (used by most tests) | |
| CBDINO_CONNSTR=$(cbdinocluster -v connstr --tls --analytics "$CBDINO_CLUSTER_ID") | |
| # Direct-to-node connection string (required for mTLS tests because | |
| # the nginx passive load balancer terminates TLS at L7, so client | |
| # certificates are not forwarded to the analytics service). | |
| # Extract the first cluster node IP from cbdinocluster ps --json. | |
| NODE_IP=$(cbdinocluster ps --json 2>/dev/null \ | |
| | python3 -c "import sys,json; data=json.load(sys.stdin); [print(n['ip_address']) or sys.exit(0) for c in data for n in c.get('nodes',[]) if n.get('is_cluster_node')]" 2>/dev/null \ | |
| || echo "") | |
| if [ -n "$NODE_IP" ]; then | |
| CBDINO_CONNSTR_DIRECT="https://${NODE_IP}:18095" | |
| else | |
| # No LB or couldn't parse — fall back to the primary connstr | |
| CBDINO_CONNSTR_DIRECT="$CBDINO_CONNSTR" | |
| fi | |
| echo "CBDINO_CLUSTER_ID=$CBDINO_CLUSTER_ID" >> "$GITHUB_ENV" | |
| echo "CBDINO_CONNSTR=$CBDINO_CONNSTR" >> "$GITHUB_ENV" | |
| echo "CBDINO_CONNSTR_DIRECT=$CBDINO_CONNSTR_DIRECT" >> "$GITHUB_ENV" | |
| echo "CBDINO_USER=Administrator" >> "$GITHUB_ENV" | |
| echo "CBDINO_PASS=password" >> "$GITHUB_ENV" | |
| echo "Load balancer connstr: $CBDINO_CONNSTR" | |
| echo "Direct node connstr: $CBDINO_CONNSTR_DIRECT" | |
| - name: Create JWT test user and generate token | |
| run: | | |
| echo "Creating jwt-test-user on cluster $CBDINO_CLUSTER_ID" | |
| cbdinocluster -v users add "$CBDINO_CLUSTER_ID" jwt-test-user \ | |
| --password testpass --can-read --can-write | |
| echo "Generating JWT for jwt-test-user" | |
| CBDINO_JWT=$(cbdinocluster jwt generate jwt-test-user --can-read --can-write) | |
| echo "CBDINO_JWT=$CBDINO_JWT" >> "$GITHUB_ENV" | |
| - name: Generate mTLS client certificates | |
| run: | | |
| echo "Creating mtls-swap-user for certificate swap test" | |
| cbdinocluster -v users add "$CBDINO_CLUSTER_ID" mtls-swap-user \ | |
| --password testpass --can-read --can-write | |
| echo "Generating client certificate for Administrator" | |
| CERT_OUTPUT=$(cbdinocluster certificates get-client-cert Administrator) | |
| # cbdinocluster outputs PEM cert followed by PEM key | |
| echo "$CERT_OUTPUT" | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/' > /tmp/client-cert.pem | |
| echo "$CERT_OUTPUT" | awk '/BEGIN RSA PRIVATE KEY/,/END RSA PRIVATE KEY/' > /tmp/client-key.pem | |
| echo "Generating client certificate for mtls-swap-user" | |
| CERT_OUTPUT_2=$(cbdinocluster certificates get-client-cert mtls-swap-user) | |
| echo "$CERT_OUTPUT_2" | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/' > /tmp/client-cert-2.pem | |
| echo "$CERT_OUTPUT_2" | awk '/BEGIN RSA PRIVATE KEY/,/END RSA PRIVATE KEY/' > /tmp/client-key-2.pem | |
| echo "CBDINO_CLIENT_CERT_PATH=/tmp/client-cert.pem" >> "$GITHUB_ENV" | |
| echo "CBDINO_CLIENT_KEY_PATH=/tmp/client-key.pem" >> "$GITHUB_ENV" | |
| echo "CBDINO_CLIENT_CERT_PATH_2=/tmp/client-cert-2.pem" >> "$GITHUB_ENV" | |
| echo "CBDINO_CLIENT_KEY_PATH_2=/tmp/client-key-2.pem" >> "$GITHUB_ENV" | |
| - name: Prepare Functional Test settings.json | |
| run: | | |
| echo "Writing functional test settings.json with cluster connection string" | |
| cat > tests/Couchbase.Analytics.FunctionalTests/settings.json <<EOF | |
| { | |
| "TestSettings": { | |
| "ConnectionString": "${CBDINO_CONNSTR}", | |
| "DirectConnectionString": "${CBDINO_CONNSTR_DIRECT}", | |
| "Username": "${CBDINO_USER}", | |
| "Password": "${CBDINO_PASS}", | |
| "JwtToken": "${CBDINO_JWT}", | |
| "ClientCertPath": "${CBDINO_CLIENT_CERT_PATH}", | |
| "ClientKeyPath": "${CBDINO_CLIENT_KEY_PATH}", | |
| "ClientCertPath2": "${CBDINO_CLIENT_CERT_PATH_2}", | |
| "ClientKeyPath2": "${CBDINO_CLIENT_KEY_PATH_2}" | |
| } | |
| } | |
| EOF | |
| echo "settings.json written:" && cat tests/Couchbase.Analytics.FunctionalTests/settings.json | |
| - name: Wait for analytics service to be ready | |
| run: | | |
| echo "Polling analytics service until it accepts queries..." | |
| for i in $(seq 1 60); do | |
| HTTP_CODE=$(curl -sS -o /dev/null -w "%{http_code}" -k \ | |
| -u "${CBDINO_USER}:${CBDINO_PASS}" \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"statement": "SELECT 1;"}' \ | |
| "${CBDINO_CONNSTR}/api/v1/request" 2>/dev/null || echo "000") | |
| if [ "$HTTP_CODE" = "200" ]; then | |
| echo "Analytics service is ready (attempt $i)" | |
| break | |
| fi | |
| echo "Attempt $i/60: HTTP $HTTP_CODE — waiting 5s..." | |
| sleep 5 | |
| done | |
| if [ "$HTTP_CODE" != "200" ]; then | |
| echo "ERROR: Analytics service did not become ready after 5 minutes" | |
| exit 1 | |
| fi | |
| - name: Create test dataverse for scope-level queries | |
| run: | | |
| echo "Creating test dataverse 'testscope' for scope-level functional tests" | |
| HTTP_CODE=$(curl -sS -o /tmp/dataverse-response.json -w "%{http_code}" -k \ | |
| -u "${CBDINO_USER}:${CBDINO_PASS}" \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"statement": "CREATE DATAVERSE testscope IF NOT EXISTS;"}' \ | |
| "${CBDINO_CONNSTR}/api/v1/request") | |
| echo "HTTP status: $HTTP_CODE" | |
| cat /tmp/dataverse-response.json | |
| echo "" | |
| if [ "$HTTP_CODE" != "200" ]; then | |
| echo "ERROR: Dataverse creation failed with HTTP $HTTP_CODE" | |
| exit 1 | |
| fi | |
| echo "Dataverse created successfully" | |
| - name: Build and Run Functional Tests | |
| run: | | |
| dotnet test tests/Couchbase.Analytics.FunctionalTests/Couchbase.Analytics.FunctionalTests.csproj \ | |
| --configuration Release \ | |
| --logger "trx;LogFileName=functional-tests.trx" \ | |
| --results-directory "TestResults" | |
| - name: Verify Functional Test Results | |
| run: | | |
| python3 - TestResults/functional-tests.trx <<'PY' | |
| import sys | |
| import xml.etree.ElementTree as ET | |
| if len(sys.argv) != 2: | |
| print(f"Usage: {sys.argv[0]} <results.trx>") | |
| sys.exit(1) | |
| path = sys.argv[1] | |
| print(f"Parsing TRX: {path}") | |
| try: | |
| tree = ET.parse(path) | |
| except Exception as e: | |
| print(f"ERROR: Failed to parse TRX file: {e}") | |
| sys.exit(1) | |
| root = tree.getroot() | |
| ns = {'ns': root.tag.split('}')[0].strip('{')} if root.tag.startswith('{') else {} | |
| rs = root.find('.//ns:ResultSummary', ns) | |
| if rs is None: | |
| rs = root.find('.//ResultSummary') | |
| if rs is None: | |
| print('ERROR: ResultSummary not found in TRX') | |
| sys.exit(1) | |
| counters = rs.find('.//ns:Counters', ns) | |
| if counters is None: | |
| counters = rs.find('.//Counters') | |
| if counters is None: | |
| print('ERROR: Counters not found in TRX') | |
| sys.exit(1) | |
| def get_int(name: str) -> int: | |
| v = counters.attrib.get(name) | |
| try: | |
| return int(v) if v is not None else 0 | |
| except ValueError: | |
| return 0 | |
| total = get_int('total') | |
| executed = get_int('executed') | |
| passed = get_int('passed') | |
| failed = ( | |
| get_int('failed') + | |
| get_int('error') + | |
| get_int('timeout') + | |
| get_int('aborted') | |
| ) | |
| print(f"Summary: total={total}, executed={executed}, passed={passed}, failed={failed}") | |
| if total == 0 or executed == 0: | |
| print('ERROR: No tests were run.') | |
| sys.exit(1) | |
| if failed > 0: | |
| print(f'ERROR: {failed} tests failed.') | |
| sys.exit(1) | |
| if passed < executed: | |
| print(f'ERROR: Not all executed tests passed (passed={passed}, executed={executed}).') | |
| sys.exit(1) | |
| print('All functional tests passed.') | |
| PY | |
| - name: Upload Functional Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: functional-test-results | |
| path: TestResults | |
| - name: Teardown cbdinocluster | |
| if: always() | |
| run: | | |
| if [ -n "${CBDINO_CLUSTER_ID}" ]; then | |
| cbdinocluster rm "$CBDINO_CLUSTER_ID" || true | |
| fi |