language_test.go

// Copyright 2016-2024, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
	testingrpc "github.com/pulumi/pulumi/sdk/v3/proto/go/testing"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

func runTestingHost(t *testing.T) (string, testingrpc.LanguageTestClient) {
	// We can't just go run the pulumi-test-language package because of
	// https://github.com/golang/go/issues/39172, so we build it to a temp file then run that.
	binary := t.TempDir() + "/pulumi-test-language"
	cmd := exec.CommandContext(t.Context(),
		"go", "build", "-o", binary, "github.com/pulumi/pulumi/pkg/v3/testing/pulumi-test-language")
	output, err := cmd.CombinedOutput()
	t.Logf("build output: %s", output)
	require.NoError(t, err)

	cmd = exec.Command(binary)
	stdout, err := cmd.StdoutPipe()
	require.NoError(t, err)
	stderr, err := cmd.StderrPipe()
	require.NoError(t, err)
	stderrReader := bufio.NewReader(stderr)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		for {
			text, err := stderrReader.ReadString('\n')
			if err != nil {
				wg.Done()
				return
			}
			t.Logf("engine: %s", text)
		}
	}()

	err = cmd.Start()
	require.NoError(t, err)

	stdoutBytes, err := io.ReadAll(stdout)
	require.NoError(t, err)

	address := string(stdoutBytes)

	conn, err := grpc.NewClient(
		address,
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.WithUnaryInterceptor(rpcutil.OpenTracingClientInterceptor()),
		grpc.WithStreamInterceptor(rpcutil.OpenTracingStreamClientInterceptor()),
		rpcutil.GrpcChannelOptions(),
	)
	require.NoError(t, err)

	client := testingrpc.NewLanguageTestClient(conn)

	t.Cleanup(func() {
		assert.NoError(t, cmd.Process.Kill())
		wg.Wait()
		// We expect this to error because we just killed it.
		contract.IgnoreError(cmd.Wait())
	})

	return address, client
}

// Add test names here that are expected to fail and the reason why they are failing
var expectedFailures = map[string]string{
	"l1-builtin-can":                   "#489 codegen not implemented",
	"l1-builtin-try":                   "#490 codegen not implemented",
	"l1-keyword-overlap":               "#493 update to pulumi 1.50 conformance failure",
	"l2-resource-provider-inheritance": "No best type found for implicitly-typed array",
	"l2-resource-asset-archive": "" +
		"The namespace 'Pulumi.AssetArchive' conflicts with the type 'AssetArchive' in 'Pulumi, Version=1.0.0.0",
	"l2-resource-config":                    "sdk packing for config: build error before pack",
	"l1-output-array":                       "error CS0826: No best type found for implicitly-typed array",
	"l1-output-map":                         "Same error as with arrays about implicitly typed maps",
	"l2-provider-grpc-config":               "dotnet build failed",
	"l2-provider-grpc-config-secret":        "dotnet build failed",
	"l2-provider-grpc-config-schema":        "dotnet build failed",
	"l2-provider-grpc-config-schema-secret": "dotnet build failed",
	"l2-proxy-index":                        "dotnet build failed",
	"l2-invoke-options-depends-on":          "dotnet build failed",
	"l2-invoke-scalar": "" +
		"result contains invalid type Dictionary: only ImmutableArray and ImmutableDictionary allowed",
	"l2-invoke-scalars": "" +
		"result contains invalid type Dictionary: only ImmutableArray and ImmutableDictionary allowed",
	"l2-invoke-secrets": "" +
		"Pulumi.Deployment+InvokeException: 'simple-invoke:index:secretInvoke' failed: value is not a string",
	"l2-map-keys":                   "dotnet build failed",
	"l2-resource-secret":            "test hanging",
	"l2-namespaced-provider":        "error CS0117: 'ResourceArgs' does not contain a definition for 'ResourceRef'", //nolint:lll
	"l2-union":                      "dotnet build failed",
	"l2-resource-option-alias":      "aliases not recognized: expected 0 create operations but got 3",
	"l2-resource-option-hide-diffs": "programgen bug: https://github.com/pulumi/pulumi/issues/20665",
	"l2-parameterized-invoke": "dotnet build failed: " +
		"DoHelloWorld does not exist in namespace Pulumi.Subpackage",
	"l2-parameterized-resource-twice":        "testdata not yet generated for .NET",
	"l2-resource-option-replacement-trigger": "dotnet build failed: Pulumi.Output namespace conflict",
	"l2-resource-option-replace-with":        "not yet implemented",
	"l1-config-types-object":                 "dotnet build failed: Cannot initialize type 'object' with a collection initializer", //nolint:lll
	"l1-elide-index":                         "https://github.com/pulumi/pulumi-dotnet/issues/865",
	"l2-elide-index":                         "https://github.com/pulumi/pulumi-dotnet/issues/868",
	"l2-module-format":                       "https://github.com/pulumi/pulumi-dotnet/issues/867",

	// TODO: This is a codegen bug, we're translating {} to null
	"l2-plain": "System.ArgumentNullException: [Input] Pulumi.Plain.Inputs.DataArgs._stringMap is required but was not given a value (Parameter '_stringMap')", //nolint:lll

	"l1-builtin-object":          "Fail after updating to 3.224: KeyNotFoundException: The given key 'keyMissing' was not present in the dictionary. ", //nolint:lll
	"l2-resource-elide-unknowns": "Fail after updating to 3.224: conflict in Output type name",                                                         //nolint:lll
	"l2-camel-names":             "Fail after updating to 3.224: 'SomeResourceArgs' does not contain a definition for 'ResourceName'",                  //nolint:lll
	"l2-resource-name-type":      "Fail after updating to 3.224: TODO: call pulumiResourceName",                                                        //nolint:lll
	"l2-resource-names":          "Fail after updating to 3.224: The type or namespace name 'Mod' does not exist in the namespace 'Pulumi.Names'",      //nolint:lll
	"l2-builtin-object":          "Fail after updating to 3.224: Invalid expression term ')'",

	"l3-range": "Fail after updating to 3.225",

	"l2-ref-ref": "Fail after updating to 3.229: dotnet build failed",

	"l2-resource-optional":        "Fail after updating to 3.229: dotnet build failed: Cannot implicitly convert type 'int[]' to 'Pulumi.InputList<double>'", //nolint:lll
	"l3-component-config-objects": "Fail after updating to 3.229: dotnet build failed: Cannot implicitly convert type 'Output<dynamic>' to 'InputMap<bool>'", //nolint:lll

	"l3-for":              "Fail after updating to 3.226",
	"l2-snake-names":      "Fail after updating to 3.226",
	"l3-for-resource":     "Fail after updating to 3.226",
	"l2-keywords":         "SDK build failed: duplicate Lambda definition and member name conflicts with enclosing type",
	"l1-builtin-to-json":  "dotnet build failed: CS0623 array initializers and CS0820 implicitly-typed variable errors",
	"l3-deferred-outputs": "dotnet build failed: operator '!' cannot be applied to Input<bool> and undefined names", //nolint:lll

	"l2-resource-config-objects": "dotnet build failed: Cannot implicitly convert type 'Output<dynamic>' to 'InputList<double>'",             //nolint:lll
	"l3-rewrite-conversions":     "dotnet build failed: multiple type conversion errors (int[] to InputList<double>, string to Input<bool>)", //nolint:lll
	// l3-range-ref was split by range kind in pulumi/pulumi#23632 (v3.248.0); all three inherit the
	// original "dotnet build failed: List<Target> missing K1/Name members" range-ref codegen bug.
	"l3-range-list-ref":                  "Fail after updating to 3.248: dotnet build failed: List<Target> missing K1/Name members", //nolint:lll
	"l3-range-map-ref":                   "Fail after updating to 3.248: dotnet build failed: List<Target> missing K1/Name members", //nolint:lll
	"l3-range-bool-ref":                  "Fail after updating to 3.248: dotnet build failed: List<Target> missing K1/Name members", //nolint:lll
	"l2-resource-option-custom-timeouts": "https://github.com/pulumi/pulumi-dotnet/issues/822",

	"l2-raw-string-bytes": "the .NET SDK does not set accepts_byte_string: strings containing non-UTF8 bytes cannot be received from the engine (added in v3.255.0)", //nolint:lll
	"l3-component-invoke": "dotnet build failed: CS0029 cannot convert Output<string> to Output<dynamic> (added in v3.255.0)",                                        //nolint:lll
	// Not a v3.255.0 regression: the runner ignored preview errors until pulumi/pulumi#24058. The .NET
	// SDK serializes an unknown resource-reference ID as "" and the engine's provider-method Call path
	// builds the provider ref from __self__.id verbatim, yielding the invalid ref 'urn::'.
	"l2-provider-call-explicit": "provider method calls fail in preview: engine rejects provider ref 'urn::' built from empty __self__.id", //nolint:lll

	"l3-component-nested":                "Fail after updating to 3.232",
	"l1-builtin-min-max":                 "Fail after updating to 3.232",
	"l3-component-primitive-conversions": "Fail after updating to 3.232",

	"l2-resource-read":        "Fail after updating to 3.234: read resource codegen not implemented",
	"l2-component-call-plain": "Fail after updating to 3.234: plain method call codegen not implemented",

	"l2-map-keys-adversarial":  "Fail after updating to 3.239",
	"l1-config-types-optional": "Fail after updating to 3.239",

	"l1-expand-final": "Fail after updating to 3.243",

	"l2-primitive-ref-optional":     "Fail after updating to 3.246: dotnet build failed: Cannot implicitly convert type 'int[]' to 'Pulumi.InputList<double>'", //nolint:lll
	"l2-resource-any":               "Fail after updating to 3.246: dotnet build failed: CS0826 no best type found for implicitly-typed array",                 //nolint:lll
	"l2-resource-schema-secret":     "Fail after updating to 3.246: dotnet build failed: Pulumi.Output namespace conflict",                                     //nolint:lll
	"l2-provider-config-enum":       "Fail after updating to 3.246: run bailed",
	"l2-config-default-from-invoke": "Fail after updating to 3.246: stack output \"result\" missing",

	"l2-docs": "dotnet codegen does not resolve '{{% ref %}}' doc cross-references in generated comments",

	"l3-range-invoke-output-traversal": "Fail after updating to 3.250: dotnet build failed: CS0021 cannot index Output<ImmutableArray<string>>", //nolint:lll

	"l2-failed-create-recover-continue-on-error": "Fail after updating to 3.252: CS0029: Cannot implicitly convert type 'string' to 'Pulumi.Input<bool>'", //nolint:lll

	"l2-invoke-depends-on-component":     "the .NET SDK does not send dependsOn on invokes: the invoke result is not unknown while its dependencies are pending creation (added in v3.256.0)",            //nolint:lll
	"l2-target-up-skipped-create-output": "the .NET SDK does not handle RegisterResourceResponse.unknown: outputs of skipped creates resolve as empty instead of unknown (added in v3.256.0)",            //nolint:lll
	"l3-component-provider-inheritance":  "the .NET SDK does not inherit the providers map into remote components: an extra default provider is created instead of the explicit one (added in v3.256.0)", //nolint:lll

	"l1-for-expression":      "dotnet build failed: programgen emits invalid C# for `for` expressions: CS1525 invalid expression term '.' (added in v3.257.0)",                                           //nolint:lll
	"l2-nested-collections":  "dotnet build failed: sdkgen emits plain nested dictionaries instead of typed outputs for deeply nested collections: CS1061 no definition for 'Outer' (added in v3.257.0)", //nolint:lll
	"l2-reserved-names":      "sdk packing for reservednames: CS0542 'ElementType' member name cannot be the same as its enclosing type (added in v3.257.0)",                                             //nolint:lll
	"l2-resource-hook-panic": "dotnet build failed: programgen emits invalid hook argument: CS1503 cannot convert from 'object' to 'System.Diagnostics.ProcessStartInfo' (added in v3.257.0)",            //nolint:lll
}

// Add program overrides here for programs that can't yet be generated correctly due to programgen bugs.
var programOverrides = map[string]*testingrpc.PrepareLanguageTestsRequest_ProgramOverride{
	// TODO[pulumi/pulumi#18741]: Remove when programgen support for call is implemented.
	"l2-component-property-deps": {
		Paths: []string{
			filepath.Join("testdata", "overrides", "l2-component-property-deps"),
		},
	},
}

func TestLanguage(t *testing.T) {
	t.Parallel()

	engineAddress, engine := runTestingHost(t)
	err := exec.CommandContext(t.Context(), "dotnet", "nuget", "locals", "all", "--clear").Run()
	require.NoError(t, err)

	tests, err := engine.GetLanguageTests(t.Context(), &testingrpc.GetLanguageTestsRequest{})
	require.NoError(t, err)

	cancel := make(chan bool)

	// Run the language plugin
	handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
		Init: func(srv *grpc.Server) error {
			host := newLanguageHost(engineAddress, "", "")
			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
			return nil
		},
		Cancel: cancel,
	})
	require.NoError(t, err)

	// Create a temp project dir for the test to run in
	rootDir := t.TempDir()

	snapshotDir := "./testdata/"

	// Prepare to run the tests
	prepare, err := engine.PrepareLanguageTests(t.Context(), &testingrpc.PrepareLanguageTestsRequest{
		LanguagePluginName:   "dotnet",
		LanguagePluginTarget: fmt.Sprintf("127.0.0.1:%d", handle.Port),
		TemporaryDirectory:   rootDir,
		SnapshotDirectory:    snapshotDir,
		CoreSdkDirectory:     "../sdk/Pulumi",
		SnapshotEdits: []*testingrpc.PrepareLanguageTestsRequest_Replacement{
			{
				Pattern:     rootDir + "/artifacts",
				Replacement: "ROOT/artifacts",
			},
		},
		ProgramOverrides: programOverrides,
	})
	require.NoError(t, err)

	for _, tt := range tests.Tests {
		tt := tt
		t.Run(tt, func(t *testing.T) {
			t.Parallel()
			if expected, ok := expectedFailures[tt]; ok {
				t.Skipf("test %s is expected to fail: %s", tt, expected)
			}
			if strings.HasPrefix(tt, "policy-") {
				t.Skipf("dotnet doesn't support policy tests yet: %s", tt)
			}
			if strings.HasPrefix(tt, "provider-") {
				t.Skipf("dotnet doesn't support provider tests yet: %s", tt)
			}

			result, err := engine.RunLanguageTest(t.Context(), &testingrpc.RunLanguageTestRequest{
				Token: prepare.Token,
				Test:  tt,
			})

			require.NoError(t, err)
			for _, msg := range result.Messages {
				t.Log(msg)
			}
			t.Logf("stdout: %s", result.Stdout)
			t.Logf("stderr: %s", result.Stderr)
			assert.True(t, result.Success)
		})
	}

	t.Cleanup(func() {
		close(cancel)
		assert.NoError(t, <-handle.Done)
	})
}