|
| 1 | +//go:build e2e |
| 2 | + |
| 3 | +package e2e |
| 4 | + |
| 5 | +import ( |
| 6 | + "context" |
| 7 | + "os" |
| 8 | + "testing" |
| 9 | + "time" |
| 10 | + |
| 11 | + flagd "github.com/open-feature/go-sdk-contrib/providers/flagd/pkg" |
| 12 | + "github.com/open-feature/go-sdk-contrib/tests/flagd/testframework" |
| 13 | + "github.com/open-feature/go-sdk-contrib/tools/provider-tck/pkg/tck" |
| 14 | + "github.com/open-feature/go-sdk/openfeature" |
| 15 | +) |
| 16 | + |
| 17 | +// The OpenFeature Provider Conformance Suite, run against the flagd provider in |
| 18 | +// both of its resolver modes. |
| 19 | +// |
| 20 | +// flagd resolves flags two quite different ways — RPC evaluates remotely over |
| 21 | +// gRPC, in-process syncs the ruleset and evaluates locally — and they are |
| 22 | +// separate suites because they are separately conformant. Any difference |
| 23 | +// between the two results is a difference an application would see when it |
| 24 | +// switches resolver, which is exactly the kind of thing the suite exists to |
| 25 | +// surface. |
| 26 | +// |
| 27 | +// The existing e2e suites in this package are untouched, and so is |
| 28 | +// flagd-testbed. The TCK drives the testbed's launchpad through the |
| 29 | +// standardised control API, which the launchpad already implements. |
| 30 | + |
| 31 | +// TestFlagdRPCConformance runs the suite against the RPC resolver. |
| 32 | +func TestFlagdRPCConformance(t *testing.T) { |
| 33 | + runConformance(t, conformanceSuite{ |
| 34 | + name: "flagd-rpc", |
| 35 | + portName: "rpc", |
| 36 | + resolver: flagd.WithRPCResolver(), |
| 37 | + |
| 38 | + // tck.Stale is NOT declared, and that is a finding rather than a |
| 39 | + // configuration choice. |
| 40 | + // |
| 41 | + // The RPC resolver emits PROVIDER_ERROR, PROVIDER_READY and |
| 42 | + // PROVIDER_CONFIGURATION_CHANGED, and never emits PROVIDER_STALE — see |
| 43 | + // pkg/service/rpc/service.go, where losing the stream sends |
| 44 | + // of.ProviderError directly. The in-process resolver, by contrast, |
| 45 | + // emits PROVIDER_STALE on connection loss and only escalates to |
| 46 | + // PROVIDER_ERROR once the retry grace period expires, which is the |
| 47 | + // behaviour the specification describes. |
| 48 | + // |
| 49 | + // So the two resolvers of the same provider report an outage |
| 50 | + // differently: an application that switches from in-process to RPC |
| 51 | + // silently stops receiving stale events. Withholding the capability |
| 52 | + // here reports the @stale scenario as skipped with its reason rather |
| 53 | + // than failing it, and the gap needs its own issue against the |
| 54 | + // provider. Declare this as soon as the RPC resolver emits |
| 55 | + // PROVIDER_STALE. |
| 56 | + capabilities: []tck.Capability{ |
| 57 | + tck.Events, |
| 58 | + tck.ConfigurationChange, |
| 59 | + tck.Object, |
| 60 | + tck.UnavailableInit, |
| 61 | + tck.StrictNumericTyping, |
| 62 | + }, |
| 63 | + |
| 64 | + // The RPC resolver asks flagd to resolve each flag, so it is ready as |
| 65 | + // soon as the stream is up. |
| 66 | + readyTimeout: 30 * time.Second, |
| 67 | + gracePeriod: 10, |
| 68 | + }) |
| 69 | +} |
| 70 | + |
| 71 | +// TestFlagdInProcessConformance runs the suite against the in-process resolver. |
| 72 | +func TestFlagdInProcessConformance(t *testing.T) { |
| 73 | + runConformance(t, conformanceSuite{ |
| 74 | + name: "flagd-in-process", |
| 75 | + portName: "in-process", |
| 76 | + resolver: flagd.WithInProcessResolver(), |
| 77 | + |
| 78 | + // The full set. The in-process resolver emits PROVIDER_STALE on |
| 79 | + // connection loss, so unlike RPC it can satisfy the @stale scenario. |
| 80 | + capabilities: tck.AllCapabilities(), |
| 81 | + |
| 82 | + // In-process syncs the whole ruleset before reporting ready, so it |
| 83 | + // needs longer than RPC to initialise. |
| 84 | + readyTimeout: 60 * time.Second, |
| 85 | + |
| 86 | + // The grace period has to outlast the outage in the @stale scenario. |
| 87 | + // The resolver goes STALE immediately on connection loss and escalates |
| 88 | + // to ERROR when this expires, so too short a value would turn a |
| 89 | + // scenario about staleness into one about failure. |
| 90 | + gracePeriod: 30, |
| 91 | + }) |
| 92 | +} |
| 93 | + |
| 94 | +// conformanceSuite is the per-resolver configuration. |
| 95 | +type conformanceSuite struct { |
| 96 | + name string |
| 97 | + portName string |
| 98 | + resolver flagd.ProviderOption |
| 99 | + capabilities []tck.Capability |
| 100 | + readyTimeout time.Duration |
| 101 | + gracePeriod int |
| 102 | +} |
| 103 | + |
| 104 | +func runConformance(t *testing.T, suite conformanceSuite) { |
| 105 | + if testing.Short() { |
| 106 | + t.Skip("skipping e2e tests in short mode") |
| 107 | + } |
| 108 | + |
| 109 | + ctx := context.Background() |
| 110 | + |
| 111 | + // The launchpad rewrites flag files into this directory. It is per-suite so |
| 112 | + // the two resolver suites cannot disturb each other. |
| 113 | + flagsDir, err := os.MkdirTemp("", "flagd-tck-*") |
| 114 | + if err != nil { |
| 115 | + t.Fatalf("could not create a flags directory: %v", err) |
| 116 | + } |
| 117 | + t.Cleanup(func() { _ = os.RemoveAll(flagsDir) }) |
| 118 | + |
| 119 | + // The stack is started once for the whole suite and never restarted. |
| 120 | + // Scenario isolation comes from the control API instead — see the |
| 121 | + // no-container-restart invariant in the control API specification. |
| 122 | + container, err := testframework.NewFlagdContainer(ctx, testframework.FlagdContainerConfig{ |
| 123 | + TestbedDir: "../flagd-testbed", |
| 124 | + FlagsDir: flagsDir, |
| 125 | + ExtraWaitTime: 2 * time.Second, |
| 126 | + }) |
| 127 | + if err != nil { |
| 128 | + t.Fatalf("could not start the flagd testbed: %v", err) |
| 129 | + } |
| 130 | + t.Cleanup(func() { |
| 131 | + if err := container.Stop(); err != nil { |
| 132 | + t.Logf("could not stop the flagd testbed: %v", err) |
| 133 | + } |
| 134 | + }) |
| 135 | + |
| 136 | + control, err := tck.NewHTTPControl(tck.HTTPControlOptions{ |
| 137 | + BaseURL: container.GetLaunchpadURL(), |
| 138 | + }) |
| 139 | + if err != nil { |
| 140 | + t.Fatalf("could not build the backend control: %v", err) |
| 141 | + } |
| 142 | + |
| 143 | + // Read once, after the stack is up: the testbed maps host ports |
| 144 | + // dynamically, so these do not exist until now, and they stay valid for the |
| 145 | + // whole suite because nothing restarts a container. |
| 146 | + host := container.GetHost() |
| 147 | + port := uint16(container.GetPort(suite.portName)) |
| 148 | + if port == 0 { |
| 149 | + t.Fatalf("the testbed exposed no %q port", suite.portName) |
| 150 | + } |
| 151 | + |
| 152 | + tck.Run(t, tck.Config{ |
| 153 | + Name: suite.name, |
| 154 | + Control: control, |
| 155 | + |
| 156 | + NewProvider: func(context.Context) (openfeature.FeatureProvider, error) { |
| 157 | + provider, err := flagd.NewProvider( |
| 158 | + suite.resolver, |
| 159 | + flagd.WithHost(host), |
| 160 | + flagd.WithPort(port), |
| 161 | + flagd.WithDeadline(1000), |
| 162 | + flagd.WithRetryGracePeriod(suite.gracePeriod), |
| 163 | + flagd.WithRetryBackoffMs(500), |
| 164 | + ) |
| 165 | + if err != nil { |
| 166 | + return nil, err |
| 167 | + } |
| 168 | + return provider, nil |
| 169 | + }, |
| 170 | + |
| 171 | + // Pointed at a closed port on localhost, never at the backend under |
| 172 | + // test — that has to stay up, and simulated outages belong to the |
| 173 | + // control API. The deadlines are deliberately short: the scenario |
| 174 | + // asserts that failure is reported promptly, so a provider that took |
| 175 | + // 30 seconds to give up would pass a test about eventual failure and |
| 176 | + // fail the one that matters. |
| 177 | + NewUnavailableProvider: func(context.Context) (openfeature.FeatureProvider, error) { |
| 178 | + provider, err := flagd.NewProvider( |
| 179 | + suite.resolver, |
| 180 | + flagd.WithHost("localhost"), |
| 181 | + flagd.WithPort(9999), |
| 182 | + flagd.WithDeadline(500), |
| 183 | + flagd.WithRetryGracePeriod(1), |
| 184 | + flagd.WithRetryBackoffMs(100), |
| 185 | + ) |
| 186 | + if err != nil { |
| 187 | + return nil, err |
| 188 | + } |
| 189 | + return provider, nil |
| 190 | + }, |
| 191 | + |
| 192 | + Capabilities: suite.capabilities, |
| 193 | + ReadyTimeout: suite.readyTimeout, |
| 194 | + EventTimeout: 15 * time.Second, |
| 195 | + }) |
| 196 | +} |
0 commit comments