PSS: Update how commands access backends, so both backend and state_store configuration can be used

internal/command/apply.go

@@ -219,19 +219,18 @@ func (c *ApplyCommand) PrepareBackend(planFile *planfile.WrappedPlanFile, args *
 			))
 			return nil, diags
 		}
+		// TODO: Update BackendForLocalPlan to use state storage, and plan to be able to contain State Store config details
 		be, beDiags = c.BackendForLocalPlan(plan.Backend)
 	} else {
 		// Both new plans and saved cloud plans load their backend from config.
-		backendConfig, configDiags := c.loadBackendConfig(".")
-		diags = diags.Append(configDiags)
-		if configDiags.HasErrors() {
+		mod, mDiags := c.Meta.loadSingleModule(".")
+		if mDiags.HasErrors() {
+			diags = diags.Append(mDiags)
 			return nil, diags
 		}
 
-		be, beDiags = c.Backend(&BackendOpts{
-			BackendConfig: backendConfig,
-			ViewType:      viewType,
-		})
+		// Load the backend
+		be, beDiags = c.prepareBackend(mod)
 	}
 
 	diags = diags.Append(beDiags)

internal/command/console.go

@@ -53,17 +53,14 @@ func (c *ConsoleCommand) Run(args []string) int {
 
 	var diags tfdiags.Diagnostics
 
-	backendConfig, backendDiags := c.loadBackendConfig(configPath)
-	diags = diags.Append(backendDiags)
+	mod, diags := c.Meta.loadSingleModule(configPath)
 	if diags.HasErrors() {
 		c.showDiagnostics(diags)
 		return 1
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-	})
+	b, backendDiags := c.prepareBackend(mod)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/graph.go

@@ -68,17 +68,14 @@ func (c *GraphCommand) Run(args []string) int {
 
 	var diags tfdiags.Diagnostics
 
-	backendConfig, backendDiags := c.loadBackendConfig(configPath)
-	diags = diags.Append(backendDiags)
+	mod, diags := c.Meta.loadSingleModule(configPath)
 	if diags.HasErrors() {
 		c.showDiagnostics(diags)
 		return 1
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-	})
+	b, backendDiags := c.prepareBackend(mod)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/import.go

@@ -164,9 +164,7 @@ func (c *ImportCommand) Run(args []string) int {
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: config.Module.Backend,
-	})
+	b, backendDiags := c.prepareBackend(config.Root.Module)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/meta_backend.go

@@ -1580,6 +1580,63 @@ func (m *Meta) updateSavedBackendHash(cHash int, sMgr *clistate.LocalState) tfdi
 	return diags
 }
 
+// prepareBackend returns an operations backend that may use a backend, cloud, or state_store block for state storage.
+// This method should be used in NON-init operations only; it's incapable of processing new init command CLI flags used
+// for partial configuration, however it will use the backend state file to use partial configuration from a previous
+// init command.
+func (m *Meta) prepareBackend(root *configs.Module) (backendrun.OperationsBackend, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	var opts *BackendOpts
+	switch {
+	case root.Backend != nil:
+		opts = &BackendOpts{
+			BackendConfig: root.Backend,
+		}
+	case root.CloudConfig != nil:
+		backendConfig := root.CloudConfig.ToBackendConfig()
+		opts = &BackendOpts{
+			BackendConfig: &backendConfig,
+		}
+	case root.StateStore != nil:
+		// In addition to config, use of a state_store requires
+		// provider factory and provider locks data
+		locks, lDiags := m.lockedDependencies()
+		diags = diags.Append(lDiags)
+		if lDiags.HasErrors() {
+			return nil, diags
+		}
+
+		factory, fDiags := m.GetStateStoreProviderFactory(root.StateStore, locks)
+		diags = diags.Append(fDiags)
+		if fDiags.HasErrors() {
+			return nil, diags
+		}
+
+		opts = &BackendOpts{
+			StateStoreConfig: root.StateStore,
+			ProviderFactory:  factory,
+			Locks:            locks,
+		}
+	default:
+		// there is no config; defaults to local state storage
+		opts = &BackendOpts{}
+	}
+
+	// This method should not be used for init commands,
+	// so we always set this value as false.
+	opts.Init = false
+
+	// Load the backend
+	be, beDiags := m.Backend(opts)
+	diags = diags.Append(beDiags)
+	if beDiags.HasErrors() {
+		return nil, diags
+	}
+
+	return be, diags
+}
+
 //-------------------------------------------------------------------
 // State Store Config Scenarios
 // The functions below cover handling all the various scenarios that

internal/command/meta_backend_test.go

@@ -2962,6 +2962,142 @@ func Test_getStateStorageProviderVersion(t *testing.T) {
 	})
 }
 
+func TestMetaBackend_prepareBackend(t *testing.T) {
+
+	t.Run("it returns a cloud backend from cloud backend config", func(t *testing.T) {
+		// Create a temporary working directory with cloud configuration in
+		td := t.TempDir()
+		testCopyDir(t, testFixturePath("cloud-config"), td)
+		t.Chdir(td)
+
+		m := testMetaBackend(t, nil)
+
+		// Get the cloud config
+		mod, loadDiags := m.loadSingleModule(td)
+		if loadDiags.HasErrors() {
+			t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err())
+		}
+
+		// We cannot initialize a cloud backend so we instead check
+		// the init error is referencing HCP Terraform
+		_, bDiags := m.prepareBackend(mod)
+		if !bDiags.HasErrors() {
+			t.Fatal("expected error but got none")
+		}
+		wantErr := "HCP Terraform or Terraform Enterprise initialization required: please run \"terraform init\""
+		if !strings.Contains(bDiags.Err().Error(), wantErr) {
+			t.Fatalf("expected error to contain %q, but got: %q",
+				wantErr,
+				bDiags.Err())
+		}
+	})
+
+	t.Run("it returns a backend from backend config", func(t *testing.T) {
+		// Create a temporary working directory with backend configuration in
+		td := t.TempDir()
+		testCopyDir(t, testFixturePath("backend-unchanged"), td)
+		t.Chdir(td)
+
+		m := testMetaBackend(t, nil)
+
+		// Get the backend config
+		mod, loadDiags := m.loadSingleModule(td)
+		if loadDiags.HasErrors() {
+			t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err())
+		}
+
+		b, bDiags := m.prepareBackend(mod)
+		if bDiags.HasErrors() {
+			t.Fatal("unexpected error: ", bDiags.Err())
+		}
+
+		if _, ok := b.(*local.Local); !ok {
+			t.Fatal("expected returned operations backend to be a Local backend")
+		}
+		// Check the type of backend inside the Local via schema
+		// In this case a `local` backend should have been returned by default.
+		//
+		// Look for the path attribute.
+		schema := b.ConfigSchema()
+		if _, ok := schema.Attributes["path"]; !ok {
+			t.Fatalf("expected the operations backend to report the schema of a local backend, but got something unexpected: %#v", schema)
+		}
+	})
+
+	t.Run("it returns a local backend when there is empty configuration", func(t *testing.T) {
+		m := testMetaBackend(t, nil)
+		emptyConfig := configs.NewEmptyConfig()
+
+		b, bDiags := m.prepareBackend(emptyConfig.Module)
+		if bDiags.HasErrors() {
+			t.Fatal("unexpected error: ", bDiags.Err())
+		}
+
+		if _, ok := b.(*local.Local); !ok {
+			t.Fatal("expected returned operations backend to be a Local backend")
+		}
+		// Check the type of backend inside the Local via schema
+		// In this case a `local` backend should have been returned by default.
+		//
+		// Look for the path attribute.
+		schema := b.ConfigSchema()
+		if _, ok := schema.Attributes["path"]; !ok {
+			t.Fatalf("expected the operations backend to report the schema of a local backend, but got something unexpected: %#v", schema)
+		}
+	})
+
+	t.Run("it returns a state_store from state_store config", func(t *testing.T) {
+		// Create a temporary working directory with backend configuration in
+		td := t.TempDir()
+		testCopyDir(t, testFixturePath("state-store-unchanged"), td)
+		t.Chdir(td)
+
+		m := testMetaBackend(t, nil)
+		m.AllowExperimentalFeatures = true
+		mock := testStateStoreMockWithChunkNegotiation(t, 12345) // chunk size needs to be set, value is arbitrary
+		m.testingOverrides = &testingOverrides{
+			Providers: map[addrs.Provider]providers.Factory{
+				addrs.NewDefaultProvider("test"): providers.FactoryFixed(mock),
+			},
+		}
+
+		// Get the backend config
+		mod, loadDiags := m.loadSingleModule(td)
+		if loadDiags.HasErrors() {
+			t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err())
+		}
+
+		// Prepare appropriate locks; config uses a hashicorp/test provider @ v1.2.3
+		locks := depsfile.NewLocks()
+		providerAddr := addrs.MustParseProviderSourceString("registry.terraform.io/hashicorp/test")
+		constraint, err := providerreqs.ParseVersionConstraints(">1.0.0")
+		if err != nil {
+			t.Fatalf("test setup failed when making constraint: %s", err)
+		}
+		locks.SetProvider(
+			providerAddr,
+			versions.MustParseVersion("1.2.3"),
+			constraint,
+			[]providerreqs.Hash{""},
+		)
+
+		b, bDiags := m.prepareBackend(mod)
+		if bDiags.HasErrors() {
+			t.Fatalf("unexpected error: %s", bDiags.Err())
+		}
+
+		if _, ok := b.(*local.Local); !ok {
+			t.Fatal("expected returned operations backend to be a Local backend")
+		}
+		// Check the state_store inside the Local via schema
+		// Look for the mock state_store's attribute called `value`.
+		schema := b.ConfigSchema()
+		if _, ok := schema.Attributes["value"]; !ok {
+			t.Fatalf("expected the operations backend to report the schema of the state_store, but got something unexpected: %#v", schema)
+		}
+	})
+}
+
 func testMetaBackend(t *testing.T, args []string) *Meta {
 	var m Meta
 	m.Ui = new(cli.MockUi)
@@ -3011,6 +3147,21 @@ func testStateStoreMock(t *testing.T) *testing_provider.MockProvider {
 	}
 }
 
+// testStateStoreMockWithChunkNegotiation is just like testStateStoreMock but the returned mock is set up so it'll be configured
+// without this error: `Failed to negotiate acceptable chunk size`
+//
+// This is meant to be a convenience method when a test is definitely not testing anything related to state store configuration.
+func testStateStoreMockWithChunkNegotiation(t *testing.T, chunkSize int64) *testing_provider.MockProvider {
+	t.Helper()
+	mock := testStateStoreMock(t)
+	mock.ConfigureStateStoreResponse = &providers.ConfigureStateStoreResponse{
+		Capabilities: providers.StateStoreServerCapabilities{
+			ChunkSize: chunkSize,
+		},
+	}
+	return mock
+}
+
 func configBodyForTest(t *testing.T, config string) hcl.Body {
 	t.Helper()
 	f, diags := hclsyntax.ParseConfig([]byte(config), "", hcl.Pos{Line: 1, Column: 1})

internal/command/output.go

@@ -62,10 +62,15 @@ func (c *OutputCommand) Outputs(statePath string) (map[string]*states.OutputValu
 		c.Meta.statePath = statePath
 	}
 
+	mod, diags := c.Meta.loadSingleModule(".")
+	if diags.HasErrors() {
+		return nil, diags
+	}
+
 	// Load the backend
-	b, backendDiags := c.Backend(nil)
+	b, backendDiags := c.prepareBackend(mod)
 	diags = diags.Append(backendDiags)
-	if diags.HasErrors() {
+	if backendDiags.HasErrors() {
 		return nil, diags
 	}
 

internal/command/plan.go

@@ -124,16 +124,13 @@ func (c *PlanCommand) PrepareBackend(args *arguments.State, viewType arguments.V
 	// difficult but would make their use easier to understand.
 	c.Meta.applyStateArguments(args)
 
-	backendConfig, diags := c.loadBackendConfig(".")
+	mod, diags := c.Meta.loadSingleModule(".")
 	if diags.HasErrors() {
 		return nil, diags
 	}
 
 	// Load the backend
-	be, beDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-		ViewType:      viewType,
-	})
+	be, beDiags := c.prepareBackend(mod)
 	diags = diags.Append(beDiags)
 	if beDiags.HasErrors() {
 		return nil, diags

internal/command/providers.go

@@ -81,9 +81,7 @@ func (c *ProvidersCommand) Run(args []string) int {
 	}
 
 	// Load the backend
-	b, backendDiags := c.Backend(&BackendOpts{
-		BackendConfig: config.Module.Backend,
-	})
+	b, backendDiags := c.prepareBackend(config.Root.Module)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/providers_schema.go

@@ -55,8 +55,14 @@ func (c *ProvidersSchemaCommand) Run(args []string) int {
 
 	var diags tfdiags.Diagnostics
 
+	mod, diags := c.Meta.loadSingleModule(".")
+	if diags.HasErrors() {
+		c.showDiagnostics(diags)
+		return 1
+	}
+
 	// Load the backend
-	b, backendDiags := c.Backend(nil)
+	b, backendDiags := c.prepareBackend(mod)
 	diags = diags.Append(backendDiags)
 	if backendDiags.HasErrors() {
 		c.showDiagnostics(diags)

internal/command/query.go

@@ -154,16 +154,17 @@ func (c *QueryCommand) Run(rawArgs []string) int {
 }
 
 func (c *QueryCommand) PrepareBackend(args *arguments.State, viewType arguments.ViewType) (backendrun.OperationsBackend, tfdiags.Diagnostics) {
-	backendConfig, diags := c.loadBackendConfig(".")
+	mod, diags := c.Meta.loadSingleModule(".")
 	if diags.HasErrors() {
 		return nil, diags
 	}
 
 	// Load the backend
-	be, beDiags := c.Backend(&BackendOpts{
-		BackendConfig: backendConfig,
-		ViewType:      viewType,
-	})
+	be, beDiags := c.prepareBackend(mod)
+	diags = diags.Append(beDiags)
+	if beDiags.HasErrors() {
+		return nil, diags
+	}
 	diags = diags.Append(beDiags)
 	if beDiags.HasErrors() {
 		return nil, diags