internal/moduletest: Experimental module testing helpers

As part of ongoing research into Terraform testing we'd like to use an
experimental feature to validate our current understanding that expressing
tests as part of the Terraform language, as opposed to in some other
language run alongside, is a good and viable way to write practical
module integration tests.

This initial experimental incarnation of that idea is implemented as a
provider, just because that's an easier extension point for research
purposes than a first-class language feature would be. Whether this would
ultimately emerge as a provider similar to this or as custom language
constructs will be a matter for future research, if this first
experiment confirms that tests written in the Terraform language are the
best direction to take.

The previous incarnation of this experiment was an externally-developed
provider apparentlymart/testing, listed on the Terraform Registry. That
helped with showing that there are some useful tests that we can write
in the Terraform language, but integrating such a provider into Terraform
will allow us to make use of it in the also-experimental "terraform test"
command, which will follow in subsequent commits, to see how this might
fit into a development workflow.

go.mod

@@ -114,6 +114,7 @@ require (
 	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
 	github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557
 	github.com/zclconf/go-cty v1.7.1
+	github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b
 	github.com/zclconf/go-cty-yaml v1.0.2
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect

go.sum

@@ -617,6 +617,8 @@ github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLE
 github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
 github.com/zclconf/go-cty v1.7.1 h1:AvsC01GMhMLFL8CgEYdHGM+yLnnDOwhPAYcgTkeF0Gw=
 github.com/zclconf/go-cty v1.7.1/go.mod h1:VDR4+I79ubFBGm1uJac1226K5yANQFHeauxPBoP54+o=
+github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI=
+github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0=
 github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

internal/moduletest/assertion.go

+package moduletest
+
+import (
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+// Assertion is the description of a single test assertion, whether
+// successful or unsuccessful.
+type Assertion struct {
+	Outcome Status
+
+	// Description is a user-provided, human-readable description of what
+	// this assertion represents.
+	Description string
+
+	// Message is typically relevant only for TestFailed or TestError
+	// assertions, giving a human-readable description of the problem,
+	// formatted in the way our format package expects to receive paragraphs
+	// for terminal word wrapping.
+	Message string
+
+	// Diagnostics includes diagnostics specific to the current test assertion,
+	// if available.
+	Diagnostics tfdiags.Diagnostics
+}
+
+// Component represents a component being tested, each of which can have
+// several associated test assertions.
+type Component struct {
+	Assertions map[string]*Assertion
+}
+
+// Status is an enumeration of possible outcomes of a test assertion.
+type Status rune
+
+const (
+	// Pending indicates that the test was registered (during planning)
+	// but didn't register an outcome during apply, perhaps due to being
+	// blocked by some other upstream failure.
+	Pending Status = '?'
+
+	// Passed indicates that the test condition succeeded.
+	Passed Status = 'P'
+
+	// Failed indicates that the test condition was valid but did not
+	// succeed.
+	Failed Status = 'F'
+
+	// Error indicates that the test condition was invalid or that the
+	// test report failed in some other way.
+	Error Status = 'E'
+)

internal/moduletest/doc.go

+// Package moduletest contains the support code for some experimental features
+// we're using to evaluate strategies for having an opinionated approach to
+// testing of Terraform modules.
+//
+// At the moment nothing in this module is considered stable, so any features
+// that are usable by end-users ought to emit experiment warnings saying that
+// everything is subject to change even in patch releases.
+package moduletest

internal/moduletest/provider_test.go

+package moduletest
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/terraform/providers"
+	"github.com/zclconf/go-cty-debug/ctydebug"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestProvider(t *testing.T) {
+
+	assertionConfig := cty.ObjectVal(map[string]cty.Value{
+		"component": cty.StringVal("spline_reticulator"),
+		"equal": cty.MapVal(map[string]cty.Value{
+			"match": cty.ObjectVal(map[string]cty.Value{
+				"description": cty.StringVal("this should match"),
+				"got":         cty.StringVal("a"),
+				"want":        cty.StringVal("a"),
+			}),
+			"unmatch": cty.ObjectVal(map[string]cty.Value{
+				"description": cty.StringVal("this should not match"),
+				"got":         cty.StringVal("a"),
+				"want":        cty.StringVal("b"),
+			}),
+		}),
+		"check": cty.MapVal(map[string]cty.Value{
+			"pass": cty.ObjectVal(map[string]cty.Value{
+				"description": cty.StringVal("this should pass"),
+				"condition":   cty.True,
+			}),
+			"fail": cty.ObjectVal(map[string]cty.Value{
+				"description": cty.StringVal("this should fail"),
+				"condition":   cty.False,
+			}),
+		}),
+	})
+
+	// The provider code expects to receive an object that was decoded from
+	// HCL using the schema, so to make sure we're testing a more realistic
+	// situation here we'll require the config to conform to the schema. If
+	// this fails, it's a bug in the configuration definition above rather
+	// than in the provider itself.
+	for _, err := range assertionConfig.Type().TestConformance(testAssertionsSchema.Block.ImpliedType()) {
+		t.Error(err)
+	}
+
+	p := NewProvider()
+
+	configureResp := p.Configure(providers.ConfigureRequest{
+		Config: cty.EmptyObjectVal,
+	})
+	if got, want := len(configureResp.Diagnostics), 1; got != want {
+		t.Fatalf("got %d Configure diagnostics, but want %d", got, want)
+	}
+	if got, want := configureResp.Diagnostics[0].Description().Summary, "The test provider is experimental"; got != want {
+		t.Fatalf("wrong diagnostic message\ngot:  %s\nwant: %s", got, want)
+	}
+
+	validateResp := p.ValidateResourceTypeConfig(providers.ValidateResourceTypeConfigRequest{
+		TypeName: "test_assertions",
+		Config:   assertionConfig,
+	})
+	if got, want := len(validateResp.Diagnostics), 0; got != want {
+		t.Fatalf("got %d ValidateResourceTypeConfig diagnostics, but want %d", got, want)
+	}
+
+	planResp := p.PlanResourceChange(providers.PlanResourceChangeRequest{
+		TypeName:         "test_assertions",
+		Config:           assertionConfig,
+		PriorState:       cty.NullVal(assertionConfig.Type()),
+		ProposedNewState: assertionConfig,
+	})
+	if got, want := len(planResp.Diagnostics), 0; got != want {
+		t.Fatalf("got %d PlanResourceChange diagnostics, but want %d", got, want)
+	}
+	planned := planResp.PlannedState
+	if got, want := planned, assertionConfig; !want.RawEquals(got) {
+		t.Fatalf("wrong planned new value\n%s", ctydebug.DiffValues(want, got))
+	}
+
+	gotComponents := p.TestResults()
+	wantComponents := map[string]*Component{
+		"spline_reticulator": {
+			Assertions: map[string]*Assertion{
+				"pass": {
+					Outcome:     Pending,
+					Description: "this should pass",
+				},
+				"fail": {
+					Outcome:     Pending,
+					Description: "this should fail",
+				},
+				"match": {
+					Outcome:     Pending,
+					Description: "this should match",
+				},
+				"unmatch": {
+					Outcome:     Pending,
+					Description: "this should not match",
+				},
+			},
+		},
+	}
+	if diff := cmp.Diff(wantComponents, gotComponents); diff != "" {
+		t.Fatalf("wrong test results after planning\n%s", diff)
+	}
+
+	applyResp := p.ApplyResourceChange(providers.ApplyResourceChangeRequest{
+		TypeName:     "test_assertions",
+		Config:       assertionConfig,
+		PriorState:   cty.NullVal(assertionConfig.Type()),
+		PlannedState: planned,
+	})
+	if got, want := len(applyResp.Diagnostics), 0; got != want {
+		t.Fatalf("got %d ApplyResourceChange diagnostics, but want %d", got, want)
+	}
+	final := applyResp.NewState
+	if got, want := final, assertionConfig; !want.RawEquals(got) {
+		t.Fatalf("wrong new value\n%s", ctydebug.DiffValues(want, got))
+	}
+
+	gotComponents = p.TestResults()
+	wantComponents = map[string]*Component{
+		"spline_reticulator": {
+			Assertions: map[string]*Assertion{
+				"pass": {
+					Outcome:     Passed,
+					Description: "this should pass",
+					Message:     "condition passed",
+				},
+				"fail": {
+					Outcome:     Failed,
+					Description: "this should fail",
+					Message:     "condition failed",
+				},
+				"match": {
+					Outcome:     Passed,
+					Description: "this should match",
+					Message:     "correct value\n    got: \"a\"\n",
+				},
+				"unmatch": {
+					Outcome:     Failed,
+					Description: "this should not match",
+					Message:     "wrong value\n    got:  \"a\"\n    want: \"b\"\n",
+				},
+			},
+		},
+	}
+	if diff := cmp.Diff(wantComponents, gotComponents); diff != "" {
+		t.Fatalf("wrong test results after applying\n%s", diff)
+	}
+
+}