Framework and API changes to support OpenAPI (#5546)

logical/framework/backend.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/hashicorp/errwrap"
 	log "github.com/hashicorp/go-hclog"
-
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/vault/helper/errutil"
 	"github.com/hashicorp/vault/helper/license"
@@ -202,15 +201,22 @@ func (b *Backend) HandleRequest(ctx context.Context, req *logical.Request) (*log
 		raw[k] = v
 	}
 
-	// Look up the callback for this operation
+	// Look up the callback for this operation, preferring the
+	// path.Operations definition if present.
 	var callback OperationFunc
-	var ok bool
-	if path.Callbacks != nil {
-		callback, ok = path.Callbacks[req.Operation]
+
+	if path.Operations != nil {
+		if op, ok := path.Operations[req.Operation]; ok {
+			callback = op.Handler()
+		}
+	} else {
+		callback = path.Callbacks[req.Operation]
 	}
+	ok := callback != nil
+
 	if !ok {
 		if req.Operation == logical.HelpOperation {
-			callback = path.helpCallback()
+			callback = path.helpCallback(b)
 			ok = true
 		}
 	}
@@ -229,7 +235,6 @@ func (b *Backend) HandleRequest(ctx context.Context, req *logical.Request) (*log
 		}
 	}
 
-	// Call the callback with the request and the data
 	return callback(ctx, req, &fd)
 }
 
@@ -370,7 +375,13 @@ func (b *Backend) handleRootHelp() (*logical.Response, error) {
 		return nil, err
 	}
 
-	return logical.HelpResponse(help, nil), nil
+	// Build OpenAPI response for the entire backend
+	doc := NewOASDocument()
+	if err := documentPaths(b, doc); err != nil {
+		b.Logger().Warn("error generating OpenAPI", "error", err)
+	}
+
+	return logical.HelpResponse(help, nil, doc), nil
 }
 
 func (b *Backend) handleRevokeRenew(ctx context.Context, req *logical.Request) (*logical.Response, error) {
@@ -492,6 +503,8 @@ type FieldSchema struct {
 	Type        FieldType
 	Default     interface{}
 	Description string
+	Required    bool
+	Deprecated  bool
 }
 
 // DefaultOrZero returns the default value if it is set, or otherwise

logical/framework/backend_test.go

@@ -2,13 +2,13 @@ package framework
 
 import (
 	"context"
+	"net/http"
 	"reflect"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	"net/http"
-
 	"github.com/hashicorp/vault/logical"
 )
 
@@ -52,10 +52,17 @@ func TestBackendHandleRequest(t *testing.T) {
 			},
 		}, nil
 	}
+	handler := func(ctx context.Context, req *logical.Request, data *FieldData) (*logical.Response, error) {
+		return &logical.Response{
+			Data: map[string]interface{}{
+				"amount": data.Get("amount"),
+			},
+		}, nil
+	}
 
 	b := &Backend{
 		Paths: []*Path{
-			&Path{
+			{
 				Pattern: "foo/bar",
 				Fields: map[string]*FieldSchema{
 					"value": &FieldSchema{Type: TypeInt},
@@ -64,19 +71,46 @@ func TestBackendHandleRequest(t *testing.T) {
 					logical.ReadOperation: callback,
 				},
 			},
+			{
+				Pattern: "foo/baz/handler",
+				Fields: map[string]*FieldSchema{
+					"amount": &FieldSchema{Type: TypeInt},
+				},
+				Operations: map[logical.Operation]OperationHandler{
+					logical.ReadOperation: &PathOperation{Callback: handler},
+				},
+			},
+			{
+				Pattern: "foo/both/handler",
+				Fields: map[string]*FieldSchema{
+					"amount": &FieldSchema{Type: TypeInt},
+				},
+				Callbacks: map[logical.Operation]OperationFunc{
+					logical.ReadOperation: callback,
+				},
+				Operations: map[logical.Operation]OperationHandler{
+					logical.ReadOperation: &PathOperation{Callback: handler},
+				},
+			},
 		},
 	}
 
-	resp, err := b.HandleRequest(context.Background(), &logical.Request{
-		Operation: logical.ReadOperation,
-		Path:      "foo/bar",
-		Data:      map[string]interface{}{"value": "42"},
-	})
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if resp.Data["value"] != 42 {
-		t.Fatalf("bad: %#v", resp)
+	for _, path := range []string{"foo/bar", "foo/baz/handler", "foo/both/handler"} {
+		key := "value"
+		if strings.Contains(path, "handler") {
+			key = "amount"
+		}
+		resp, err := b.HandleRequest(context.Background(), &logical.Request{
+			Operation: logical.ReadOperation,
+			Path:      path,
+			Data:      map[string]interface{}{key: "42"},
+		})
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		if resp.Data[key] != 42 {
+			t.Fatalf("bad: %#v", resp)
+		}
 	}
 }
 

logical/framework/openapi.go

+package framework
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/wrapping"
+	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/version"
+	"github.com/mitchellh/mapstructure"
+)
+
+// OpenAPI specification (OAS): https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md
+const OASVersion = "3.0.2"
+
+// NewOASDocument returns an empty OpenAPI document.
+func NewOASDocument() *OASDocument {
+	return &OASDocument{
+		Version: OASVersion,
+		Info: OASInfo{
+			Title:       "HashiCorp Vault API",
+			Description: "HTTP API that gives you full access to Vault. All API routes are prefixed with `/v1/`.",
+			Version:     version.GetVersion().Version,
+			License: OASLicense{
+				Name: "Mozilla Public License 2.0",
+				URL:  "https://www.mozilla.org/en-US/MPL/2.0",
+			},
+		},
+		Paths: make(map[string]*OASPathItem),
+	}
+}
+
+// NewOASDocumentFromMap builds an OASDocument from an existing map version of a document.
+// If a document has been decoded from JSON or received from a plugin, it will be as a map[string]interface{}
+// and needs special handling beyond the default mapstructure decoding.
+func NewOASDocumentFromMap(input map[string]interface{}) (*OASDocument, error) {
+
+	// The Responses map uses integer keys (the response code), but once translated into JSON
+	// (e.g. during the plugin transport) these become strings. mapstructure will not coerce these back
+	// to integers without a custom decode hook.
+	decodeHook := func(src reflect.Type, tgt reflect.Type, inputRaw interface{}) (interface{}, error) {
+
+		// Only alter data if:
+		//  1. going from string to int
+		//  2. string represent an int in status code range (100-599)
+		if src.Kind() == reflect.String && tgt.Kind() == reflect.Int {
+			if input, ok := inputRaw.(string); ok {
+				if intval, err := strconv.Atoi(input); err == nil {
+					if intval >= 100 && intval < 600 {
+						return intval, nil
+					}
+				}
+			}
+		}
+		return inputRaw, nil
+	}
+
+	doc := new(OASDocument)
+
+	config := &mapstructure.DecoderConfig{
+		DecodeHook: decodeHook,
+		Result:     doc,
+	}
+
+	decoder, err := mapstructure.NewDecoder(config)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := decoder.Decode(input); err != nil {
+		return nil, err
+	}
+
+	return doc, nil
+}
+
+type OASDocument struct {
+	Version string                  `json:"openapi" mapstructure:"openapi"`
+	Info    OASInfo                 `json:"info"`
+	Paths   map[string]*OASPathItem `json:"paths"`
+}
+
+type OASInfo struct {
+	Title       string     `json:"title"`
+	Description string     `json:"description"`
+	Version     string     `json:"version"`
+	License     OASLicense `json:"license"`
+}
+
+type OASLicense struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type OASPathItem struct {
+	Description     string         `json:"description,omitempty"`
+	Parameters      []OASParameter `json:"parameters,omitempty"`
+	Sudo            bool           `json:"x-vault-sudo,omitempty" mapstructure:"x-vault-sudo"`
+	Unauthenticated bool           `json:"x-vault-unauthenticated,omitempty" mapstructure:"x-vault-unauthenticated"`
+	CreateSupported bool           `json:"x-vault-create-supported,omitempty" mapstructure:"x-vault-create-supported"`
+
+	Get    *OASOperation `json:"get,omitempty"`
+	Post   *OASOperation `json:"post,omitempty"`
+	Delete *OASOperation `json:"delete,omitempty"`
+}
+
+// NewOASOperation creates an empty OpenAPI Operations object.
+func NewOASOperation() *OASOperation {
+	return &OASOperation{
+		Responses: make(map[int]*OASResponse),
+	}
+}
+
+type OASOperation struct {
+	Summary     string               `json:"summary,omitempty"`
+	Description string               `json:"description,omitempty"`
+	Tags        []string             `json:"tags,omitempty"`
+	Parameters  []OASParameter       `json:"parameters,omitempty"`
+	RequestBody *OASRequestBody      `json:"requestBody,omitempty"`
+	Responses   map[int]*OASResponse `json:"responses"`
+	Deprecated  bool                 `json:"deprecated,omitempty"`
+}
+
+type OASParameter struct {
+	Name        string     `json:"name"`
+	Description string     `json:"description,omitempty"`
+	In          string     `json:"in"`
+	Schema      *OASSchema `json:"schema,omitempty"`
+	Required    bool       `json:"required,omitempty"`
+	Deprecated  bool       `json:"deprecated,omitempty"`
+}
+
+type OASRequestBody struct {
+	Description string     `json:"description,omitempty"`
+	Content     OASContent `json:"content,omitempty"`
+}
+
+type OASContent map[string]*OASMediaTypeObject
+
+type OASMediaTypeObject struct {
+	Schema *OASSchema `json:"schema,omitempty"`
+}
+
+type OASSchema struct {
+	Type        string                `json:"type,omitempty"`
+	Description string                `json:"description,omitempty"`
+	Properties  map[string]*OASSchema `json:"properties,omitempty"`
+	Items       *OASSchema            `json:"items,omitempty"`
+	Format      string                `json:"format,omitempty"`
+	Pattern     string                `json:"pattern,omitempty"`
+	Example     interface{}           `json:"example,omitempty"`
+	Deprecated  bool                  `json:"deprecated,omitempty"`
+}
+
+type OASResponse struct {
+	Description string     `json:"description"`
+	Content     OASContent `json:"content,omitempty"`
+}
+
+var OASStdRespOK = &OASResponse{
+	Description: "OK",
+}
+
+var OASStdRespNoContent = &OASResponse{
+	Description: "empty body",
+}
+
+// Regex for handling optional and named parameters in paths, and string cleanup.
+// Predefined here to avoid substantial recompilation.
+
+// Capture optional path elements in ungreedy (?U) fashion
+// Both "(leases/)?renew" and "(/(?P<name>.+))?" formats are detected
+var optRe = regexp.MustCompile(`(?U)\([^(]*\)\?|\(/\(\?P<[^(]*\)\)\?`)
+
+var reqdRe = regexp.MustCompile(`\(?\?P<(\w+)>[^)]*\)?`) // Capture required parameters, e.g. "(?P<name>regex)"
+var altRe = regexp.MustCompile(`\((.*)\|(.*)\)`)         // Capture alternation elements, e.g. "(raw/?$|raw/(?P<path>.+))"
+var pathFieldsRe = regexp.MustCompile(`{(\w+)}`)         // Capture OpenAPI-style named parameters, e.g. "lookup/{urltoken}",
+var cleanCharsRe = regexp.MustCompile("[()^$?]")         // Set of regex characters that will be stripped during cleaning
+var cleanSuffixRe = regexp.MustCompile(`/\?\$?$`)        // Path suffix patterns that will be stripped during cleaning
+var wsRe = regexp.MustCompile(`\s+`)                     // Match whitespace, to be compressed during cleaning
+
+// documentPaths parses all paths in a framework.Backend into OpenAPI paths.
+func documentPaths(backend *Backend, doc *OASDocument) error {
+	for _, p := range backend.Paths {
+		if err := documentPath(p, backend.SpecialPaths(), backend.BackendType, doc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// documentPath parses a framework.Path into one or more OpenAPI paths.
+func documentPath(p *Path, specialPaths *logical.Paths, backendType logical.BackendType, doc *OASDocument) error {
+	var sudoPaths []string
+	var unauthPaths []string
+
+	if specialPaths != nil {
+		sudoPaths = specialPaths.Root
+		unauthPaths = specialPaths.Unauthenticated
+	}
+
+	// Convert optional parameters into distinct patterns to be process independently.
+	paths := expandPattern(p.Pattern)
+
+	for _, path := range paths {
+		// Construct a top level PathItem which will be populated as the path is processed.
+		pi := OASPathItem{
+			Description: cleanString(p.HelpSynopsis),
+		}
+
+		pi.Sudo = specialPathMatch(path, sudoPaths)
+		pi.Unauthenticated = specialPathMatch(path, unauthPaths)
+
+		// If the newer style Operations map isn't defined, create one from the legacy fields.
+		operations := p.Operations
+		if operations == nil {
+			operations = make(map[logical.Operation]OperationHandler)
+
+			for opType, cb := range p.Callbacks {
+				operations[opType] = &PathOperation{
+					Callback: cb,
+					Summary:  p.HelpSynopsis,
+				}
+			}
+		}
+
+		// Process path and header parameters, which are common to all operations.
+		// Body fields will be added to individual operations.
+		pathFields, bodyFields := splitFields(p.Fields, path)
+
+		for name, field := range pathFields {
+			location := "path"
+			required := true
+
+			// Header parameters are part of the Parameters group but with
+			// a dedicated "header" location, a header parameter is not required.
+			if field.Type == TypeHeader {
+				location = "header"
+				required = false
+			}
+
+			t := convertType(field.Type)
+			p := OASParameter{
+				Name:        name,
+				Description: cleanString(field.Description),
+				In:          location,
+				Schema: &OASSchema{
+					Type:    t.baseType,
+					Pattern: t.pattern,
+				},
+				Required:   required,
+				Deprecated: field.Deprecated,
+			}
+			pi.Parameters = append(pi.Parameters, p)
+		}
+
+		// Sort parameters for a stable output
+		sort.Slice(pi.Parameters, func(i, j int) bool {
+			return strings.ToLower(pi.Parameters[i].Name) < strings.ToLower(pi.Parameters[j].Name)
+		})
+
+		// Process each supported operation by building up an Operation object
+		// with descriptions, properties and examples from the framework.Path data.
+		for opType, opHandler := range operations {
+			props := opHandler.Properties()
+			if props.Unpublished {
+				continue
+			}
+
+			if opType == logical.CreateOperation {
+				pi.CreateSupported = true
+
+				// If both Create and Update are defined, only process Update.
+				if operations[logical.UpdateOperation] != nil {
+					continue
+				}
+			}
+
+			// If both List and Read are defined, only process Read.
+			if opType == logical.ListOperation && operations[logical.ReadOperation] != nil {
+				continue
+			}
+
+			op := NewOASOperation()
+
+			op.Summary = props.Summary
+			op.Description = props.Description
+			op.Deprecated = props.Deprecated
+
+			// Add any fields not present in the path as body parameters for POST.
+			if opType == logical.CreateOperation || opType == logical.UpdateOperation {
+				s := &OASSchema{
+					Type:       "object",
+					Properties: make(map[string]*OASSchema),
+				}
+
+				for name, field := range bodyFields {
+					openapiField := convertType(field.Type)
+					p := OASSchema{
+						Type:        openapiField.baseType,
+						Description: cleanString(field.Description),
+						Format:      openapiField.format,
+						Pattern:     openapiField.pattern,
+						Deprecated:  field.Deprecated,
+					}
+					if openapiField.baseType == "array" {
+						p.Items = &OASSchema{
+							Type: openapiField.items,
+						}
+					}
+					s.Properties[name] = &p
+				}
+
+				// If examples were given, use the first one as the sample
+				// of this schema.
+				if len(props.Examples) > 0 {
+					s.Example = props.Examples[0].Data
+				}
+
+				// Set the final request body. Only JSON request data is supported.
+				if len(s.Properties) > 0 || s.Example != nil {
+					op.RequestBody = &OASRequestBody{
+						Content: OASContent{
+							"application/json": &OASMediaTypeObject{
+								Schema: s,
+							},
+						},
+					}
+				}
+			}
+
+			// LIST is represented as GET with a `list` query parameter
+			if opType == logical.ListOperation || (opType == logical.ReadOperation && operations[logical.ListOperation] != nil) {
+				op.Parameters = append(op.Parameters, OASParameter{
+					Name:        "list",
+					Description: "Return a list if `true`",
+					In:          "query",
+					Schema:      &OASSchema{Type: "string"},
+				})
+			}
+
+			// Add tags based on backend type
+			var tags []string
+			switch backendType {
+			case logical.TypeLogical:
+				tags = []string{"secrets"}
+			case logical.TypeCredential:
+				tags = []string{"auth"}
+			}
+
+			op.Tags = append(op.Tags, tags...)
+
+			// Set default responses.
+			if len(props.Responses) == 0 {
+				if opType == logical.DeleteOperation {
+					op.Responses[204] = OASStdRespNoContent
+				} else {
+					op.Responses[200] = OASStdRespOK
+				}
+			}
+
+			// Add any defined response details.
+			for code, responses := range props.Responses {
+				var description string
+				content := make(OASContent)
+
+				for i, resp := range responses {
+					if i == 0 {
+						description = resp.Description
+					}
+					if resp.Example != nil {
+						mediaType := resp.MediaType
+						if mediaType == "" {
+							mediaType = "application/json"
+						}
+
+						// create a version of the response that will not emit null items
+						cr, err := cleanResponse(resp.Example)
+						if err != nil {
+							return err
+						}
+
+						// Only one example per media type is allowed, so first one wins
+						if _, ok := content[mediaType]; !ok {
+							content[mediaType] = &OASMediaTypeObject{
+								Schema: &OASSchema{
+									Example: cr,
+								},
+							}
+						}
+					}
+				}
+
+				op.Responses[code] = &OASResponse{
+					Description: description,
+					Content:     content,
+				}
+			}
+
+			switch opType {
+			case logical.CreateOperation, logical.UpdateOperation:
+				pi.Post = op
+			case logical.ReadOperation, logical.ListOperation:
+				pi.Get = op
+			case logical.DeleteOperation:
+				pi.Delete = op
+			}
+		}
+
+		doc.Paths["/"+path] = &pi
+	}
+
+	return nil
+}
+
+func specialPathMatch(path string, specialPaths []string) bool {
+	// Test for exact or prefix match of special paths.
+	for _, sp := range specialPaths {
+		if sp == path ||
+			(strings.HasSuffix(sp, "*") && strings.HasPrefix(path, sp[0:len(sp)-1])) {
+			return true
+		}
+	}
+	return false
+}
+
+// expandPattern expands a regex pattern by generating permutations of any optional parameters
+// and changing named parameters into their {openapi} equivalents.
+func expandPattern(pattern string) []string {
+	var paths []string
+
+	// GenericNameRegex adds a regex that complicates our parsing. It is much easier to
+	// detect and remove it now than to compensate for in the other regexes.
+	//
+	// example: (?P<foo>\\w(([\\w-.]+)?\\w)?) -> (?P<foo>)
+	base := GenericNameRegex("")
+	start := strings.Index(base, ">")
+	end := strings.LastIndex(base, ")")
+	regexToRemove := ""
+	if start != -1 && end != -1 && end > start {
+		regexToRemove = base[start+1 : end]
+	}
+
+	pattern = strings.Replace(pattern, regexToRemove, "", -1)
+
+	// Initialize paths with the original pattern or the halves of an
+	// alternation, which is also present in some patterns.
+	matches := altRe.FindAllStringSubmatch(pattern, -1)
+	if len(matches) > 0 {
+		paths = []string{matches[0][1], matches[0][2]}
+	} else {
+		paths = []string{pattern}
+	}
+
+	// Expand all optional regex elements into two paths. This approach is really only useful up to 2 optional
+	// groups, but we probably don't want to deal with the exponential increase beyond that anyway.
+	for i := 0; i < len(paths); i++ {
+		p := paths[i]
+
+		// match is a 2-element slice that will have a start and end index
+		// for the left-most match of a regex of form: (lease/)?
+		match := optRe.FindStringIndex(p)
+
+		if match != nil {
+			// create a path that includes the optional element but without
+			// parenthesis or the '?' character.
+			paths[i] = p[:match[0]] + p[match[0]+1:match[1]-2] + p[match[1]:]
+
+			// create a path that excludes the optional element.
+			paths = append(paths, p[:match[0]]+p[match[1]:])
+			i--
+		}
+	}
+
+	// Replace named parameters (?P<foo>) with {foo}
+	var replacedPaths []string
+
+	for _, path := range paths {
+		result := reqdRe.FindAllStringSubmatch(path, -1)
+		if result != nil {
+			for _, p := range result {
+				par := p[1]
+				path = strings.Replace(path, p[0], fmt.Sprintf("{%s}", par), 1)
+			}
+		}
+		// Final cleanup
+		path = cleanSuffixRe.ReplaceAllString(path, "")
+		path = cleanCharsRe.ReplaceAllString(path, "")
+		replacedPaths = append(replacedPaths, path)
+	}
+
+	return replacedPaths
+}
+
+// schemaType is a subset of the JSON Schema elements used as a target
+// for conversions from Vault's standard FieldTypes.
+type schemaType struct {
+	baseType string
+	items    string
+	format   string
+	pattern  string
+}
+
+// convertType translates a FieldType into an OpenAPI type.
+// In the case of arrays, a subtype is returned as well.
+func convertType(t FieldType) schemaType {
+	ret := schemaType{}
+
+	switch t {
+	case TypeString, TypeHeader:
+		ret.baseType = "string"
+	case TypeNameString:
+		ret.baseType = "string"
+		ret.pattern = `\w([\w-.]*\w)?`
+	case TypeLowerCaseString:
+		ret.baseType = "string"
+		ret.format = "lowercase"
+	case TypeInt:
+		ret.baseType = "number"
+	case TypeDurationSecond:
+		ret.baseType = "number"
+		ret.format = "seconds"
+	case TypeBool:
+		ret.baseType = "boolean"
+	case TypeMap:
+		ret.baseType = "object"
+		ret.format = "map"
+	case TypeKVPairs:
+		ret.baseType = "object"
+		ret.format = "kvpairs"
+	case TypeSlice:
+		ret.baseType = "array"
+		ret.items = "object"
+	case TypeStringSlice, TypeCommaStringSlice:
+		ret.baseType = "array"
+		ret.items = "string"
+	case TypeCommaIntSlice:
+		ret.baseType = "array"
+		ret.items = "number"
+	default:
+		log.L().Warn("error parsing field type", "type", t)
+		ret.format = "unknown"
+	}
+
+	return ret
+}
+
+// cleanString prepares s for inclusion in the output
+func cleanString(s string) string {
+	// clean leading/trailing whitespace, and replace whitespace runs into a single space
+	s = strings.TrimSpace(s)
+	s = wsRe.ReplaceAllString(s, " ")
+	return s
+}
+
+// splitFields partitions fields into path and body groups
+// The input pattern is expected to have been run through expandPattern,
+// with paths parameters denotes in {braces}.
+func splitFields(allFields map[string]*FieldSchema, pattern string) (pathFields, bodyFields map[string]*FieldSchema) {
+	pathFields = make(map[string]*FieldSchema)
+	bodyFields = make(map[string]*FieldSchema)
+
+	for _, match := range pathFieldsRe.FindAllStringSubmatch(pattern, -1) {
+		name := match[1]
+		pathFields[name] = allFields[name]
+	}
+
+	for name, field := range allFields {
+		if _, ok := pathFields[name]; !ok {
+			// Header fields are in "parameters" with other path fields
+			if field.Type == TypeHeader {
+				pathFields[name] = field
+			} else {
+				bodyFields[name] = field
+			}
+		}
+	}
+
+	return pathFields, bodyFields
+}
+
+// cleanedResponse is identical to logical.Response but with nulls
+// removed from from JSON encoding
+type cleanedResponse struct {
+	Secret   *logical.Secret            `json:"secret,omitempty"`
+	Auth     *logical.Auth              `json:"auth,omitempty"`
+	Data     map[string]interface{}     `json:"data,omitempty"`
+	Redirect string                     `json:"redirect,omitempty"`
+	Warnings []string                   `json:"warnings,omitempty"`
+	WrapInfo *wrapping.ResponseWrapInfo `json:"wrap_info,omitempty"`
+}
+
+func cleanResponse(resp *logical.Response) (*cleanedResponse, error) {
+	var r cleanedResponse
+
+	if err := mapstructure.Decode(resp, &r); err != nil {
+		return nil, err
+	}
+
+	return &r, nil
+}

logical/framework/openapi_test.go

+package framework
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/go-test/deep"
+	"github.com/hashicorp/vault/helper/jsonutil"
+	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/version"
+)
+
+func TestOpenAPI_Regex(t *testing.T) {
+	t.Run("Required", func(t *testing.T) {
+		tests := []struct {
+			input    string
+			captures []string
+		}{
+			{`/foo/bar/(?P<val>.*)`, []string{"val"}},
+			{`/foo/bar/` + GenericNameRegex("val"), []string{"val"}},
+			{`/foo/bar/` + GenericNameRegex("first") + "/b/" + GenericNameRegex("second"), []string{"first", "second"}},
+			{`/foo/bar`, []string{}},
+		}
+
+		for _, test := range tests {
+			result := reqdRe.FindAllStringSubmatch(test.input, -1)
+			if len(result) != len(test.captures) {
+				t.Fatalf("Capture error (%s): expected %d matches, actual: %d", test.input, len(test.captures), len(result))
+			}
+
+			for i := 0; i < len(result); i++ {
+				if result[i][1] != test.captures[i] {
+					t.Fatalf("Capture error (%s): expected %s, actual: %s", test.input, test.captures[i], result[i][1])
+				}
+			}
+		}
+	})
+	t.Run("Optional", func(t *testing.T) {
+		input := "foo/(maybe/)?bar"
+		expStart := len("foo/")
+		expEnd := len(input) - len("bar")
+
+		match := optRe.FindStringIndex(input)
+		if diff := deep.Equal(match, []int{expStart, expEnd}); diff != nil {
+			t.Fatal(diff)
+		}
+
+		input = "/foo/maybe/bar"
+		match = optRe.FindStringIndex(input)
+		if match != nil {
+			t.Fatalf("Expected nil match (%s), got %+v", input, match)
+		}
+	})
+	t.Run("Alternation", func(t *testing.T) {
+		input := `(raw/?$|raw/(?P<path>.+))`
+
+		matches := altRe.FindAllStringSubmatch(input, -1)
+		exp1 := "raw/?$"
+		exp2 := "raw/(?P<path>.+)"
+		if matches[0][1] != exp1 || matches[0][2] != exp2 {
+			t.Fatalf("Capture error. Expected %s and %s, got %v", exp1, exp2, matches[0][1:])
+		}
+
+		input = `/foo/bar/` + GenericNameRegex("val")
+
+		matches = altRe.FindAllStringSubmatch(input, -1)
+		if matches != nil {
+			t.Fatalf("Expected nil match (%s), got %+v", input, matches)
+		}
+	})
+	t.Run("Path fields", func(t *testing.T) {
+		input := `/foo/bar/{inner}/baz/{outer}`
+
+		matches := pathFieldsRe.FindAllStringSubmatch(input, -1)
+
+		exp1 := "inner"
+		exp2 := "outer"
+		if matches[0][1] != exp1 || matches[1][1] != exp2 {
+			t.Fatalf("Capture error. Expected %s and %s, got %v", exp1, exp2, matches)
+		}
+
+		input = `/foo/bar/inner/baz/outer`
+		matches = pathFieldsRe.FindAllStringSubmatch(input, -1)
+
+		if matches != nil {
+			t.Fatalf("Expected nil match (%s), got %+v", input, matches)
+		}
+	})
+	t.Run("Filtering", func(t *testing.T) {
+		tests := []struct {
+			input  string
+			regex  *regexp.Regexp
+			output string
+		}{
+			{
+				input:  `ab?cde^fg(hi?j$k`,
+				regex:  cleanCharsRe,
+				output: "abcdefghijk",
+			},
+			{
+				input:  `abcde/?`,
+				regex:  cleanSuffixRe,
+				output: "abcde",
+			},
+			{
+				input:  `abcde/?$`,
+				regex:  cleanSuffixRe,
+				output: "abcde",
+			},
+			{
+				input:  `abcde`,
+				regex:  wsRe,
+				output: "abcde",
+			},
+			{
+				input:  `  a         b    cd   e   `,
+				regex:  wsRe,
+				output: "abcde",
+			},
+		}
+
+		for _, test := range tests {
+			result := test.regex.ReplaceAllString(test.input, "")
+			if result != test.output {
+				t.Fatalf("Clean Regex error (%s). Expected %s, got %s", test.input, test.output, result)
+			}
+		}
+
+	})
+}
+
+func TestOpenAPI_ExpandPattern(t *testing.T) {
+	tests := []struct {
+		in_pattern   string
+		out_pathlets []string
+	}{
+		{"rekey/backup", []string{"rekey/backup"}},
+		{"rekey/backup$", []string{"rekey/backup"}},
+		{"auth/(?P<path>.+?)/tune$", []string{"auth/{path}/tune"}},
+		{"auth/(?P<path>.+?)/tune/(?P<more>.*?)$", []string{"auth/{path}/tune/{more}"}},
+		{"tools/hash(/(?P<urlalgorithm>.+))?", []string{
+			"tools/hash",
+			"tools/hash/{urlalgorithm}",
+		}},
+		{"(leases/)?renew(/(?P<url_lease_id>.+))?", []string{
+			"leases/renew",
+			"leases/renew/{url_lease_id}",
+			"renew",
+			"renew/{url_lease_id}",
+		}},
+		{`config/ui/headers/` + GenericNameRegex("header"), []string{"config/ui/headers/{header}"}},
+		{`leases/lookup/(?P<prefix>.+?)?`, []string{
+			"leases/lookup/",
+			"leases/lookup/{prefix}",
+		}},
+		{`(raw/?$|raw/(?P<path>.+))`, []string{
+			"raw",
+			"raw/{path}",
+		}},
+		{"lookup" + OptionalParamRegex("urltoken"), []string{
+			"lookup",
+			"lookup/{urltoken}",
+		}},
+		{"roles/?$", []string{
+			"roles",
+		}},
+		{"roles/?", []string{
+			"roles",
+		}},
+		{"accessors/$", []string{
+			"accessors/",
+		}},
+		{"verify/" + GenericNameRegex("name") + OptionalParamRegex("urlalgorithm"), []string{
+			"verify/{name}",
+			"verify/{name}/{urlalgorithm}",
+		}},
+	}
+
+	for i, test := range tests {
+		out := expandPattern(test.in_pattern)
+		sort.Strings(out)
+		if !reflect.DeepEqual(out, test.out_pathlets) {
+			t.Fatalf("Test %d: Expected %v got %v", i, test.out_pathlets, out)
+		}
+	}
+}
+
+func TestOpenAPI_SplitFields(t *testing.T) {
+	fields := map[string]*FieldSchema{
+		"a": {Description: "path"},
+		"b": {Description: "body"},
+		"c": {Description: "body"},
+		"d": {Description: "body"},
+		"e": {Description: "path"},
+	}
+
+	pathFields, bodyFields := splitFields(fields, "some/{a}/path/{e}")
+
+	lp := len(pathFields)
+	lb := len(bodyFields)
+	l := len(fields)
+	if lp+lb != l {
+		t.Fatalf("split length error: %d + %d != %d", lp, lb, l)
+	}
+
+	for name, field := range pathFields {
+		if field.Description != "path" {
+			t.Fatalf("expected field %s to be in 'path', found in %s", name, field.Description)
+		}
+	}
+	for name, field := range bodyFields {
+		if field.Description != "body" {
+			t.Fatalf("expected field %s to be in 'body', found in %s", name, field.Description)
+		}
+	}
+}
+
+func TestOpenAPI_SpecialPaths(t *testing.T) {
+	tests := []struct {
+		pattern     string
+		rootPaths   []string
+		root        bool
+		unauthPaths []string
+		unauth      bool
+	}{
+		{"foo", []string{}, false, []string{"foo"}, true},
+		{"foo", []string{"foo"}, true, []string{"bar"}, false},
+		{"foo/bar", []string{"foo"}, false, []string{"foo/*"}, true},
+		{"foo/bar", []string{"foo/*"}, true, []string{"foo"}, false},
+		{"foo/", []string{"foo/*"}, true, []string{"a", "b", "foo/"}, true},
+		{"foo", []string{"foo*"}, true, []string{"a", "fo*"}, true},
+		{"foo/bar", []string{"a", "b", "foo/*"}, true, []string{"foo/baz/*"}, false},
+	}
+	for i, test := range tests {
+		doc := NewOASDocument()
+		path := Path{
+			Pattern: test.pattern,
+		}
+		sp := &logical.Paths{
+			Root:            test.rootPaths,
+			Unauthenticated: test.unauthPaths,
+		}
+		documentPath(&path, sp, logical.TypeLogical, doc)
+		result := test.root
+		if doc.Paths["/"+test.pattern].Sudo != result {
+			t.Fatalf("Test (root) %d: Expected %v got %v", i, test.root, result)
+		}
+		result = test.unauth
+		if doc.Paths["/"+test.pattern].Unauthenticated != result {
+			t.Fatalf("Test (unauth) %d: Expected %v got %v", i, test.unauth, result)
+		}
+	}
+}
+
+func TestOpenAPI_Paths(t *testing.T) {
+	origDepth := deep.MaxDepth
+	defer func() { deep.MaxDepth = origDepth }()
+	deep.MaxDepth = 20
+
+	t.Run("Legacy callbacks", func(t *testing.T) {
+		p := &Path{
+			Pattern: "lookup/" + GenericNameRegex("id"),
+
+			Fields: map[string]*FieldSchema{
+				"id": &FieldSchema{
+					Type:        TypeString,
+					Description: "My id parameter",
+				},
+				"token": &FieldSchema{
+					Type:        TypeString,
+					Description: "My token",
+				},
+			},
+
+			Callbacks: map[logical.Operation]OperationFunc{
+				logical.ReadOperation:   nil,
+				logical.UpdateOperation: nil,
+			},
+
+			HelpSynopsis:    "Synopsis",
+			HelpDescription: "Description",
+		}
+
+		sp := &logical.Paths{
+			Root:            []string{},
+			Unauthenticated: []string{},
+		}
+		testPath(t, p, sp, expected("legacy"))
+	})
+
+	t.Run("Operations", func(t *testing.T) {
+		p := &Path{
+			Pattern: "foo/" + GenericNameRegex("id"),
+			Fields: map[string]*FieldSchema{
+				"id": {
+					Type:        TypeString,
+					Description: "id path parameter",
+				},
+				"flavors": {
+					Type:        TypeCommaStringSlice,
+					Description: "the flavors",
+				},
+				"name": {
+					Type:        TypeNameString,
+					Description: "the name",
+				},
+				"x-abc-token": {
+					Type:        TypeHeader,
+					Description: "a header value",
+				},
+			},
+			HelpSynopsis:    "Synopsis",
+			HelpDescription: "Description",
+			Operations: map[logical.Operation]OperationHandler{
+				logical.ReadOperation: &PathOperation{
+					Summary:     "My Summary",
+					Description: "My Description",
+				},
+				logical.UpdateOperation: &PathOperation{
+					Summary:     "Update Summary",
+					Description: "Update Description",
+				},
+				logical.CreateOperation: &PathOperation{
+					Summary:     "Create Summary",
+					Description: "Create Description",
+				},
+				logical.ListOperation: &PathOperation{
+					Summary:     "List Summary",
+					Description: "List Description",
+				},
+				logical.DeleteOperation: &PathOperation{
+					Summary:     "This shouldn't show up",
+					Unpublished: true,
+				},
+			},
+		}
+
+		sp := &logical.Paths{
+			Root: []string{"foo*"},
+		}
+		testPath(t, p, sp, expected("operations"))
+	})
+
+	t.Run("Responses", func(t *testing.T) {
+		p := &Path{
+			Pattern:         "foo",
+			HelpSynopsis:    "Synopsis",
+			HelpDescription: "Description",
+			Operations: map[logical.Operation]OperationHandler{
+				logical.ReadOperation: &PathOperation{
+					Summary:     "My Summary",
+					Description: "My Description",
+					Responses: map[int][]Response{
+						202: {{
+							Description: "Amazing",
+							Example: &logical.Response{
+								Data: map[string]interface{}{
+									"amount": 42,
+								},
+							},
+						}},
+					},
+				},
+				logical.DeleteOperation: &PathOperation{
+					Summary: "Delete stuff",
+				},
+			},
+		}
+
+		sp := &logical.Paths{
+			Unauthenticated: []string{"x", "y", "foo"},
+		}
+
+		testPath(t, p, sp, expected("responses"))
+	})
+}
+
+func TestOpenAPI_CustomDecoder(t *testing.T) {
+	p := &Path{
+		Pattern:      "foo",
+		HelpSynopsis: "Synopsis",
+		Operations: map[logical.Operation]OperationHandler{
+			logical.ReadOperation: &PathOperation{
+				Summary: "My Summary",
+				Responses: map[int][]Response{
+					100: {{
+						Description: "OK",
+						Example:     &logical.Response{},
+					}},
+					200: {{
+						Description: "Good",
+						Example:     &logical.Response{},
+					}},
+					599: {{
+						Description: "Bad",
+						Example:     &logical.Response{},
+					}},
+				},
+			},
+		},
+	}
+
+	docOrig := NewOASDocument()
+	documentPath(p, nil, logical.TypeLogical, docOrig)
+
+	docJSON, err := json.Marshal(docOrig)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var intermediate map[string]interface{}
+	if err := jsonutil.DecodeJSON(docJSON, &intermediate); err != nil {
+		t.Fatal(err)
+	}
+
+	docNew, err := NewOASDocumentFromMap(intermediate)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if diff := deep.Equal(docOrig, docNew); diff != nil {
+		t.Fatal(diff)
+	}
+}
+
+func testPath(t *testing.T, path *Path, sp *logical.Paths, expectedJSON string) {
+	t.Helper()
+
+	doc := NewOASDocument()
+	documentPath(path, sp, logical.TypeLogical, doc)
+
+	docJSON, err := json.MarshalIndent(doc, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Compare json by first decoding, then comparing with a deep equality check.
+	var expected, actual interface{}
+	if err := jsonutil.DecodeJSON(docJSON, &actual); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := jsonutil.DecodeJSON([]byte(expectedJSON), &expected); err != nil {
+		t.Fatal(err)
+	}
+
+	if diff := deep.Equal(actual, expected); diff != nil {
+		//fmt.Println(string(docJSON)) // uncomment to debug generated JSON (very helpful when fixing tests)
+		t.Fatal(diff)
+	}
+}
+
+func expected(name string) string {
+	data, err := ioutil.ReadFile(filepath.Join("testdata", name+".json"))
+	if err != nil {
+		panic(err)
+	}
+
+	content := strings.Replace(string(data), "<vault_version>", version.GetVersion().Version, 1)
+
+	return content
+}

logical/framework/path.go

@@ -29,6 +29,12 @@ func OptionalParamRegex(name string) string {
 	return fmt.Sprintf("(/(?P<%s>.+))?", name)
 }
 
+// Helper which returns a regex string for capturing an entire endpoint path
+// as the given name.
+func MatchAllRegex(name string) string {
+	return fmt.Sprintf(`(?P<%s>.*)`, name)
+}
+
 // PathAppend is a helper for appending lists of paths into a single
 // list.
 func PathAppend(paths ...[]*Path) []*Path {
@@ -58,6 +64,13 @@ type Path struct {
 	// whereas all fields are available in the Write operation.
 	Fields map[string]*FieldSchema
 
+	// Operations is the set of operations supported and the associated OperationsHandler.
+	//
+	// If both Create and Update operations are present, documentation and examples from
+	// the Update definition will be used. Similarly if both Read and List are present,
+	// Read will be used for documentation.
+	Operations map[logical.Operation]OperationHandler
+
 	// Callbacks are the set of callbacks that are called for a given
 	// operation. If a callback for a specific operation is not present,
 	// then logical.ErrUnsupportedOperation is automatically generated.
@@ -66,6 +79,8 @@ type Path struct {
 	// automatically handle if the Help field is set. If both the Help
 	// field is set and there is a callback registered here, then the
 	// callback will be called.
+	//
+	// Deprecated: Operations should be used instead and will take priority if present.
 	Callbacks map[logical.Operation]OperationFunc
 
 	// ExistenceCheck, if implemented, is used to query whether a given
@@ -80,6 +95,10 @@ type Path struct {
 	// enabled for the set of paths
 	FeatureRequired license.Features
 
+	// Deprecated denotes that this path is considered deprecated. This may
+	// be reflected in help and documentation.
+	Deprecated bool
+
 	// Help is text describing how to use this path. This will be used
 	// to auto-generate the help operation. The Path will automatically
 	// generate a parameter listing and URL structure based on the
@@ -95,7 +114,86 @@ type Path struct {
 	HelpDescription string
 }
 
-func (p *Path) helpCallback() OperationFunc {
+// OperationHandler defines and describes a specific operation handler.
+type OperationHandler interface {
+	Handler() OperationFunc
+	Properties() OperationProperties
+}
+
+// OperationProperties describes an operation for documentation, help text,
+// and other clients. A Summary should always be provided, whereas other
+// fields can be populated as needed.
+type OperationProperties struct {
+	// Summary is a brief (usually one line) description of the operation.
+	Summary string
+
+	// Description is extended documentation of the operation and may contain
+	// Markdown-formatted text markup.
+	Description string
+
+	// Examples provides samples of the expected request data. The most
+	// relevant example should be first in the list, as it will be shown in
+	// documentation that supports only a single example.
+	Examples []RequestExample
+
+	// Responses provides a list of response description for a given response
+	// code. The most relevant response should be first in the list, as it will
+	// be shown in documentation that only allows a single example.
+	Responses map[int][]Response
+
+	// Unpublished indicates that this operation should not appear in public
+	// documentation or help text. The operation may still have documentation
+	// attached that can be used internally.
+	Unpublished bool
+
+	// Deprecated indicates that this operation should be avoided.
+	Deprecated bool
+}
+
+// RequestExample is example of request data.
+type RequestExample struct {
+	Description string                 // optional description of the request
+	Data        map[string]interface{} // map version of sample JSON request data
+
+	// Optional example response to the sample request. This approach is considered
+	// provisional for now, and this field may be changed or removed.
+	Response *Response
+}
+
+// Response describes and optional demonstrations an operation response.
+type Response struct {
+	Description string            // summary of the the response and should always be provided
+	MediaType   string            // media type of the response, defaulting to "application/json" if empty
+	Example     *logical.Response // example response data
+}
+
+// PathOperation is a concrete implementation of OperationHandler.
+type PathOperation struct {
+	Callback    OperationFunc
+	Summary     string
+	Description string
+	Examples    []RequestExample
+	Responses   map[int][]Response
+	Unpublished bool
+	Deprecated  bool
+}
+
+func (p *PathOperation) Handler() OperationFunc {
+	return p.Callback
+}
+
+func (p *PathOperation) Properties() OperationProperties {
+	return OperationProperties{
+		Summary:     strings.TrimSpace(p.Summary),
+		Description: strings.TrimSpace(p.Description),
+		Responses:   p.Responses,
+		Examples:    p.Examples,
+		Unpublished: p.Unpublished,
+		Deprecated:  p.Deprecated,
+	}
+}
+
+func (p *Path) helpCallback(b *Backend) OperationFunc {
 	return func(ctx context.Context, req *logical.Request, data *FieldData) (*logical.Response, error) {
 		var tplData pathTemplateData
 		tplData.Request = req.Path
@@ -137,7 +235,13 @@ func (p *Path) helpCallback() OperationFunc {
 			return nil, errwrap.Wrapf("error executing template: {{err}}", err)
 		}
 
-		return logical.HelpResponse(help, nil), nil
+		// Build OpenAPI response for this path
+		doc := NewOASDocument()
+		if err := documentPath(p, b.SpecialPaths(), b.BackendType, doc); err != nil {
+			b.Logger().Warn("error generating OpenAPI", "error", err)
+		}
+
+		return logical.HelpResponse(help, nil, doc), nil
 	}
 }
 

logical/framework/path_test.go

+package framework
+
+import (
+	"testing"
+
+	"github.com/go-test/deep"
+)
+
+func TestPath_Regex(t *testing.T) {
+	tests := []struct {
+		pattern   string
+		input     string
+		pathMatch bool
+		captures  map[string]string
+	}{
+		{
+			pattern:   "a/b/" + GenericNameRegex("val"),
+			input:     "a/b/foo",
+			pathMatch: true,
+			captures:  map[string]string{"val": "foo"},
+		},
+		{
+			pattern:   "a/b/" + GenericNameRegex("val"),
+			input:     "a/b/foo/more",
+			pathMatch: false,
+			captures:  nil,
+		},
+		{
+			pattern:   "a/b/" + GenericNameRegex("val"),
+			input:     "a/b/abc-.123",
+			pathMatch: true,
+			captures:  map[string]string{"val": "abc-.123"},
+		},
+		{
+			pattern:   "a/b/" + GenericNameRegex("val") + "/c/d",
+			input:     "a/b/foo/c/d",
+			pathMatch: true,
+			captures:  map[string]string{"val": "foo"},
+		},
+		{
+			pattern:   "a/b/" + GenericNameRegex("val") + "/c/d",
+			input:     "a/b/foo/c/d/e",
+			pathMatch: false,
+			captures:  nil,
+		},
+		{
+			pattern:   "a/b" + OptionalParamRegex("val"),
+			input:     "a/b",
+			pathMatch: true,
+			captures:  map[string]string{"val": ""},
+		},
+		{
+			pattern:   "a/b" + OptionalParamRegex("val"),
+			input:     "a/b/foo",
+			pathMatch: true,
+			captures:  map[string]string{"val": "foo"},
+		},
+		{
+			pattern:   "foo/" + MatchAllRegex("val"),
+			input:     "foos/ball",
+			pathMatch: false,
+			captures:  nil,
+		},
+		{
+			pattern:   "foos/" + MatchAllRegex("val"),
+			input:     "foos/ball",
+			pathMatch: true,
+			captures:  map[string]string{"val": "ball"},
+		},
+		{
+			pattern:   "foos/ball/" + MatchAllRegex("val"),
+			input:     "foos/ball/with/more/stuff/at_the/end",
+			pathMatch: true,
+			captures:  map[string]string{"val": "with/more/stuff/at_the/end"},
+		},
+		{
+			pattern:   MatchAllRegex("val"),
+			input:     "foos/ball/with/more/stuff/at_the/end",
+			pathMatch: true,
+			captures:  map[string]string{"val": "foos/ball/with/more/stuff/at_the/end"},
+		},
+	}
+
+	for i, test := range tests {
+		b := Backend{
+			Paths: []*Path{{Pattern: test.pattern}},
+		}
+		path, captures := b.route(test.input)
+		pathMatch := path != nil
+		if pathMatch != test.pathMatch {
+			t.Fatalf("[%d] unexpected path match result (%s): expected %t, actual %t", i, test.pattern, test.pathMatch, pathMatch)
+		}
+		if diff := deep.Equal(captures, test.captures); diff != nil {
+			t.Fatal(diff)
+		}
+	}
+
+}

logical/framework/testdata/legacy.json

+{
+  "openapi": "3.0.2",
+  "info": {
+    "title": "HashiCorp Vault API",
+    "description": "HTTP API that gives you full access to Vault. All API routes are prefixed with `/v1/`.",
+    "version": "<vault_version>",
+    "license": {
+      "name": "Mozilla Public License 2.0",
+      "url": "https://www.mozilla.org/en-US/MPL/2.0"
+    }
+  },
+  "paths": {
+    "/lookup/{id}": {
+      "description": "Synopsis",
+      "parameters": [
+        {
+          "name": "id",
+          "description": "My id parameter",
+          "in": "path",
+          "schema": {
+            "type": "string"
+          },
+          "required": true
+        }
+      ],
+      "get": {
+        "summary": "Synopsis",
+        "tags": ["secrets"],
+        "responses": {
+          "200": {
+            "description": "OK"
+          }
+        }
+      },
+      "post": {
+        "summary": "Synopsis",
+        "tags": ["secrets"],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "token": {
+                    "type": "string",
+                    "description": "My token"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "OK"
+          }
+        }
+      }
+    }
+  }
+}
+

logical/framework/testdata/operations.json

+{
+  "openapi": "3.0.2",
+  "info": {
+    "title": "HashiCorp Vault API",
+    "description": "HTTP API that gives you full access to Vault. All API routes are prefixed with `/v1/`.",
+    "version": "<vault_version>",
+    "license": {
+      "name": "Mozilla Public License 2.0",
+      "url": "https://www.mozilla.org/en-US/MPL/2.0"
+    }
+  },
+  "paths": {
+    "/foo/{id}": {
+      "description": "Synopsis",
+      "x-vault-create-supported": true,
+      "x-vault-sudo": true,
+      "parameters": [
+        {
+          "name": "id",
+          "description": "id path parameter",
+          "in": "path",
+          "schema": {
+            "type": "string"
+          },
+          "required": true
+        },
+        {
+          "name": "x-abc-token",
+          "description": "a header value",
+          "in": "header",
+          "schema": {
+            "type": "string"
+          }
+        }
+      ],
+      "get": {
+        "tags": ["secrets"],
+        "summary": "My Summary",
+        "description": "My Description",
+        "responses": {
+          "200": {
+            "description": "OK"
+          }
+        },
+        "parameters": [
+          {
+            "name": "list",
+            "description": "Return a list if `true`",
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ]
+      },
+      "post": {
+        "tags": ["secrets"],
+        "summary": "Update Summary",
+        "description": "Update Description",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "flavors": {
+                    "type": "array",
+                    "description": "the flavors",
+                    "items": {
+                      "type": "string"
+                    }
+                  },
+                  "name": {
+                    "type": "string",
+                    "description": "the name",
+                    "pattern": "\\w([\\w-.]*\\w)?"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "OK"
+          }
+        }
+      }
+    }
+  }
+}

logical/framework/testdata/responses.json

+{
+  "openapi": "3.0.2",
+  "info": {
+    "title": "HashiCorp Vault API",
+    "description": "HTTP API that gives you full access to Vault. All API routes are prefixed with `/v1/`.",
+    "version": "<vault_version>",
+    "license": {
+      "name": "Mozilla Public License 2.0",
+      "url": "https://www.mozilla.org/en-US/MPL/2.0"
+    }
+  },
+  "paths": {
+    "/foo": {
+      "description": "Synopsis",
+      "x-vault-unauthenticated": true,
+      "delete": {
+        "tags": ["secrets"],
+        "summary": "Delete stuff",
+        "responses": {
+          "204": {
+            "description": "empty body"
+          }
+        }
+      },
+      "get": {
+        "tags": ["secrets"],
+        "summary": "My Summary",
+        "description": "My Description",
+        "responses": {
+          "202": {
+            "description": "Amazing",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "example": {
+                    "data": {
+                      "amount": 42
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+

logical/response.go

@@ -89,11 +89,12 @@ func (r *Response) Error() error {
 }
 
 // HelpResponse is used to format a help response
-func HelpResponse(text string, seeAlso []string) *Response {
+func HelpResponse(text string, seeAlso []string, oapiDoc interface{}) *Response {
 	return &Response{
 		Data: map[string]interface{}{
 			"help":     text,
 			"see_also": seeAlso,
+			"openapi":  oapiDoc,
 		},
 	}
 }

scripts/gen_openapi.sh

+#!/bin/sh
+
+set -e
+
+# Generate an OpenAPI document for all backends.
+#
+# Assumptions:
+#
+#   1. Vault has been checked out at an appropriate version and built
+#   2. vault executable is in your path
+#   3. Vault isn't alredy running
+
+echo "Starting Vault..."
+if pgrep -x "vault" > /dev/null
+then
+    echo "Vault is already running. Aborting."
+    exit 1
+fi
+
+vault server -dev -dev-root-token-id=root &
+sleep 2
+VAULT_PID=$!
+
+echo "Mounting all builtin backends..."
+
+#  auth backends
+vault auth enable alicloud
+vault auth enable app-id
+vault auth enable approle
+vault auth enable aws
+vault auth enable azure
+vault auth enable centrify
+vault auth enable cert
+vault auth enable gcp
+vault auth enable github
+vault auth enable jwt
+vault auth enable kubernetes
+vault auth enable ldap
+vault auth enable okta
+vault auth enable radius
+vault auth enable userpass
+
+# secrets backends
+vault secrets enable ad
+vault secrets enable alicloud
+vault secrets enable aws
+vault secrets enable azure
+vault secrets enable cassandra
+vault secrets enable consul
+vault secrets enable database
+vault secrets enable gcp
+vault secrets enable kv
+vault secrets enable mongodb
+vault secrets enable mssql
+vault secrets enable mysql
+vault secrets enable nomad
+vault secrets enable pki
+vault secrets enable postgresql
+vault secrets enable rabbitmq
+vault secrets enable ssh
+vault secrets enable totp
+vault secrets enable transit
+
+curl -H "X-Vault-Token: root" "http://127.0.0.1:8200/v1/sys/internal/specs/openapi" > openapi.json
+
+kill $VAULT_PID
+sleep 1
+
+echo "\nopenapi.json generated."

vault/logical_system.go

@@ -108,6 +108,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend {
 				"wrapping/lookup",
 				"wrapping/pubkey",
 				"replication/status",
+				"internal/specs/openapi",
 				"internal/ui/mounts",
 				"internal/ui/mounts/*",
 				"internal/ui/namespaces",
@@ -141,7 +142,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend {
 	b.Backend.Paths = append(b.Backend.Paths, b.wrappingPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.toolsPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.capabilitiesPaths()...)
-	b.Backend.Paths = append(b.Backend.Paths, b.internalUIPaths()...)
+	b.Backend.Paths = append(b.Backend.Paths, b.internalPaths()...)
 	b.Backend.Paths = append(b.Backend.Paths, b.remountPath())
 
 	if core.rawEnabled {
@@ -2958,6 +2959,110 @@ func (b *SystemBackend) pathInternalUIResultantACL(ctx context.Context, req *log
 	return resp, nil
 }
 
+func (b *SystemBackend) pathInternalOpenAPI(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
+
+	// Limit output to authorized paths
+	resp, err := b.pathInternalUIMountsRead(ctx, req, d)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set up target document and convert to map[string]interface{} which is what will
+	// be received from plugin backends.
+	doc := framework.NewOASDocument()
+
+	procMountGroup := func(group, mountPrefix string) error {
+		for mount := range resp.Data[group].(map[string]interface{}) {
+			backend := b.Core.router.MatchingBackend(ctx, mountPrefix+mount)
+
+			if backend == nil {
+				continue
+			}
+
+			req := &logical.Request{
+				Operation: logical.HelpOperation,
+			}
+
+			resp, err := backend.HandleRequest(ctx, req)
+			if err != nil {
+				return err
+			}
+
+			var backendDoc *framework.OASDocument
+
+			// Normalize response type, which will be different if received
+			// from an external plugin.
+			switch v := resp.Data["openapi"].(type) {
+			case *framework.OASDocument:
+				backendDoc = v
+			case map[string]interface{}:
+				backendDoc, err = framework.NewOASDocumentFromMap(v)
+				if err != nil {
+					return err
+				}
+			default:
+				continue
+			}
+
+			// Prepare to add tags to default builtins that are
+			// type "unknown" and won't already be tagged.
+			var tag string
+			switch mountPrefix + mount {
+			case "cubbyhole/", "secret/":
+				tag = "secrets"
+			case "sys/":
+				tag = "system"
+			case "auth/token/":
+				tag = "auth"
+			case "identity/":
+				tag = "identity"
+			}
+
+			// Merge backend paths with existing document
+			for path, obj := range backendDoc.Paths {
+				path := strings.TrimPrefix(path, "/")
+
+				// Add tags to all of the operations if necessary
+				if tag != "" {
+					for _, op := range []*framework.OASOperation{obj.Get, obj.Post, obj.Delete} {
+						// TODO: a special override for identity is used used here because the backend
+						// is currently categorized as "secret", which will likely change. Also of interest
+						// is removing all tag handling here and providing the mount information to OpenAPI.
+						if op != nil && (len(op.Tags) == 0 || tag == "identity") {
+							op.Tags = []string{tag}
+						}
+					}
+				}
+
+				doc.Paths["/"+mountPrefix+mount+path] = obj
+			}
+		}
+		return nil
+	}
+
+	if err := procMountGroup("secret", ""); err != nil {
+		return nil, err
+	}
+	if err := procMountGroup("auth", "auth/"); err != nil {
+		return nil, err
+	}
+
+	buf, err := json.Marshal(doc)
+	if err != nil {
+		return nil, err
+	}
+
+	resp = &logical.Response{
+		Data: map[string]interface{}{
+			logical.HTTPStatusCode:  200,
+			logical.HTTPRawBody:     buf,
+			logical.HTTPContentType: "application/json",
+		},
+	}
+
+	return resp, nil
+}
+
 func sanitizeMountPath(path string) string {
 	if !strings.HasSuffix(path, "/") {
 		path += "/"

vault/logical_system_paths.go

@@ -423,8 +423,14 @@ func (b *SystemBackend) toolsPaths() []*framework.Path {
 	}
 }
 
-func (b *SystemBackend) internalUIPaths() []*framework.Path {
+func (b *SystemBackend) internalPaths() []*framework.Path {
 	return []*framework.Path{
+		{
+			Pattern: "internal/specs/openapi",
+			Callbacks: map[logical.Operation]framework.OperationFunc{
+				logical.ReadOperation: b.pathInternalOpenAPI,
+			},
+		},
 		{
 			Pattern: "internal/ui/mounts",
 			Callbacks: map[logical.Operation]framework.OperationFunc{

vault/logical_system_test.go

@@ -19,9 +19,12 @@ import (
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/audit"
 	"github.com/hashicorp/vault/helper/builtinplugins"
+	"github.com/hashicorp/vault/helper/jsonutil"
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/logical/framework"
+	"github.com/hashicorp/vault/version"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -2447,3 +2450,102 @@ func TestSystemBackend_InternalUIMount(t *testing.T) {
 		t.Fatal("expected permission denied error")
 	}
 }
+
+func TestSystemBackend_OpenAPI(t *testing.T) {
+	_, b, rootToken := testCoreSystemBackend(t)
+	var oapi map[string]interface{}
+
+	// Ensure no paths are reported if there is no token
+	req := logical.TestRequest(t, logical.ReadOperation, "internal/specs/openapi")
+	resp, err := b.HandleRequest(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	body := resp.Data["http_raw_body"].([]byte)
+	err = jsonutil.DecodeJSON(body, &oapi)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	exp := map[string]interface{}{
+		"openapi": framework.OASVersion,
+		"info": map[string]interface{}{
+			"title":       "HashiCorp Vault API",
+			"description": "HTTP API that gives you full access to Vault. All API routes are prefixed with `/v1/`.",
+			"version":     version.GetVersion().Version,
+			"license": map[string]interface{}{
+				"name": "Mozilla Public License 2.0",
+				"url":  "https://www.mozilla.org/en-US/MPL/2.0",
+			},
+		},
+		"paths": map[string]interface{}{},
+	}
+
+	if diff := deep.Equal(oapi, exp); diff != nil {
+		t.Fatal(diff)
+	}
+
+	// Check that default paths are present with a root token
+	req = logical.TestRequest(t, logical.ReadOperation, "internal/specs/openapi")
+	req.ClientToken = rootToken
+	resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	body = resp.Data["http_raw_body"].([]byte)
+	err = jsonutil.DecodeJSON(body, &oapi)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	doc, err := framework.NewOASDocumentFromMap(oapi)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pathSamples := []struct {
+		path string
+		tag  string
+	}{
+		{"/auth/token/lookup", "auth"},
+		{"/cubbyhole/.*", "secrets"}, // TODO update after sys docs update
+		{"/identity/group/id", "identity"},
+		{"/secret/.*", "secrets"}, // TODO update after sys docs update
+		{"/sys/policy", "system"},
+	}
+
+	for _, path := range pathSamples {
+		if doc.Paths[path.path] == nil {
+			t.Fatalf("didn't find expected path '%s'.", path)
+		}
+		tag := doc.Paths[path.path].Get.Tags[0]
+		if tag != path.tag {
+			t.Fatalf("path: %s; expected tag: %s, actual: %s", path.path, tag, path.tag)
+		}
+	}
+
+	// Simple sanity check of response size (which is much larger than most
+	// Vault responses), mainly to catch mass omission of expected path data.
+	minLen := 70000
+	if len(body) < minLen {
+		t.Fatalf("response size too small; expected: min %d, actual: %d", minLen, len(body))
+	}
+
+	// Test path-help response
+	req = logical.TestRequest(t, logical.HelpOperation, "rotate")
+	req.ClientToken = rootToken
+	resp, err = b.HandleRequest(namespace.RootContext(nil), req)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	doc = resp.Data["openapi"].(*framework.OASDocument)
+	if len(doc.Paths) != 1 {
+		t.Fatalf("expected 1 path, actual: %d", len(doc.Paths))
+	}
+
+	if doc.Paths["/rotate"] == nil {
+		t.Fatalf("expected to find path '/rotate'")
+	}
+}