improve documentation

move metrics to telemetry; copy to client config

client/client.go

@@ -157,9 +157,6 @@ type Client struct {
 	// baseLabels are used when emitting tagged metrics. All client metrics will
 	// have these tags, and optionally more.
 	baseLabels []metrics.Label
-
-	// Subset of global telemetry configuration options for the client
-	clientTelemetry *ClientTelemetry
 }
 
 var (
@@ -169,15 +166,8 @@ var (
 	noServersErr = errors.New("no servers")
 )
 
-// ClientTelemetry is a subset of global telemetry configuration options that
-// are relevant for the client
-type ClientTelemetry struct {
-	DisableTaggedMetrics       bool
-	BackwardsCompatibleMetrics bool
-}
-
 // NewClient is used to create a new client from the given configuration
-func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger, telemetry *ClientTelemetry) (*Client, error) {
+func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger) (*Client, error) {
 	// Create the tls wrapper
 	var tlsWrap tlsutil.RegionWrapper
 	if cfg.TLSConfig.EnableRPC {
@@ -190,7 +180,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic
 
 	// Create the client
 	c := &Client{
-		clientTelemetry:     telemetry,
 		config:              cfg,
 		consulCatalog:       consulCatalog,
 		consulService:       consulService,
@@ -1890,14 +1879,14 @@ func (c *Client) emitStats() {
 
 // setGaugeForMemoryStats proxies metrics for memory specific statistics
 func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats) {
-	if !c.clientTelemetry.DisableTaggedMetrics {
+	if !c.config.DisableTaggedMetrics {
 		metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "total"}, float32(hStats.Memory.Total), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "available"}, float32(hStats.Memory.Available), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "used"}, float32(hStats.Memory.Used), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "free"}, float32(hStats.Memory.Free), c.baseLabels)
 	}
 
-	if c.clientTelemetry.BackwardsCompatibleMetrics {
+	if c.config.BackwardsCompatibleMetrics {
 		metrics.SetGauge([]string{"client", "host", "memory", nodeID, "total"}, float32(hStats.Memory.Total))
 		metrics.SetGauge([]string{"client", "host", "memory", nodeID, "available"}, float32(hStats.Memory.Available))
 		metrics.SetGauge([]string{"client", "host", "memory", nodeID, "used"}, float32(hStats.Memory.Used))
@@ -1908,7 +1897,7 @@ func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats)
 // setGaugeForCPUStats proxies metrics for CPU specific statistics
 func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
 	for _, cpu := range hStats.CPU {
-		if !c.clientTelemetry.DisableTaggedMetrics {
+		if !c.config.DisableTaggedMetrics {
 			labels := append(c.baseLabels, metrics.Label{"cpu", cpu.CPU})
 
 			metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "total"}, float32(cpu.Total), labels)
@@ -1917,7 +1906,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
 			metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "system"}, float32(cpu.System), labels)
 		}
 
-		if c.clientTelemetry.BackwardsCompatibleMetrics {
+		if c.config.BackwardsCompatibleMetrics {
 			metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "total"}, float32(cpu.Total))
 			metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "user"}, float32(cpu.User))
 			metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "idle"}, float32(cpu.Idle))
@@ -1929,7 +1918,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
 // setGaugeForDiskStats proxies metrics for disk specific statistics
 func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
 	for _, disk := range hStats.DiskStats {
-		if !c.clientTelemetry.DisableTaggedMetrics {
+		if !c.config.DisableTaggedMetrics {
 			labels := append(c.baseLabels, metrics.Label{"disk", disk.Device})
 
 			metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "size"}, float32(disk.Size), labels)
@@ -1939,7 +1928,7 @@ func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
 			metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "inodes_percent"}, float32(disk.InodesUsedPercent), labels)
 		}
 
-		if c.clientTelemetry.BackwardsCompatibleMetrics {
+		if c.config.BackwardsCompatibleMetrics {
 			metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "size"}, float32(disk.Size))
 			metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "used"}, float32(disk.Used))
 			metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "available"}, float32(disk.Available))
@@ -1958,14 +1947,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
 	allocated := c.getAllocatedResources(node)
 
 	// Emit allocated
-	if !c.clientTelemetry.DisableTaggedMetrics {
+	if !c.config.DisableTaggedMetrics {
 		metrics.SetGaugeWithLabels([]string{"client", "allocated", "memory"}, float32(allocated.MemoryMB), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocated", "disk"}, float32(allocated.DiskMB), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocated", "cpu"}, float32(allocated.CPU), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocated", "iops"}, float32(allocated.IOPS), c.baseLabels)
 	}
 
-	if c.clientTelemetry.BackwardsCompatibleMetrics {
+	if c.config.BackwardsCompatibleMetrics {
 		metrics.SetGauge([]string{"client", "allocated", "memory", nodeID}, float32(allocated.MemoryMB))
 		metrics.SetGauge([]string{"client", "allocated", "disk", nodeID}, float32(allocated.DiskMB))
 		metrics.SetGauge([]string{"client", "allocated", "cpu", nodeID}, float32(allocated.CPU))
@@ -1973,12 +1962,12 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
 	}
 
 	for _, n := range allocated.Networks {
-		if !c.clientTelemetry.DisableTaggedMetrics {
+		if !c.config.DisableTaggedMetrics {
 			labels := append(c.baseLabels, metrics.Label{"device", n.Device})
 			metrics.SetGaugeWithLabels([]string{"client", "allocated", "network"}, float32(n.MBits), labels)
 		}
 
-		if c.clientTelemetry.BackwardsCompatibleMetrics {
+		if c.config.BackwardsCompatibleMetrics {
 			metrics.SetGauge([]string{"client", "allocated", "network", n.Device, nodeID}, float32(n.MBits))
 		}
 	}
@@ -1989,14 +1978,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
 	unallocatedCpu := total.CPU - res.CPU - allocated.CPU
 	unallocatedIops := total.IOPS - res.IOPS - allocated.IOPS
 
-	if !c.clientTelemetry.DisableTaggedMetrics {
+	if !c.config.DisableTaggedMetrics {
 		metrics.SetGaugeWithLabels([]string{"client", "unallocated", "memory"}, float32(unallocatedMem), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "unallocated", "disk"}, float32(unallocatedDisk), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "unallocated", "cpu"}, float32(unallocatedCpu), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "unallocated", "iops"}, float32(unallocatedIops), c.baseLabels)
 	}
 
-	if c.clientTelemetry.BackwardsCompatibleMetrics {
+	if c.config.BackwardsCompatibleMetrics {
 		metrics.SetGauge([]string{"client", "unallocated", "memory", nodeID}, float32(unallocatedMem))
 		metrics.SetGauge([]string{"client", "unallocated", "disk", nodeID}, float32(unallocatedDisk))
 		metrics.SetGauge([]string{"client", "unallocated", "cpu", nodeID}, float32(unallocatedCpu))
@@ -2014,7 +2003,7 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
 
 		unallocatedMbits := totalMbits - n.MBits
 
-		if !c.clientTelemetry.DisableTaggedMetrics {
+		if !c.config.DisableTaggedMetrics {
 			labels := append(c.baseLabels, metrics.Label{"device", n.Device})
 			metrics.SetGaugeWithLabels([]string{"client", "unallocated", "network"}, float32(unallocatedMbits), labels)
 		}
@@ -2027,10 +2016,10 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
 
 // No lables are required so we emit with only a key/value syntax
 func (c *Client) setGaugeForUptime(hStats *stats.HostStats) {
-	if !c.clientTelemetry.DisableTaggedMetrics {
+	if !c.config.DisableTaggedMetrics {
 		metrics.SetGaugeWithLabels([]string{"uptime"}, float32(hStats.Uptime), c.baseLabels)
 	}
-	if c.clientTelemetry.BackwardsCompatibleMetrics {
+	if c.config.BackwardsCompatibleMetrics {
 		metrics.SetGauge([]string{"uptime"}, float32(hStats.Uptime))
 	}
 }
@@ -2073,7 +2062,7 @@ func (c *Client) emitClientMetrics() {
 		}
 	}
 
-	if !c.clientTelemetry.DisableTaggedMetrics {
+	if !c.config.DisableTaggedMetrics {
 		metrics.SetGaugeWithLabels([]string{"client", "allocations", "migrating"}, float32(migrating), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocations", "blocked"}, float32(blocked), c.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocations", "pending"}, float32(pending), c.baseLabels)
@@ -2081,7 +2070,7 @@ func (c *Client) emitClientMetrics() {
 		metrics.SetGaugeWithLabels([]string{"client", "allocations", "terminal"}, float32(terminal), c.baseLabels)
 	}
 
-	if c.clientTelemetry.BackwardsCompatibleMetrics {
+	if c.config.BackwardsCompatibleMetrics {
 		metrics.SetGauge([]string{"client", "allocations", "migrating", nodeID}, float32(migrating))
 		metrics.SetGauge([]string{"client", "allocations", "blocked", nodeID}, float32(blocked))
 		metrics.SetGauge([]string{"client", "allocations", "pending", nodeID}, float32(pending))

client/client_test.go

@@ -122,8 +122,7 @@ func testClient(t *testing.T, cb func(c *config.Config)) *Client {
 	catalog := consul.NewMockCatalog(logger)
 	mockService := newMockConsulServiceClient()
 	mockService.logger = logger
-	telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
-	client, err := NewClient(conf, catalog, mockService, logger, telemetry)
+	client, err := NewClient(conf, catalog, mockService, logger)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -795,8 +794,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
 	catalog := consul.NewMockCatalog(logger)
 	mockService := newMockConsulServiceClient()
 	mockService.logger = logger
-	telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
-	c2, err := NewClient(c1.config, catalog, mockService, logger, telemetry)
+	c2, err := NewClient(c1.config, catalog, mockService, logger)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

command/agent/agent.go

@@ -335,9 +335,13 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 
 	conf.ConsulConfig = a.config.Consul
 	conf.VaultConfig = a.config.Vault
+
+	// Set up Telemetry configuration
 	conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
 	conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
 	conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics
+	conf.DisableTaggedMetrics = a.config.Telemetry.DisableTaggedMetrics
+	conf.BackwardsCompatibleMetrics = a.config.Telemetry.BackwardsCompatibleMetrics
 
 	// Set the TLS related configs
 	conf.TLSConfig = a.config.TLSConfig
@@ -493,13 +497,7 @@ func (a *Agent) setupClient() error {
 		}
 	}
 
-	// Create the client
-	clientTelemetry := &client.ClientTelemetry{
-		DisableTaggedMetrics:       a.config.Telemetry.DisableTaggedMetrics,
-		BackwardsCompatibleMetrics: a.config.Telemetry.BackwardsCompatibleMetrics,
-	}
-
-	client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger, clientTelemetry)
+	client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger)
 	if err != nil {
 		return fmt.Errorf("client setup failed: %v", err)
 	}

command/agent/agent_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/hashicorp/nomad/helper"
 	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/stretchr/testify/assert"
 )
 
 func getPort() int {
@@ -316,6 +317,29 @@ func TestAgent_ClientConfig(t *testing.T) {
 	}
 }
 
+// Clients should inherit telemetry configuration
+func TestAget_Client_TelemetryConfiguration(t *testing.T) {
+	assert := assert.New(t)
+
+	conf := DefaultConfig()
+	conf.DevMode = true
+	conf.Telemetry.DisableTaggedMetrics = true
+	conf.Telemetry.BackwardsCompatibleMetrics = true
+
+	a := &Agent{config: conf}
+
+	c, err := a.clientConfig()
+	assert.Nil(err)
+
+	telemetry := conf.Telemetry
+
+	assert.Equal(c.StatsCollectionInterval, telemetry.collectionInterval)
+	assert.Equal(c.PublishNodeMetrics, telemetry.PublishNodeMetrics)
+	assert.Equal(c.PublishAllocationMetrics, telemetry.PublishAllocationMetrics)
+	assert.Equal(c.DisableTaggedMetrics, telemetry.DisableTaggedMetrics)
+	assert.Equal(c.BackwardsCompatibleMetrics, telemetry.BackwardsCompatibleMetrics)
+}
+
 // TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP
 // API health check depending on configuration.
 func TestAgent_HTTPCheck(t *testing.T) {

command/agent/config-test-fixtures/basic.hcl

@@ -96,8 +96,8 @@ telemetry {
     collection_interval = "3s"
     publish_allocation_metrics = true
     publish_node_metrics = true
-  disable_tagged_metrics = true
-  backwards_compatible_metrics = true
+    disable_tagged_metrics = true
+    backwards_compatible_metrics = true
 }
 leave_on_interrupt = true
 leave_on_terminate = true

command/agent/testagent.go

@@ -192,7 +192,7 @@ func (a *TestAgent) start() (*Agent, error) {
 	metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm)
 
 	if inm == nil {
-		return nil, fmt.Errorf("unable to set up in memory metrics neede for agent initialization")
+		return nil, fmt.Errorf("unable to set up in memory metrics needed for agent initialization")
 	}
 
 	agent, err := NewAgent(a.Config, a.LogOutput, inm)

website/source/docs/agent/configuration/telemetry.html.md

@@ -60,11 +60,17 @@ The following options are available on all telemetry configurations.
 
 - `backwards_compatible_metrics` `(bool: false)` - Specifies if Nomad should
   publish metrics that are backwards compatible with versions below 0.7, as
-  post version 0.7, Nomad emits tagged metrics.
+  post version 0.7, Nomad emits tagged metrics. and all new metrics will
+  only be added to tagged metrics. Note that this option is used to transition
+  monitoring to tagged metrics and will eventually be deprecated.
+
 
 - `disable_tagged_metrics` `(bool: false)` - Specifies if Nomad should not emit
   tagged metrics and only emit metrics compatible with versions below Nomad
-  0.7.
+  0.7. Note that this option is used to transition monitoring to tagged
+  metrics and will eventually be deprecated.
+
+
 
 ### `statsite`
 
@@ -170,6 +176,3 @@ These `telemetry` parameters apply to
   best use of this is to as a hint for which broker should be used based on
   *where* this particular instance is running (e.g. a specific geographic location or
   datacenter, dc:sfo).
-
-
-