Merge pull request #5355 from hashicorp/dani/windows-dockerstats

docker: Support Stats on Windows

drivers/docker/driver.go

@@ -42,10 +42,6 @@ var (
 	// running operations such as waiting on containers and collect stats
 	waitClient *docker.Client
 
-	// The statistics the Docker driver exposes
-	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"}
-	DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
-
 	// recoverableErrTimeouts returns a recoverable error if the error was due
 	// to timeouts
 	recoverableErrTimeouts = func(err error) error {

drivers/docker/stats.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"runtime"
 	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
 	cstructs "github.com/hashicorp/nomad/client/structs"
-	"github.com/hashicorp/nomad/helper/stats"
+	"github.com/hashicorp/nomad/drivers/docker/util"
 	nstructs "github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -86,6 +85,7 @@ func (h *taskHandle) collectStats(ctx context.Context, ch chan *cstructs.TaskRes
 		return
 	}
 }
+
 func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-chan *docker.Stats, interval time.Duration) {
 	var resourceUsage *cstructs.TaskResourceUsage
 
@@ -117,7 +117,7 @@ func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-cha
 			}
 			// s should always be set, but check and skip just in case
 			if s != nil {
-				resourceUsage = dockerStatsToTaskResourceUsage(s)
+				resourceUsage = util.DockerStatsToTaskResourceUsage(s)
 				// send stats next interation if this is the first time received
 				// from docker
 				if !hasSentInitialStats {
@@ -128,50 +128,3 @@ func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-cha
 		}
 	}
 }
-
-func dockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage {
-	ms := &cstructs.MemoryStats{
-		RSS:      s.MemoryStats.Stats.Rss,
-		Cache:    s.MemoryStats.Stats.Cache,
-		Swap:     s.MemoryStats.Stats.Swap,
-		Usage:    s.MemoryStats.Usage,
-		MaxUsage: s.MemoryStats.MaxUsage,
-		Measured: DockerMeasuredMemStats,
-	}
-
-	cs := &cstructs.CpuStats{
-		ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
-		ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
-		Measured:         DockerMeasuredCpuStats,
-	}
-
-	// Calculate percentage
-	cs.Percent = calculatePercent(
-		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
-		s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, runtime.NumCPU())
-	cs.SystemMode = calculatePercent(
-		s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
-		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
-	cs.UserMode = calculatePercent(
-		s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
-		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
-	cs.TotalTicks = (cs.Percent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU())
-
-	return &cstructs.TaskResourceUsage{
-		ResourceUsage: &cstructs.ResourceUsage{
-			MemoryStats: ms,
-			CpuStats:    cs,
-		},
-		Timestamp: s.Read.UTC().UnixNano(),
-	}
-}
-
-func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
-	numerator := newSample - oldSample
-	denom := newTotal - oldTotal
-	if numerator <= 0 || denom <= 0 {
-		return 0.0
-	}
-
-	return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
-}

drivers/docker/stats_test.go

 package docker
 
 import (
+	"runtime"
 	"testing"
 	"time"
 
@@ -25,6 +26,9 @@ func TestDriver_DockerStatsCollector(t *testing.T) {
 	stats.MemoryStats.Stats.Swap = 0
 	stats.MemoryStats.Usage = 5651904
 	stats.MemoryStats.MaxUsage = 6651904
+	stats.MemoryStats.Commit = 123231
+	stats.MemoryStats.CommitPeak = 321323
+	stats.MemoryStats.PrivateWorkingSet = 62222
 
 	go dockerStatsCollector(dst, src, time.Second)
 
@@ -36,13 +40,22 @@ func TestDriver_DockerStatsCollector(t *testing.T) {
 
 	select {
 	case ru := <-dst:
-		require.Equal(stats.MemoryStats.Stats.Rss, ru.ResourceUsage.MemoryStats.RSS)
-		require.Equal(stats.MemoryStats.Stats.Cache, ru.ResourceUsage.MemoryStats.Cache)
-		require.Equal(stats.MemoryStats.Stats.Swap, ru.ResourceUsage.MemoryStats.Swap)
-		require.Equal(stats.MemoryStats.Usage, ru.ResourceUsage.MemoryStats.Usage)
-		require.Equal(stats.MemoryStats.MaxUsage, ru.ResourceUsage.MemoryStats.MaxUsage)
-		require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods)
-		require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime)
+		if runtime.GOOS != "windows" {
+			require.Equal(stats.MemoryStats.Stats.Rss, ru.ResourceUsage.MemoryStats.RSS)
+			require.Equal(stats.MemoryStats.Stats.Cache, ru.ResourceUsage.MemoryStats.Cache)
+			require.Equal(stats.MemoryStats.Stats.Swap, ru.ResourceUsage.MemoryStats.Swap)
+			require.Equal(stats.MemoryStats.Usage, ru.ResourceUsage.MemoryStats.Usage)
+			require.Equal(stats.MemoryStats.MaxUsage, ru.ResourceUsage.MemoryStats.MaxUsage)
+			require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods)
+			require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime)
+		} else {
+			require.Equal(stats.MemoryStats.PrivateWorkingSet, ru.ResourceUsage.MemoryStats.RSS)
+			require.Equal(stats.MemoryStats.Commit, ru.ResourceUsage.MemoryStats.Usage)
+			require.Equal(stats.MemoryStats.CommitPeak, ru.ResourceUsage.MemoryStats.MaxUsage)
+			require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods)
+			require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime)
+
+		}
 	case <-time.After(time.Second):
 		require.Fail("receiving stats should not block here")
 	}

drivers/docker/util/stats_posix.go

+// +build !windows
+
+package util
+
+import (
+	"runtime"
+
+	docker "github.com/fsouza/go-dockerclient"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/helper/stats"
+)
+
+var (
+	DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
+	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"}
+)
+
+func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage {
+	ms := &cstructs.MemoryStats{
+		RSS:      s.MemoryStats.Stats.Rss,
+		Cache:    s.MemoryStats.Stats.Cache,
+		Swap:     s.MemoryStats.Stats.Swap,
+		Usage:    s.MemoryStats.Usage,
+		MaxUsage: s.MemoryStats.MaxUsage,
+		Measured: DockerMeasuredMemStats,
+	}
+
+	cs := &cstructs.CpuStats{
+		ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
+		ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
+		Measured:         DockerMeasuredCPUStats,
+	}
+
+	// Calculate percentage
+	cs.Percent = CalculateCPUPercent(
+		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
+		s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, runtime.NumCPU())
+	cs.SystemMode = CalculateCPUPercent(
+		s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
+		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
+	cs.UserMode = CalculateCPUPercent(
+		s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
+		s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU())
+	cs.TotalTicks = (cs.Percent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU())
+
+	return &cstructs.TaskResourceUsage{
+		ResourceUsage: &cstructs.ResourceUsage{
+			MemoryStats: ms,
+			CpuStats:    cs,
+		},
+		Timestamp: s.Read.UTC().UnixNano(),
+	}
+}

drivers/docker/util/stats_windows.go

+package util
+
+import (
+	"runtime"
+
+	docker "github.com/fsouza/go-dockerclient"
+	cstructs "github.com/hashicorp/nomad/client/structs"
+	"github.com/hashicorp/nomad/helper/stats"
+)
+
+var (
+	// The statistics the Docker driver exposes
+	DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
+	DockerMeasuredMemStats = []string{"RSS", "Usage", "Max Usage"}
+)
+
+func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage {
+	ms := &cstructs.MemoryStats{
+		RSS:      s.MemoryStats.PrivateWorkingSet,
+		Usage:    s.MemoryStats.Commit,
+		MaxUsage: s.MemoryStats.CommitPeak,
+		Measured: DockerMeasuredMemStats,
+	}
+
+	cpuPercent := 0.0
+
+	// https://github.com/moby/moby/blob/cbb885b07af59225eef12a8159e70d1485616d57/integration-cli/docker_api_stats_test.go#L47-L58
+	// Max number of 100ns intervals between the previous time read and now
+	possIntervals := uint64(s.Read.Sub(s.PreRead).Nanoseconds()) // Start with number of ns intervals
+	possIntervals /= 100                                         // Convert to number of 100ns intervals
+	possIntervals *= uint64(s.NumProcs)                          // Multiple by the number of processors
+
+	// Intervals used
+	intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - s.PreCPUStats.CPUUsage.TotalUsage
+
+	// Percentage avoiding divide-by-zero
+	if possIntervals > 0 {
+		cpuPercent = float64(intervalsUsed) / float64(possIntervals) * 100.0
+	}
+
+	cs := &cstructs.CpuStats{
+		ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
+		ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
+		Percent:          cpuPercent,
+		TotalTicks:       (cpuPercent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU()),
+		Measured:         DockerMeasuredCPUStats,
+	}
+
+	return &cstructs.TaskResourceUsage{
+		ResourceUsage: &cstructs.ResourceUsage{
+			MemoryStats: ms,
+			CpuStats:    cs,
+		},
+		Timestamp: s.Read.UTC().UnixNano(),
+	}
+}

drivers/docker/util/util.go

+package util
+
+func CalculateCPUPercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
+	numerator := newSample - oldSample
+	denom := newTotal - oldTotal
+	if numerator <= 0 || denom <= 0 {
+		return 0.0
+	}
+
+	return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
+}