POC: support agent auto-upgrade

Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>
start sending the actual configured stacks
2022-02-25 13:25:46 +10:00 · 2022-02-21 14:23:52 +10:00 · 2022-02-14 17:33:02 +10:00 · 2022-02-14 17:32:42 +10:00 · 2022-02-14 17:22:39 +10:00 · 2022-02-11 13:11:35 +10:00
21 changed files with 764 additions and 38 deletions
--- a/api/cli/cli.go
+++ b/api/cli/cli.go
@@ -47,6 +47,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		HTTPDisabled:              kingpin.Flag("http-disabled", "Serve portainer only on https").Default(defaultHTTPDisabled).Bool(),
 		HTTPEnabled:               kingpin.Flag("http-enabled", "Serve portainer on http").Default(defaultHTTPEnabled).Bool(),
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
+		SSLCacert:                 kingpin.Flag("sslcacert", "Path to the SSL CA certificate used to validate the edge agent cert").String(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
 		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
--- a/api/cli/defaults.go
+++ b/api/cli/defaults.go
@@ -18,6 +18,7 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
+	defaultSSLCacertPath       = "/certs/portainer-ca.crt"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"
 	defaultBaseURL             = "/"
--- a/api/cli/defaults_windows.go
+++ b/api/cli/defaults_windows.go
@@ -15,6 +15,7 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
+	defaultSSLCacertPath       = "C:\\certs\\portainer-ca.crt"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
 	defaultSnapshotInterval    = "5m"
--- a/api/cmd/portainer/log.go
+++ b/api/cmd/portainer/log.go
@@ -1,9 +1,7 @@
 package main

 import (
-	"fmt"
 	"log"
-	"strings"

 	"github.com/sirupsen/logrus"
 )
@@ -12,30 +10,14 @@ type portainerFormatter struct {
 	logrus.TextFormatter
 }

-func (f *portainerFormatter) Format(entry *logrus.Entry) ([]byte, error) {
-	var levelColor int
-	switch entry.Level {
-	case logrus.DebugLevel, logrus.TraceLevel:
-		levelColor = 31 // gray
-	case logrus.WarnLevel:
-		levelColor = 33 // yellow
-	case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
-		levelColor = 31 // red
-	default:
-		levelColor = 36 // blue
-	}
-	return []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m %s %s\n", levelColor, strings.ToUpper(entry.Level.String()), entry.Time.Format(f.TimestampFormat), entry.Message)), nil
-}
-
 func configureLogger() {
 	logger := logrus.New() // logger is to implicitly substitute stdlib's log
 	log.SetOutput(logger.Writer())

-	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
-	formatterLogrus := &portainerFormatter{logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true, TimestampFormat: "2006/01/02 15:04:05", FullTimestamp: true}}
+	formatter := &logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true}

 	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatterLogrus)
+	logrus.SetFormatter(formatter)

 	logger.SetLevel(logrus.DebugLevel)
 	logrus.SetLevel(logrus.DebugLevel)
--- a/api/cmd/portainer/main.go
+++ b/api/cmd/portainer/main.go
@@ -208,7 +208,7 @@ func initGitService() portainer.GitService {
 	return git.NewService()
 }

-func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
+func initSSLService(addr, dataPath, certPath, keyPath, cacertPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
 	slices := strings.Split(addr, ":")
 	host := slices[0]
 	if host == "" {
@@ -217,7 +217,7 @@ func initSSLService(addr, dataPath, certPath, keyPath string, fileService portai

 	sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)

-	err := sslService.Init(host, certPath, keyPath)
+	err := sslService.Init(host, certPath, keyPath, cacertPath)
 	if err != nil {
 		return nil, err
 	}
@@ -568,7 +568,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	cryptoService := initCryptoService()
 	digitalSignatureService := initDigitalSignatureService()

-	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
+	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, *flags.SSLCacert, fileService, dataStore, shutdownTrigger)
 	if err != nil {
 		logrus.Fatal(err)
 	}
--- a/api/filesystem/filesystem.go
+++ b/api/filesystem/filesystem.go
@@ -60,6 +60,8 @@ const (
 	DefaultSSLCertFilename = "cert.pem"
 	// DefaultSSLKeyFilename represents the default ssl key file name
 	DefaultSSLKeyFilename = "key.pem"
+	// DefaultSSLCacertFilename represents the default CA ssl certificate file name for mTLS
+	DefaultSSLCacertFilename = "ca-cert.pem"
 )

 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -161,7 +163,7 @@ func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExi
 	}

 	if !exists {
-		return errors.New("File doesn't exist")
+		return errors.New(fmt.Sprintf("File (%s) doesn't exist", fromFilePath))
 	}

 	finput, err := os.Open(fromFilePath)
@@ -352,6 +354,9 @@ func (service *Service) DeleteTLSFile(folder string, fileType portainer.TLSFileT

 // GetFileContent returns the content of a file as bytes.
 func (service *Service) GetFileContent(trustedRoot, filePath string) ([]byte, error) {
+	if trustedRoot == "" {
+		trustedRoot = "/"
+	}
 	content, err := os.ReadFile(JoinPaths(trustedRoot, filePath))
 	if err != nil {
 		if filePath == "" {
@@ -627,6 +632,25 @@ func (service *Service) CopySSLCertPair(certPath, keyPath string) (string, strin
 	return defCertPath, defKeyPath, nil
 }

+// GetDefaultSSLCacertsPath returns the ssl cacert path
+func (service *Service) GetDefaultSSLCacertsPath() string {
+	cacertPath := JoinPaths(SSLCertPath, DefaultSSLCacertFilename)
+
+	return service.wrapFileStore(cacertPath)
+}
+
+// CopySSLCacert copies the specified cacert pem file
+func (service *Service) CopySSLCacert(cacertPath string) (string, error) {
+	defCacertPath := service.GetDefaultSSLCacertsPath()
+
+	err := service.Copy(cacertPath, defCacertPath, true)
+	if err != nil {
+		return "", err
+	}
+
+	return defCacertPath, nil
+}
+
 // FileExists checks for the existence of the specified file.
 func FileExists(filePath string) (bool, error) {
 	if _, err := os.Stat(filePath); err != nil {
--- a/api/go.mod
+++ b/api/go.mod
@@ -19,10 +19,12 @@ require (
 	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-ldap/ldap/v3 v3.1.8
 	github.com/gofrs/uuid v4.0.0+incompatible
+	github.com/google/gops v0.3.22
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/websocket v1.4.2
+	github.com/hashicorp/go-version v1.4.0
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/joho/godotenv v1.3.0
 	github.com/jpillora/chisel v0.0.0-20190724232113-f3a8df20e389
@@ -34,10 +36,12 @@ require (
 	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
 	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
 	github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f
+	github.com/prometheus/client_golang v1.7.1
 	github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
+	github.com/swaggo/swag v1.7.8
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -50,6 +54,9 @@ require (
 )

 require (
+	github.com/KyleBanks/depth v1.2.1 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
 	github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 // indirect
@@ -57,6 +64,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 // indirect
 	github.com/aws/smithy-go v1.9.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/containerd/containerd v1.5.7 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
@@ -70,6 +79,10 @@ require (
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.1.0 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/spec v0.20.4 // indirect
+	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
@@ -79,10 +92,13 @@ require (
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669 // indirect
 	github.com/jpillora/requestlog v0.0.0-20181015073026-df8817be5f82 // indirect
 	github.com/jpillora/sizestr v0.0.0-20160130011556-e2ea2fa42fb9 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
@@ -93,6 +109,9 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect
@@ -101,11 +120,12 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/sys v0.0.0-20210902050250-f475640dd07b // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/tools v0.1.7 // indirect
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
 	google.golang.org/grpc v1.33.2 // indirect
--- a/api/go.sum
+++ b/api/go.sum
@@ -41,6 +41,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
+github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
@@ -62,9 +64,14 @@ github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+github.com/agiledragon/gomonkey/v2 v2.3.1 h1:k+UnUY0EMNYUFUAQVETGY9uUTxjMdnUkP0ARyJS1zzs=
+github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -105,6 +112,7 @@ github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
@@ -120,7 +128,9 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8n
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
@@ -334,13 +344,23 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
+github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
+github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
@@ -404,6 +424,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gops v0.3.22 h1:lyvhDxfPLHAOR2xIYwjPhN387qHxyU21Sk9sz/GhmhQ=
+github.com/google/gops v0.3.22/go.mod h1:7diIdLsqpCihPSX3fQagksT/Ku/y4RL9LHTlKyEUDl8=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -446,6 +468,8 @@ github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FK
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4=
+github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
@@ -473,6 +497,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669 h1:l5rH/CnVVu+HPxjtxjM90nHrm4nov3j3RF9/62UjgLs=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669/go.mod h1:kOeLNvjNBGSV3uYtFjvb72+fnZCMFJF1XDvRIjdom0g=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
@@ -493,6 +519,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -518,12 +545,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
@@ -603,6 +633,12 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
+github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -627,17 +663,20 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -649,6 +688,7 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777 h1:rDj3WeO+TiWyxfcydUnKegWAZoR5kQsnW0wzhggdOrw=
@@ -663,6 +703,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -675,6 +716,7 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -705,10 +747,14 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/swaggo/swag v1.7.8 h1:w249t0l/kc/DKMGlS0fppNJQxKyJ8heNaUWB6nsH3zc=
+github.com/swaggo/swag v1.7.8/go.mod h1:gZ+TJ2w/Ve1RwQsA2IRoSOTidHz6DX+PIG8GWvbnoLU=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
@@ -718,6 +764,7 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
@@ -738,9 +785,11 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
@@ -804,6 +853,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -841,8 +892,10 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
-golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE=
+golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -922,10 +975,14 @@ golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -936,8 +993,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -982,6 +1040,8 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
+golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1085,6 +1145,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1144,6 +1205,7 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
--- a/api/http/handler/endpointedge/async.go
+++ b/api/http/handler/endpointedge/async.go
@@ -0,0 +1,393 @@
+package endpointedge
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	version "github.com/hashicorp/go-version"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/sirupsen/logrus"
+)
+
+// https://datatracker.ietf.org/doc/html/rfc6902
+// Doing this manually, because at this point, i don't want to marshal to json, make a diff - for now, just using 'add' (as its really an upsert)
+type JSONPatch struct {
+	Operation string      `json:"op"`
+	Path      string      `json:"path"`
+	Value     interface{} `json:"value"`
+}
+
+// TODO: copied from edgestack_status_update
+type updateStatusPayload struct {
+	Error      string
+	Status     *portainer.EdgeStackStatusType
+	EndpointID *portainer.EndpointID
+}
+type edgeJobResponse struct {
+	// EdgeJob Identifier
+	ID portainer.EdgeJobID `json:"Id" example:"2"`
+	// Whether to collect logs
+	CollectLogs bool `json:"CollectLogs" example:"true"`
+	// A cron expression to schedule this job
+	CronExpression string `json:"CronExpression" example:"* * * * *"`
+	// Script to run
+	Script string `json:"Script" example:"echo hello"`
+	// Version of this EdgeJob
+	Version int `json:"Version" example:"2"`
+}
+
+// An empty request ~~ just a ping.
+type Snapshot struct {
+	Docker     *portainer.DockerSnapshot
+	Kubernetes *portainer.KubernetesSnapshot
+}
+type AsyncRequest struct {
+	CommandId   string    `json: optional`
+	Snapshot    *Snapshot `json: optional` // todo
+	StackStatus map[portainer.EdgeStackID]updateStatusPayload
+}
+
+func (payload *AsyncRequest) Validate(r *http.Request) error {
+	// TODO:
+
+	return nil
+}
+
+type AsyncResponse struct {
+	CommandInterval  time.Duration `json: optional`
+	PingInterval     time.Duration `json: optional`
+	SnapshotInterval time.Duration `json: optional`
+
+	ServerCommandId      string      // should be easy to detect if its larger / smaller:  this is the response that tells the agent there are new commands waiting for it
+	SendDiffSnapshotTime time.Time   `json: optional` // might be optional
+	Commands             []JSONPatch `json: optional` // todo
+	Status               string      // give the agent some idea if the server thinks its OK, or if it should STOP
+}
+
+// Yup, this should be environment specific, and not global
+var lastcheckinStatusMutex sync.Mutex
+
+// for testing with mTLS..:
+//sven@p1:~/src/portainer/portainer$ curl -k --cacert ~/.config/portainer/certs/ca.pem --cert ~/.config/portainer/certs/agent-cert.pem --key ~/.config/portainer/certs/agent-key.pem -X POST --header "X-PortainerAgent-EdgeID: 7e2b0143-c511-43c3-844c-a7a91ab0bedc" --data '{"CommandId": "okok", "Snapshot": {}}'  https://p1:9443/api/endpoints/edge/async/
+//{"CommandInterval":0,"PingInterval":0,"SnapshotInterval":0,"ServerCommandId":"8888","SendDiffSnapshotTime":"0001-01-01T00:00:00Z","Commands":{}}
+
+// @id endpointAsync
+// @summary Get environment(endpoint) status
+// @description Environment(Endpoint) for edge agent to check status of environment(endpoint)
+// @description **Access policy**: restricted only to Edge environments(endpoints) TODO: with mTLS cert....
+// @tags endpoints
+// @security ApiKeyAuth
+// @security jwt
+// @param id path int true "Environment(Endpoint) identifier"
+// @success 200 {object} AsyncResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied to access environment(endpoint)"
+// @failure 404 "Environment(Endpoint) not found"
+// @failure 500 "Server error"
+// @router /endpoints/edge/async/ [post]
+func (handler *Handler) endpointAsync(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	// TODO: get endpointID from the mTLS cert info
+	edgeIdentifier := r.Header.Get(portainer.PortainerAgentEdgeIDHeader)
+	if edgeIdentifier == "" {
+		logrus.WithField("portainer.PortainerAgentEdgeIDHeader", edgeIdentifier).Debug("missing agent edge id")
+
+		return &httperror.HandlerError{http.StatusInternalServerError, "missing Edge identifier", errors.New("missing Edge identifier")}
+	}
+
+	// TODO: if the mTLS certs are valid, and we don't have a matching environment registered, CREATE IT (and maybe untrusted...)
+	endpoint, err := handler.getEdgeEndpoint(edgeIdentifier)
+	if err != nil {
+		// TODO: if its a valid cert, or the user hasn't limited to mTLS / portainer set id, the
+		// create new untrusted environment
+		// portainer.HTTPResponseAgentPlatform tells us what platform it is too...
+		logrus.WithField("portainer.PortainerAgentEdgeIDHeader", edgeIdentifier).WithField("Agent Addr", r.RemoteAddr).Debug("edge id not found in existing endpoints!")
+	}
+	// if agent mTLS is on, drop the connection if the client cert isn't CA'd (or if its revoked)
+	err = handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
+	}
+
+	// TODO: an assumption that needs testing, is that using the right EdgeId means we're also talking to the same DOckerd / Kube cluster.
+	//       I suspect that without getting the Dockerd UUID, or the "by convention" kube uuid, we can't know if someone's re-used info which then would cause some kind of state flapping.
+	//		Don't ask me what happens in non-async mode if you have more than one agent running - surely that would make the tunnel go boom?
+	requestLogger := logrus.
+		WithField("Agent Addr", r.RemoteAddr).
+		WithField("Agent Version", r.Header.Get(portainer.HTTPAgentVersionHeaderName)).
+		WithField("Agent PID", r.Header.Get(portainer.HTTPAgentPIDName)).
+		WithField("Agent EdgeID", r.Header.Get(portainer.PortainerAgentEdgeIDHeader)) //.
+		//WithField("Agent UniqueID", r.Header.Get(portainer.HTTPAgentUUIDHeaderName))
+
+	// Any request we can identify as coming from a valid agent is treated as a Ping
+	endpoint.LastCheckInDate = time.Now().Unix()
+	endpoint.Status = portainer.EndpointStatusUp
+
+	// TODO: update endpoint contact time
+	lastcheckinStatusMutex.Lock()
+	if endpoint.AgentHistory == nil {
+		endpoint.AgentHistory = make(map[string]portainer.AgentInfo)
+	}
+	info, ok := endpoint.AgentHistory[r.RemoteAddr]
+	if !ok {
+		info = portainer.AgentInfo{
+			LastCheckInDate: endpoint.LastCheckInDate,
+			Version:         r.Header.Get(portainer.HTTPAgentVersionHeaderName),
+			RemoteAddr:      r.RemoteAddr,
+			Status:          "OK",
+		}
+	}
+	info.CheckInCount = info.CheckInCount + 1
+	info.Status = "OK"
+	info.LastCheckInDate = endpoint.LastCheckInDate
+	requestLogger.Debugf("Checkin count = %d", info.CheckInCount)
+
+	endpoint.AgentHistory[r.RemoteAddr] = info
+
+	// Determine if there's more than one active agent, and if so, tell them to STOP
+	okCount := 0
+	infoVersion, _ := version.NewVersion(info.Version)
+	for key, val := range endpoint.AgentHistory {
+		timeToLastCheckIn := time.Second * time.Duration(endpoint.LastCheckInDate-val.LastCheckInDate)
+		// Timeout for last best agent (currently based on version number)
+		if timeToLastCheckIn > time.Second*time.Duration(endpoint.EdgeCheckinInterval*100) {
+			delete(endpoint.AgentHistory, key)
+			continue
+		}
+		if timeToLastCheckIn > time.Second*time.Duration(endpoint.EdgeCheckinInterval*10) {
+			val.Status = "GONE"
+			endpoint.AgentHistory[key] = val
+			continue
+		}
+		if timeToLastCheckIn > time.Second*time.Duration(endpoint.EdgeCheckinInterval*2) {
+			val.Status = "TROUBLED"
+			endpoint.AgentHistory[key] = val
+			continue
+		}
+		if val.Status == "OK" {
+			// if there's a info.Version difference, choose the more up to date agent...
+			valVersion, _ := version.NewVersion(val.Version)
+			if infoVersion.GreaterThan(valVersion) {
+				val.Status = "STOP VERSION - " + info.Version
+				endpoint.AgentHistory[val.RemoteAddr] = val
+				continue
+			}
+			if valVersion.GreaterThan(infoVersion) {
+				info.Status = "STOP VERSION - " + val.Version
+				endpoint.AgentHistory[info.RemoteAddr] = info
+				info = val
+				continue
+			}
+
+			// TODO: UX question - trade off between user expectation on upgrade, vs stability at staeday state.
+			// if we have two or more otherwise just as good agents, pick the newer one, on the presumption that it was started on purpose
+			// the risk with this is the flapping you get if you gave two identical agents with --restart always - so maybe it should get tuned
+			// for eg, only use the younger one if the user has initiated an upgrade? otherwise prefer the old?
+			if val.CheckInCount < info.CheckInCount {
+				info.Status = "STOP AGE"
+				endpoint.AgentHistory[info.RemoteAddr] = info
+				info = val
+				continue
+			}
+			okCount++
+		}
+	}
+
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to Unable to persist environment changes inside the database", err}
+	}
+	lastcheckinStatusMutex.Unlock()
+
+	var payload AsyncRequest
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		// an "" request ~~ same as {}
+		requestLogger.WithError(err).WithField("payload", r).Debug("decode payload")
+	}
+
+	//if endpoint.AgentHistory[r.RemoteAddr].Status != "OK" {
+	// okCount of zero can happen - basically, we havn't timed out a newer agent's last contact, and so we're hoping it will come back, so we refuse the old version
+	// we could instead allow this, cos in some circumstances, it may be better to have more than one agent giving the user control
+	requestLogger.Debugf("Checkin STATUS = %s (okCount = %d)", endpoint.AgentHistory[r.RemoteAddr].Status, okCount)
+	//}
+
+	asyncResponse := AsyncResponse{
+		ServerCommandId: "8888", // the most current id of a new command on the server
+		Status:          endpoint.AgentHistory[r.RemoteAddr].Status,
+	}
+
+	// TODO: need a way to detect that these are changed, and send them to the agent...
+	// CommandInterval time.Duration `json: optional`
+	// PingInterval  time.Duration `json: optional`
+	// SnapshotInterval time.Duration `json: optional`
+
+	if payload.CommandId == "" && payload.Snapshot == nil {
+		// just a ping.
+		return response.JSON(w, asyncResponse)
+	}
+
+	if payload.Snapshot != nil {
+		asyncResponse.SendDiffSnapshotTime = handler.saveSnapshot(requestLogger, endpoint, payload)
+	}
+	if payload.CommandId != "" {
+		asyncResponse.Commands = handler.sendCommandsSince(requestLogger, endpoint, payload.CommandId)
+	}
+
+	return response.JSON(w, asyncResponse)
+}
+
+// TODO: yup, next step is for these to be JSONDiff's and to be rehydrated
+func (handler *Handler) saveSnapshot(requestLogger *logrus.Entry, endpoint *portainer.Endpoint, payload AsyncRequest) time.Time {
+	for stackID, status := range payload.StackStatus {
+		stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
+		// TODO: work out what we can do with the errors
+		if err == nil {
+			stack.Status[*status.EndpointID] = portainer.EdgeStackStatus{
+				Type:       *status.Status,
+				Error:      status.Error,
+				EndpointID: *status.EndpointID,
+			}
+			err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
+		}
+	}
+
+	switch endpoint.Type {
+	// case portainer.AzureEnvironment:
+	// 	return time.Now()
+	case portainer.KubernetesLocalEnvironment, portainer.AgentOnKubernetesEnvironment, portainer.EdgeAgentOnKubernetesEnvironment:
+		requestLogger.Debug("Got a Kubernetes Snapshot")
+		endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{*payload.Snapshot.Kubernetes}
+		return time.Unix(payload.Snapshot.Kubernetes.Time, 0)
+	case portainer.DockerEnvironment, portainer.AgentOnDockerEnvironment, portainer.EdgeAgentOnDockerEnvironment:
+		requestLogger.Debug("Got a Docker Snapshot")
+		endpoint.Snapshots = []portainer.DockerSnapshot{*payload.Snapshot.Docker}
+		return time.Unix(payload.Snapshot.Docker.Time, 0)
+	default:
+		return time.Time{}
+	}
+}
+
+func (handler *Handler) sendCommandsSince(requestLogger *logrus.Entry, endpoint *portainer.Endpoint, commandId string) []JSONPatch {
+	var commandList []JSONPatch
+
+	// TODO: later, figure out if it is scalable to do diff's, as it means the server needs to store what it sent to all million agents (if the database had time based versioning, this would be trivial...)
+	//       I suspect the easiest thing will be to add a "modified timestamp" to edge stacks and edge jobs, and to send them only when the modified time > requested time
+	requestLogger.WithField("endpoint", endpoint.Name).WithField("from command", commandId).Debug("Sending commands")
+
+	// schedules := []edgeJobResponse{}
+	tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)
+	for _, job := range tunnel.Jobs {
+		schedule := edgeJobResponse{
+			ID:             job.ID,
+			CronExpression: job.CronExpression,
+			CollectLogs:    job.Endpoints[endpoint.ID].CollectLogs,
+			Version:        job.Version,
+		}
+
+		file, err := handler.FileService.GetFileContent("/", job.ScriptPath)
+		if err != nil {
+			// TODO: this should maybe just skip thi job?
+			requestLogger.WithError(err).Error("Unable to retrieve Edge job script file")
+			continue
+		}
+
+		schedule.Script = base64.RawStdEncoding.EncodeToString(file)
+		cmd := JSONPatch{
+			Operation: "add",
+			Path:      fmt.Sprintf("/edgejob/%d", schedule.ID),
+			Value:     schedule,
+		}
+		commandList = append(commandList, cmd)
+	}
+
+	relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpoint.ID)
+	if err != nil {
+		requestLogger.WithError(err).Error("Unable to retrieve relation object from the database")
+		return commandList
+	}
+
+	// TODO: this is the datatype the agent uses in the end
+	type edgeStackData struct {
+		ID               portainer.EdgeStackID
+		Version          int
+		StackFileContent string
+		Name             string
+	}
+
+	for stackID := range relation.EdgeStacks {
+		stack, err := handler.DataStore.EdgeStack().EdgeStack(stackID)
+		if err != nil {
+			requestLogger.WithError(err).Error("Unable to retrieve edge stack from the database")
+			continue
+		}
+
+		edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
+		if handler.DataStore.IsErrObjectNotFound(err) {
+			requestLogger.WithError(err).Error("Unable to find an edge stack with the specified identifier inside the database")
+			continue
+		} else if err != nil {
+			requestLogger.WithError(err).Error("Unable to find an edge stack with the specified identifier inside the database")
+			continue
+		}
+
+		fileName := edgeStack.EntryPoint
+		if endpointutils.IsDockerEndpoint(endpoint) {
+			if fileName == "" {
+				requestLogger.Error("Docker is not supported by this stack")
+				continue
+			}
+		}
+
+		if endpointutils.IsKubernetesEndpoint(endpoint) {
+			fileName = edgeStack.ManifestPath
+
+			if fileName == "" {
+				requestLogger.Error("Kubernetes is not supported by this stack")
+				continue
+			}
+		}
+
+		stackFileContent, err := handler.FileService.GetFileContent(edgeStack.ProjectPath, fileName)
+		if err != nil {
+			requestLogger.WithError(err).Error("Unable to retrieve Compose file from disk")
+			continue
+		}
+
+		stackStatus := edgeStackData{
+			StackFileContent: string(stackFileContent),
+			Name:             edgeStack.Name,
+			ID:               stack.ID,
+			Version:          stack.Version,
+		}
+
+		cmd := JSONPatch{
+			Operation: "add",
+			Path:      fmt.Sprintf("/edgestack/%d", stack.ID),
+			Value:     stackStatus,
+		}
+		commandList = append(commandList, cmd)
+	}
+	return commandList
+}
+
+// TODO: this probably should be in the data layer.. (like, somewhere that depends dataservices/errors)
+func (handler *Handler) getEdgeEndpoint(edgeIdentifier string) (*portainer.Endpoint, error) {
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
+	if err != nil {
+		return nil, err
+	}
+	for _, endpoint := range endpoints {
+		if endpoint.EdgeID == edgeIdentifier {
+			return &endpoint, nil
+		}
+	}
+	return nil, err
+}
--- a/api/http/handler/endpointedge/handler.go
+++ b/api/http/handler/endpointedge/handler.go
@@ -31,5 +31,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.PublicAccess(httperror.LoggerHandler(h.endpointEdgeStackInspect))).Methods(http.MethodGet)
 	h.Handle("/{id}/edge/jobs/{jobID}/logs",
 		bouncer.PublicAccess(httperror.LoggerHandler(h.endpointEdgeJobsLogs))).Methods(http.MethodPost)
+	h.Handle("/edge/async/",
+		bouncer.PublicAccess(httperror.LoggerHandler(h.endpointAsync))).Methods(http.MethodPost)
 	return h
 }
--- a/api/http/handler/endpoints/endpoint_snapshot.go
+++ b/api/http/handler/endpoints/endpoint_snapshot.go
@@ -43,6 +43,7 @@ func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request)

 	snapshotError := handler.SnapshotService.SnapshotEndpoint(endpoint)

+	// TODO: so huh? why are we getting the endpoint a second time? if there's a reason to do this - please add that as a comment!
 	latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
 	if latestEndpointReference == nil {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
--- a/api/http/handler/endpoints/endpoint_status_inspect.go
+++ b/api/http/handler/endpoints/endpoint_status_inspect.go
@@ -11,6 +11,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/sirupsen/logrus"
 )

 type stackStatusResponse struct {
@@ -48,6 +49,7 @@ type endpointStatusInspectResponse struct {
 	Stacks []stackStatusResponse `json:"stacks"`
 }

+// TODO: first up, why is this not in ../endpointedge/???
 // @id EndpointStatusInspect
 // @summary Get environment(endpoint) status
 // @description Environment(Endpoint) for edge agent to check status of environment(endpoint)
@@ -70,6 +72,7 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req

 	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if handler.DataStore.IsErrObjectNotFound(err) {
+		logrus.WithError(err).WithField("env", endpointID).WithField("remote", r.RemoteAddr).Error("Unable to find an environment with the specified identifier inside the database")
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err}
--- a/api/http/handler/handler.go
+++ b/api/http/handler/handler.go
@@ -21,6 +21,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
 	"github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/ldap"
+	"github.com/portainer/portainer/api/http/handler/metrics"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -77,6 +78,7 @@ type Handler struct {
 	UserHandler            *users.Handler
 	WebSocketHandler       *websocket.Handler
 	WebhookHandler         *webhooks.Handler
+	MetricsHandler         *metrics.Handler
 }

 // @title PortainerCE API
@@ -241,6 +243,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.WebhookHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/storybook"):
 		http.StripPrefix("/storybook", h.StorybookHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/metrics"):
+		if h.MetricsHandler != nil {
+			h.MetricsHandler.ServeHTTP(w, r)
+		}
 	case strings.HasPrefix(r.URL.Path, "/"):
 		h.FileHandler.ServeHTTP(w, r)
 	}
--- a/api/http/handler/metrics/handler.go
+++ b/api/http/handler/metrics/handler.go
@@ -0,0 +1,33 @@
+package metrics
+
+import (
+	"github.com/google/gops/agent"
+	"github.com/gorilla/mux"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+// Handler is the HTTP handler used to handle Prometheus metrics operations.
+type Handler struct {
+	*mux.Router
+}
+
+// NewHandler creates a handler to manage settings operations.
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+	h.Handle("/metrics", promhttp.Handler())
+	//	h.Handle("/metrics", bouncer.PublicAccess(promhttp.Handler()))
+	logrus.Debugf("metricsHandler creation")
+
+	// also add gops agent support
+	if err := agent.Listen(agent.Options{}); err != nil {
+		logrus.WithError(err).Debugf("failed to start gops agent")
+	} else {
+		logrus.Debug("started gops agent")
+	}
+
+	return h
+}
--- a/api/http/security/bouncer.go
+++ b/api/http/security/bouncer.go
@@ -1,8 +1,10 @@
 package security

 import (
+	"crypto/x509"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
@@ -12,6 +14,9 @@ import (
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/dataservices"
 	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/sirupsen/logrus"
 )

 type (
@@ -37,6 +42,21 @@ type (

 const apiKeyHeader = "X-API-KEY"

+var (
+	apiProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "portainer_api_bouncer_results",
+		Help: "The total number of request access success/failures",
+	},
+		[]string{"permission", "path"},
+	)
+	agentApiProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "portainer_agent_api_bouncer_results",
+		Help: "The total number of agent request access success/failures",
+	},
+		[]string{"permission", "path"},
+	)
+)
+
 // NewRequestBouncer initializes a new RequestBouncer
 func NewRequestBouncer(dataStore dataservices.DataStore, jwtService dataservices.JWTService, apiKeyService apikey.APIKeyService) *RequestBouncer {
 	return &RequestBouncer{
@@ -96,58 +116,111 @@ func (bouncer *RequestBouncer) AuthenticatedAccess(h http.Handler) http.Handler
 func (bouncer *RequestBouncer) AuthorizedEndpointOperation(r *http.Request, endpoint *portainer.Endpoint) error {
 	tokenData, err := RetrieveTokenData(r)
 	if err != nil {
+		apiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "error"}).Inc()
 		return err
 	}

 	if tokenData.Role == portainer.AdministratorRole {
+		apiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "admin"}).Inc()
 		return nil
 	}

 	memberships, err := bouncer.dataStore.TeamMembership().TeamMembershipsByUserID(tokenData.ID)
 	if err != nil {
+		apiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "error"}).Inc()
 		return err
 	}

 	group, err := bouncer.dataStore.EndpointGroup().EndpointGroup(endpoint.GroupID)
 	if err != nil {
+		apiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "error"}).Inc()
 		return err
 	}

 	if !authorizedEndpointAccess(endpoint, group, tokenData.ID, memberships) {
+		apiProcessed.With(prometheus.Labels{"permission": "denied"}).Inc()
 		return httperrors.ErrEndpointAccessDenied
 	}

+	apiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "ok"}).Inc()
 	return nil
 }

 // AuthorizedEdgeEndpointOperation verifies that the request was received from a valid Edge environment(endpoint)
 func (bouncer *RequestBouncer) AuthorizedEdgeEndpointOperation(r *http.Request, endpoint *portainer.Endpoint) error {
+	// tls.RequireAndVerifyClientCert would be nice, but that would require the same certs for browser and api use
+	sslsettings, _ := bouncer.dataStore.SSLSettings().Settings()
+	if sslsettings.CacertPath != "" {
+		// if a caCert is set, then reject any requests that don't have a client Auth cert signed with it
+		if len(r.TLS.PeerCertificates) == 0 {
+			logrus.Error("No clientAuth Agent certificate offered")
+			return errors.New("No clientAuth Agent certificate offered")
+		}
+
+		caChainIdx := len(r.TLS.VerifiedChains)
+		chainCaCert := r.TLS.VerifiedChains[0][caChainIdx]
+
+		caCert, _ := ioutil.ReadFile(sslsettings.CacertPath)
+		certPool := x509.NewCertPool()
+		certPool.AppendCertsFromPEM(caCert)
+
+		logrus.
+			WithField("chain Subject", chainCaCert.Subject.String()).
+			WithField("tls DNSNames", chainCaCert.DNSNames).
+			WithField("Agent Addr", r.RemoteAddr).
+			WithField("Agent Version", r.Header.Get(portainer.HTTPAgentVersionHeaderName)).
+			WithField("Agent PID", r.Header.Get(portainer.HTTPAgentPIDName)).
+			WithField("Agent EdgeID", r.Header.Get(portainer.PortainerAgentEdgeIDHeader)).
+			Debugf("TLS client chain")
+
+		opts := x509.VerifyOptions{
+			//DNSName: name,	// Not normally used on server side - important on the client side
+			Roots:     certPool, // as used in ListenAndServe
+			KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		}
+		remoteCert := r.TLS.PeerCertificates[0]
+
+		if _, err := remoteCert.Verify(opts); err != nil {
+			logrus.WithError(err).Error("Agent certificate not signed by the CACert")
+			return errors.New("Agent certificate wasn't signed by required CA Cert")
+		}
+
+		// TODO: test revoke cert list.
+	}
+
 	if endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment && endpoint.Type != portainer.EdgeAgentOnDockerEnvironment {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_error"}).Inc()
 		return errors.New("Invalid environment type")
 	}

 	edgeIdentifier := r.Header.Get(portainer.PortainerAgentEdgeIDHeader)
 	if edgeIdentifier == "" {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_noiderror"}).Inc()
 		return errors.New("missing Edge identifier")
 	}

 	if endpoint.EdgeID != "" && endpoint.EdgeID != edgeIdentifier {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_iderror"}).Inc()
 		return errors.New("invalid Edge identifier")
 	}

 	if endpoint.LastCheckInDate > 0 || endpoint.UserTrusted {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_ok"}).Inc()
 		return nil
 	}

 	settings, err := bouncer.dataStore.Settings().Settings()
 	if err != nil {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_error"}).Inc()
 		return fmt.Errorf("could not retrieve the settings: %w", err)
 	}

 	if settings.DisableTrustOnFirstConnect {
+		agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_untrusted"}).Inc()
 		return errors.New("the device has not been trusted yet")
 	}

+	agentApiProcessed.With(prometheus.Labels{"path": r.RequestURI, "permission": "edge_ok"}).Inc()
 	return nil
 }

--- a/api/http/server.go
+++ b/api/http/server.go
@@ -3,6 +3,7 @@ package http
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"log"
 	"net/http"
@@ -34,6 +35,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
 	kubehandler "github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/ldap"
+	"github.com/portainer/portainer/api/http/handler/metrics"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -61,6 +63,7 @@ import (
 	"github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/scheduler"
 	stackdeployer "github.com/portainer/portainer/api/stacks"
+	"github.com/sirupsen/logrus"
 )

 // Server implements the portainer.Server interface
@@ -263,6 +266,11 @@ func (server *Server) Start() error {
 	webhookHandler.DataStore = server.DataStore
 	webhookHandler.DockerClientFactory = server.DockerClientFactory

+	var metricsHandler *metrics.Handler
+	if server.DataStore.Settings().IsFeatureFlagEnabled("dev-metrics") {
+		metricsHandler = metrics.NewHandler(requestBouncer)
+	}
+
 	server.Handler = &handler.Handler{
 		RoleHandler:            roleHandler,
 		AuthHandler:            authHandler,
@@ -299,6 +307,7 @@ func (server *Server) Start() error {
 		UserHandler:            userHandler,
 		WebSocketHandler:       websocketHandler,
 		WebhookHandler:         webhookHandler,
+		MetricsHandler:         metricsHandler,
 	}

 	handler := offlineGate.WaitingMiddleware(time.Minute, server.Handler)
@@ -330,6 +339,17 @@ func (server *Server) Start() error {
 		return server.SSLService.GetRawCertificate(), nil
 	}

+	if caCert := server.SSLService.GetCacertificatePem(); len(caCert) > 0 {
+		logrus.Debugf("using CA certificate for %s", server.BindAddressHTTPS)
+		certPool := x509.NewCertPool()
+		certPool.AppendCertsFromPEM(caCert)
+
+		httpsServer.TLSConfig.ClientCAs = certPool
+		// can't use tls.RequireAndVerifyClientCert, and this port is also used for the browser (though it would be a strong feature to allow the user to enable)
+		httpsServer.TLSConfig.ClientAuth = tls.VerifyClientCertIfGiven
+		httpsServer.TLSConfig.BuildNameToCertificate()
+	}
+
 	go shutdown(server.ShutdownCtx, httpsServer)
 	return httpsServer.ListenAndServeTLS("", "")
 }
--- a/api/internal/ssl/ssl.go
+++ b/api/internal/ssl/ssl.go
@@ -3,6 +3,7 @@ package ssl
 import (
 	"context"
 	"crypto/tls"
+	"io/ioutil"
 	"log"
 	"os"
 	"time"
@@ -31,7 +32,7 @@ func NewService(fileService portainer.FileService, dataStore dataservices.DataSt
 }

 // Init initializes the service
-func (service *Service) Init(host, certPath, keyPath string) error {
+func (service *Service) Init(host, certPath, keyPath, cacertPath string) error {
 	pathSupplied := certPath != "" && keyPath != ""
 	if pathSupplied {
 		newCertPath, newKeyPath, err := service.fileService.CopySSLCertPair(certPath, keyPath)
@@ -39,7 +40,19 @@ func (service *Service) Init(host, certPath, keyPath string) error {
 			return errors.Wrap(err, "failed copying supplied certs")
 		}

-		return service.cacheInfo(newCertPath, newKeyPath, false)
+		newCacertPath := ""
+		if cacertPath != "" {
+			newCacertPath, err = service.fileService.CopySSLCacert(cacertPath)
+			if err != nil {
+				return errors.Wrap(err, "failed copying supplied cacert")
+			}
+		}
+
+		return service.cacheInfo(newCertPath, newKeyPath, newCacertPath, false)
+	}
+	if cacertPath != "" {
+		return errors.Errorf("supplying a CA cert path (%s) requires an SSL cert and key file", cacertPath)
+
 	}

 	settings, err := service.GetSSLSettings()
@@ -68,10 +81,24 @@ func (service *Service) Init(host, certPath, keyPath string) error {
 		return errors.Wrap(err, "failed generating self signed certs")
 	}

-	return service.cacheInfo(certPath, keyPath, true)
+	return service.cacheInfo(certPath, keyPath, "", true)

 }

+// GetRawCertificate gets the raw certificate
+func (service *Service) GetCacertificatePem() (pemData []byte) {
+	settings, _ := service.GetSSLSettings()
+	if settings.CacertPath == "" {
+		return pemData
+	}
+	caCert, err := ioutil.ReadFile(settings.CacertPath)
+	if err != nil {
+		log.Printf("reading ca cert: %s", err)
+		return pemData
+	}
+	return caCert
+}
+
 // GetRawCertificate gets the raw certificate
 func (service *Service) GetRawCertificate() *tls.Certificate {
 	return service.rawCert
@@ -98,7 +125,13 @@ func (service *Service) SetCertificates(certData, keyData []byte) error {
 		return err
 	}

-	service.cacheInfo(certPath, keyPath, false)
+	settings, err := service.dataStore.SSLSettings().Settings()
+	if err != nil {
+		return err
+	}
+	// Don't unset the settings.CacertPath when uploading a new cert from the UI
+	// TODO: should also add UI to update thecacert, or to disable it..
+	service.cacheInfo(certPath, keyPath, settings.CacertPath, false)

 	service.shutdownTrigger()

@@ -127,6 +160,7 @@ func (service *Service) SetHTTPEnabled(httpEnabled bool) error {
 	return nil
 }

+//TODO: why is this being cached in memory? is it actually loaded more than once?
 func (service *Service) cacheCertificate(certPath, keyPath string) error {
 	rawCert, err := tls.LoadX509KeyPair(certPath, keyPath)
 	if err != nil {
@@ -138,7 +172,7 @@ func (service *Service) cacheCertificate(certPath, keyPath string) error {
 	return nil
 }

-func (service *Service) cacheInfo(certPath, keyPath string, selfSigned bool) error {
+func (service *Service) cacheInfo(certPath, keyPath, cacertPath string, selfSigned bool) error {
 	err := service.cacheCertificate(certPath, keyPath)
 	if err != nil {
 		return err
@@ -151,6 +185,7 @@ func (service *Service) cacheInfo(certPath, keyPath string, selfSigned bool) err

 	settings.CertPath = certPath
 	settings.KeyPath = keyPath
+	settings.CacertPath = cacertPath
 	settings.SelfSigned = selfSigned

 	err = service.dataStore.SSLSettings().UpdateSettings(settings)
--- a/api/portainer.go
+++ b/api/portainer.go
@@ -116,6 +116,7 @@ type (
 		HTTPDisabled              *bool
 		HTTPEnabled               *bool
 		SSL                       *bool
+		SSLCacert                 *string
 		SSLCert                   *string
 		SSLKey                    *string
 		Rollback                  *bool
@@ -343,6 +344,17 @@ type (

 		// Deprecated in DBVersion == 22
 		Tags []string `json:"Tags"`
+
+		AgentHistory map[string]AgentInfo
+	}
+
+	AgentInfo struct {
+		LastCheckInDate int64
+		CheckInCount    int64
+		Version         string
+		RemoteAddr      string
+		Status          string
+		//UniqueId        string
 	}

 	// EndpointAuthorizations represents the authorizations associated to a set of environments(endpoints)
@@ -837,6 +849,7 @@ type (
 	SSLSettings struct {
 		CertPath    string `json:"certPath"`
 		KeyPath     string `json:"keyPath"`
+		CacertPath  string `json:"cacertPath"`
 		SelfSigned  bool   `json:"selfSigned"`
 		HTTPEnabled bool   `json:"httpEnabled"`
 	}
@@ -1236,6 +1249,7 @@ type (
 		GetDefaultSSLCertsPath() (string, string)
 		StoreSSLCertPair(cert, key []byte) (string, string, error)
 		CopySSLCertPair(certPath, keyPath string) (string, string, error)
+		CopySSLCacert(cacertPath string) (string, error)
 		StoreFDOProfileFileFromBytes(fdoProfileIdentifier string, data []byte) (string, error)
 	}

@@ -1363,6 +1377,10 @@ const (
 	PortainerAgentKubernetesSATokenHeader = "X-PortainerAgent-SA-Token"
 	// PortainerAgentSignatureMessage represents the message used to create a digital signature
 	// to be used when communicating with an agent
+	HTTPAgentVersionHeaderName = "X-Portainer-Agent-Version"
+	HTTPAgentPIDName           = "X-Portainer-Process-Id"
+	HTTPAgentUUIDHeaderName    = "X-Portainer-Agent-UUID"
+
 	PortainerAgentSignatureMessage = "Portainer-App"
 	// DefaultSnapshotInterval represents the default interval between each environment snapshot job
 	DefaultSnapshotInterval = "5m"
@@ -1383,7 +1401,7 @@ const (
 )

 // List of supported features
-var SupportedFeatureFlags = []Feature{}
+var SupportedFeatureFlags = []Feature{"dev-metrics"}

 const (
 	_ AuthenticationMethod = iota
--- a/build/mtlscerts.sh
+++ b/build/mtlscerts.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# very much just copied from https://lemariva.com/blog/2019/12/portainer-managing-docker-engine-remotely
+# production use should involve a real external certificate management system
+
+export HOST=portainer.p1.alho.st
+export CERTDIR=~/.config/portainer/certs/
+
+mkdir -p ${CERTDIR}
+cd ${CERTDIR}
+echo "Generating example mTLS certs into $(pwd)"
+
+if [[ ! -f "ca.pem" ]]; then
+    echo "Generate the CA Cert"
+    openssl genrsa -aes256 -out ca-key.pem 4096
+    # enter a pass phrase to protect the ca-key
+
+    openssl req -new -x509 -days 365 -key ca-key.pem -sha256 -out ca.pem
+else
+    echo "ca.pem ca cert already exists"
+fi
+
+if [[ ! -f "server-cert.pem" ]]; then
+    echo "Generate the Portainer server cert"
+    openssl genrsa -out server-key.pem 4096
+
+    openssl req -subj "/CN=$HOST" -sha256 -new -key server-key.pem -out server.csr
+    echo subjectAltName = DNS:$HOST,IP:10.0.0.200,IP:127.0.0.1,IP:10.10.10.189 >> extfile.cnf
+    echo extendedKeyUsage = serverAuth >> extfile.cnf
+
+    openssl x509 -req -days 365 -sha256 -in server.csr -CA ca.pem -CAkey ca-key.pem \
+    -CAcreateserial -out server-cert.pem -extfile extfile.cnf
+else
+    echo "server-cert.pem ca cert already exists"
+fi
+
+if [[ ! -f "agent-cert.pem" ]]; then
+    echo "Generate an Agent cert"
+    openssl genrsa -out agent-key.pem 4096
+
+    openssl req -subj '/CN=client' -new -key agent-key.pem -out agent-client.csr
+    echo extendedKeyUsage = clientAuth > agent-extfile.cnf
+
+    openssl x509 -req -days 365 -sha256 -in agent-client.csr -CA ca.pem -CAkey ca-key.pem \
+    -CAcreateserial -out agent-cert.pem -extfile agent-extfile.cnf
+else
+    echo "agent-cert.pem ca cert already exists"
+fi
+
+echo "done: Generated example mTLS certs into $(pwd)"
--- a/gruntfile.js
+++ b/gruntfile.js
@@ -159,6 +159,7 @@ function shell_run_container() {
  const portainerData = '${PORTAINER_DATA:-/tmp/portainer}';
  const portainerRoot = process.env.PORTAINER_PROJECT ? process.env.PORTAINER_PROJECT : process.env.PWD;
  const portainerFlags = '${PORTAINER_FLAGS:-}';
+  const portainerDockerFlags = '${PORTAINER_DOCKER_FLAGS:-}';

  return `
    docker rm -f portainer
@@ -166,6 +167,7 @@ function shell_run_container() {
      -p 8000:8000 \
      -p 9000:9000 \
      -p 9443:9443 \
+      ${portainerDockerFlags} \
      -v ${portainerRoot}/dist:/app \
      -v ${portainerData}:/data \
      -v /var/run/docker.sock:/var/run/docker.sock:z \
--- a/yarn.lock
+++ b/yarn.lock
@@ -4547,7 +4547,6 @@ angular-moment-picker@^0.10.2:
  dependencies:
    angular-mocks "1.6.1"
    angular-sanitize "1.6.1"
-    lodash-es "^4.17.15"

 angular-resource@1.8.2:
  version "1.8.2"
@@ -12598,7 +12597,7 @@ locate-path@^6.0.0:
  dependencies:
    p-locate "^5.0.0"

-lodash-es@^4.17.21:
+lodash-es@^4.17.15, lodash-es@^4.17.21:
  version "4.17.21"
  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
  integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==
Author	SHA1	Message	Date
Sven Dowideit	b5133e3aa6	POC: support agent auto-upgrade Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-25 13:25:46 +10:00
Sven Dowideit	4f0e09fad8	start sending the actual configured stacks Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-21 14:23:52 +10:00
Sven Dowideit	428d2a03bf	and using mtls certs Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-14 17:33:02 +10:00
Sven Dowideit	82a13560b3	and using mtls certs Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-14 17:32:42 +10:00
Sven Dowideit	6f354833b7	enough working so using curl can get portainer to record the env as having a heartbeat, and the timestamp is good. Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-14 17:22:39 +10:00
Sven Dowideit	ad5ece52b1	add a PORTAINER_DOCKER_FLAGS env to yarn so its easy to use different docker proxy options Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 13:11:35 +10:00
Sven Dowideit	7ddd02bf91	if --sslcacert is set, reject any Edge requests that don't have a ClientAuth cert signed by it Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 08:19:38 +10:00
Sven Dowideit	71c8932f76	fix things to add --sslcert, --sslkey, --sslcacert to agent Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 08:19:38 +10:00
Sven Dowideit	eab2ae6230	fix logging so we don't have 3 different output formats Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 08:19:38 +10:00
Sven Dowideit	133bd15dad	need to separate the edge from non-edge api bouncer stats - edge pin swamps everything Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 08:19:38 +10:00
Sven Dowideit	b93a11b4e5	feat(dev-metrics) add a prometheus metrics endpoint Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>	2022-02-11 08:19:36 +10:00