bump version to 2.16.2 (#8083)

* fix(snapshots): remove snapshots when removing endpoints EE-4527

* Fix nil pointer dereference.

Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>

README.md

@@ -12,21 +12,15 @@ Portainer consists of a single container that can run on any cluster. It can be
 - [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
 - [Portainer BE install guide](https://install.portainer.io)
 
-## Demo
-
-You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
-
-Please note that the public demo cluster is **reset every 15min**.
-
 ## Latest Version
 
 Portainer CE is updated regularly. We aim to do an update release every couple of months.
 
-**The latest version of Portainer is 2.13.x**.
+[![latest version](https://img.shields.io/github/v/release/portainer/portainer?color=%2344cc11&label=Latest%20release&style=for-the-badge)](https://github.com/portainer/portainer/releases/latest)
 
 ## Getting started
 
-- [Deploy Portainer](https://docs.portainer.io/v/ce-2.9/start/install)
+- [Deploy Portainer](https://docs.portainer.io/start/install)
 - [Documentation](https://documentation.portainer.io)
 - [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
 

api/adminmonitor/admin_monitor.go

@@ -2,7 +2,6 @@ package adminmonitor
 
 import (
 	"context"
-	"log"
 	"net/http"
 	"strings"
 	"sync"
@@ -11,9 +10,9 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-)
 
-var logFatalf = log.Fatalf
+	"github.com/rs/zerolog/log"
+)
 
 const RedirectReasonAdminInitTimeout string = "AdminInitTimeout"
 
@@ -49,24 +48,28 @@ func (m *Monitor) Start() {
 	m.cancellationFunc = cancellationFunc
 
 	go func() {
-		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
+		log.Debug().Msg("start initialization monitor")
+
 		select {
 		case <-time.After(m.timeout):
 			initialized, err := m.WasInitialized()
 			if err != nil {
-				logFatalf("%s", err)
+				log.Fatal().Err(err).Msg("")
 			}
+
 			if !initialized {
-				log.Println("[INFO] [internal,init] The Portainer instance timed out for security purposes. To re-enable your Portainer instance, you will need to restart Portainer")
+				log.Info().Msg("the Portainer instance timed out for security purposes, to re-enable your Portainer instance, you will need to restart Portainer")
+
 				m.mu.Lock()
 				defer m.mu.Unlock()
+
 				m.adminInitDisabled = true
 				return
 			}
 		case <-cancellationCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
+			log.Debug().Msg("canceling initialization monitor")
 		case <-m.shutdownCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
+			log.Debug().Msg("shutting down initialization monitor")
 		}
 	}()
 }

api/apikey/service_test.go

@@ -2,7 +2,6 @@ package apikey
 
 import (
 	"crypto/sha256"
-	"log"
 	"strings"
 	"testing"
 	"time"
@@ -10,6 +9,8 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/rs/zerolog/log"
 )
 
 func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
@@ -20,7 +21,7 @@ func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
 func Test_GenerateApiKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -74,7 +75,7 @@ func Test_GenerateApiKey(t *testing.T) {
 func Test_GetAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -94,7 +95,7 @@ func Test_GetAPIKey(t *testing.T) {
 func Test_GetAPIKeys(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -115,7 +116,7 @@ func Test_GetAPIKeys(t *testing.T) {
 func Test_GetDigestUserAndKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -151,7 +152,7 @@ func Test_GetDigestUserAndKey(t *testing.T) {
 func Test_UpdateAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -169,8 +170,8 @@ func Test_UpdateAPIKey(t *testing.T) {
 		_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
 		is.NoError(err)
 
-		log.Println(apiKey)
-		log.Println(apiKeyGot)
+		log.Debug().Msgf("%+v", apiKey)
+		log.Debug().Msgf("%+v", apiKeyGot)
 
 		is.Equal(apiKey.LastUsed, apiKeyGot.LastUsed)
 
@@ -199,7 +200,7 @@ func Test_UpdateAPIKey(t *testing.T) {
 func Test_DeleteAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -240,7 +241,7 @@ func Test_DeleteAPIKey(t *testing.T) {
 func Test_InvalidateUserKeyCache(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())

api/archive/tar.go

@@ -34,3 +34,45 @@ func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, e
 
 	return buffer.Bytes(), nil
 }
+
+// tarFileInBuffer represents a tar archive buffer.
+type tarFileInBuffer struct {
+	b *bytes.Buffer
+	w *tar.Writer
+}
+
+func NewTarFileInBuffer() *tarFileInBuffer {
+	var b bytes.Buffer
+	return &tarFileInBuffer{
+		b: &b,
+		w: tar.NewWriter(&b),
+	}
+}
+
+// Put puts a single file to tar archive buffer.
+func (t *tarFileInBuffer) Put(fileContent []byte, fileName string, mode int64) error {
+	hdr := &tar.Header{
+		Name: fileName,
+		Mode: mode,
+		Size: int64(len(fileContent)),
+	}
+
+	if err := t.w.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	if _, err := t.w.Write(fileContent); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Bytes returns the archive as a byte array.
+func (t *tarFileInBuffer) Bytes() []byte {
+	return t.b.Bytes()
+}
+
+func (t *tarFileInBuffer) Close() error {
+	return t.w.Close()
+}

api/archive/targz_test.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -27,9 +26,7 @@ func listFiles(dir string) []string {
 }
 
 func Test_shouldCreateArhive(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
@@ -40,9 +37,7 @@ func Test_shouldCreateArhive(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
 
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
+	extractionDir := t.TempDir()
 	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
 	err = cmd.Run()
 	if err != nil {
@@ -63,9 +58,7 @@ func Test_shouldCreateArhive(t *testing.T) {
 }
 
 func Test_shouldCreateArhiveXXXXX(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
@@ -76,9 +69,7 @@ func Test_shouldCreateArhiveXXXXX(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
 
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
+	extractionDir := t.TempDir()
 	r, _ := os.Open(gzPath)
 	ExtractTarGz(r, extractionDir)
 	if err != nil {

api/archive/zip_test.go

@@ -2,16 +2,12 @@ package archive
 
 import (
 	"github.com/stretchr/testify/assert"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"testing"
 )
 
 func TestUnzipFile(t *testing.T) {
-	dir, err := ioutil.TempDir("", "unzip-test-")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	/*
 		Archive structure.
 		├── 0
@@ -21,7 +17,7 @@ func TestUnzipFile(t *testing.T) {
 		└── 0.txt
 	*/
 
-	err = UnzipFile("./testdata/sample_archive.zip", dir)
+	err := UnzipFile("./testdata/sample_archive.zip", dir)
 
 	assert.NoError(t, err)
 	archiveDir := dir + "/sample_archive"

api/backup/backup.go

@@ -7,13 +7,14 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
-	"github.com/sirupsen/logrus"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 const rwxr__r__ os.FileMode = 0744
@@ -47,9 +48,9 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 
 		err := datastore.Export(exportFilename)
 		if err != nil {
-			logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
+			log.Error().Err(err).Str("filename", exportFilename).Msg("failed to export")
 		} else {
-			logrus.Debugf("exported to %s", exportFilename)
+			log.Debug().Str("filename", exportFilename).Msg("file exported")
 		}
 	}
 

api/chisel/service.go

@@ -3,16 +3,17 @@ package chisel
 import (
 	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/dchest/uniuri"
-	chserver "github.com/jpillora/chisel/server"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
+
+	"github.com/dchest/uniuri"
+	chserver "github.com/jpillora/chisel/server"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -64,7 +65,11 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
 func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
 	go func() {
-		log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: start for %.0f minutes]\n", endpointID, maxAlive.Minutes())
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Float64("max_alive_minutes", maxAlive.Minutes()).
+			Msg("start")
+
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
@@ -76,14 +81,25 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				service.SetTunnelStatusToActive(endpointID)
 				err := service.pingAgent(endpointID)
 				if err != nil {
-					log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [warning: ping agent err=%s]\n", endpointID, err)
+					log.Debug().
+						Int("endpoint_id", int(endpointID)).
+						Err(err).
+						Msg("ping agent")
 				}
 			case <-maxAliveTicker.C:
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as %.0f minutes timeout]\n", endpointID, maxAlive.Minutes())
+				log.Debug().
+					Int("endpoint_id", int(endpointID)).
+					Float64("timeout_minutes", maxAlive.Minutes()).
+					Msg("tunnel keep alive timeout")
+
 				return
 			case <-ctx.Done():
 				err := ctx.Err()
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as err=%s]\n", endpointID, err)
+				log.Debug().
+					Int("endpoint_id", int(endpointID)).
+					Err(err).
+					Msg("tunnel stop")
+
 				return
 			}
 		}
@@ -162,7 +178,10 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 }
 
 func (service *Service) startTunnelVerificationLoop() {
-	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
+	log.Debug().
+		Float64("check_interval_seconds", tunnelCleanupInterval.Seconds()).
+		Msg("starting tunnel management process")
+
 	ticker := time.NewTicker(tunnelCleanupInterval)
 
 	for {
@@ -170,10 +189,12 @@ func (service *Service) startTunnelVerificationLoop() {
 		case <-ticker.C:
 			service.checkTunnels()
 		case <-service.shutdownCtx.Done():
-			log.Println("[DEBUG] Shutting down tunnel service")
+			log.Debug().Msg("shutting down tunnel service")
+
 			if err := service.StopTunnelServer(); err != nil {
-				log.Printf("Stopped tunnel service: %s", err)
+				log.Debug().Err(err).Msg("stopped tunnel service")
 			}
+
 			ticker.Stop()
 			return
 		}
@@ -195,22 +216,40 @@ func (service *Service) checkTunnels() {
 		}
 
 		elapsed := time.Since(tunnel.LastActivity)
-		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", endpointID, tunnel.Status, elapsed.Seconds())
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Str("status", tunnel.Status).
+			Float64("status_time_seconds", elapsed.Seconds()).
+			Msg("environment tunnel monitoring")
 
 		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() > requiredTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", endpointID, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Str("status", tunnel.Status).
+				Float64("status_time_seconds", elapsed.Seconds()).
+				Float64("timeout_seconds", requiredTimeout.Seconds()).
+				Msg("REQUIRED state timeout exceeded")
 		}
 
 		if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() < activeTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() > activeTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", endpointID, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Str("status", tunnel.Status).
+				Float64("status_time_seconds", elapsed.Seconds()).
+				Float64("timeout_seconds", activeTimeout.Seconds()).
+				Msg("ACTIVE state timeout exceeded")
 
 			err := service.snapshotEnvironment(endpointID, tunnel.Port)
 			if err != nil {
-				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %d): %s", endpointID, err)
+				log.Error().
+					Int("endpoint_id", int(endpointID)).Err(
+
+					err).
+					Msg("unable to snapshot Edge environment")
 			}
 		}
 

api/cli/cli.go

@@ -2,15 +2,14 @@ package cli
 
 import (
 	"errors"
-	"log"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 
-	"os"
-	"path/filepath"
-	"strings"
-
+	"github.com/rs/zerolog/log"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -62,6 +61,8 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		MaxBatchSize:              kingpin.Flag("max-batch-size", "Maximum size of a batch").Int(),
 		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
+		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()
@@ -101,11 +102,11 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
 func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 	if *flags.NoAnalytics {
-		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
+		log.Warn().Msg("the --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect")
 	}
 
 	if *flags.SSL {
-		log.Println("Warning: SSL is enabled by default and there is no need for the --ssl flag. It has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
+		log.Warn().Msg("SSL is enabled by default and there is no need for the --ssl flag, it has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
 	}
 }
 

api/cmd/portainer/import.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"log"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 func importFromJson(fileService portainer.FileService, store *datastore.Store) {
@@ -13,17 +12,17 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
 	importFile := "/data/import.json"
 	if exists, _ := fileService.FileExists(importFile); exists {
 		if err := store.Import(importFile); err != nil {
-			logrus.WithError(err).Debugf("Import %s failed", importFile)
-
+			log.Error().Str("filename", importFile).Err(err).Msg("import failed")
 			// TODO: should really rollback on failure, but then we have nothing.
 		} else {
-			logrus.Printf("Successfully imported %s to new portainer database", importFile)
+			log.Info().Str("filename", importFile).Msg("successfully imported the file to a new portainer database")
 		}
+
 		// TODO: this is bad - its to ensure that any defaults that were broken in import, or migrations get set back to what we want
 		// I also suspect that everything from "Init to Init" is potentially a migration
 		err := store.Init()
 		if err != nil {
-			log.Fatalf("Failed initializing data store: %v", err)
+			log.Fatal().Err(err).Msg("failed initializing data store")
 		}
 	}
 }

api/cmd/portainer/log.go

@@ -1,20 +1,55 @@
 package main
 
 import (
-	"log"
+	"fmt"
+	stdlog "log"
+	"os"
 
-	"github.com/sirupsen/logrus"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/pkgerrors"
 )
 
 func configureLogger() {
-	logger := logrus.New() // logger is to implicitly substitute stdlib's log
-	log.SetOutput(logger.Writer())
+	zerolog.ErrorStackFieldName = "stack_trace"
+	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
+	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 
-	formatter := &logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true}
+	stdlog.SetFlags(0)
+	stdlog.SetOutput(log.Logger)
 
-	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatter)
-
-	logger.SetLevel(logrus.DebugLevel)
-	logrus.SetLevel(logrus.DebugLevel)
+	log.Logger = log.Logger.With().Caller().Stack().Logger()
+}
+
+func setLoggingLevel(level string) {
+	switch level {
+	case "ERROR":
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+	case "WARN":
+		zerolog.SetGlobalLevel(zerolog.WarnLevel)
+	case "INFO":
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	case "DEBUG":
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+}
+
+func setLoggingMode(mode string) {
+	switch mode {
+	case "PRETTY":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			NoColor:       true,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage})
+	case "JSON":
+		log.Logger = log.Output(os.Stderr)
+	}
+}
+
+func formatMessage(i interface{}) string {
+	if i == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s |", i)
 }

api/cmd/portainer/main.go

@@ -4,15 +4,12 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/portainer/libhelm"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
@@ -45,34 +42,38 @@ import (
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks"
+
+	"github.com/rs/zerolog/log"
 )
 
 func initCLI() *portainer.CLIFlags {
 	var cliService portainer.CLIService = &cli.Service{}
 	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
-		logrus.Fatalf("Failed parsing flags: %v", err)
+		log.Fatal().Err(err).Msg("failed parsing flags")
 	}
 
 	err = cliService.ValidateFlags(flags)
 	if err != nil {
-		logrus.Fatalf("Failed validating flags:%v", err)
+		log.Fatal().Err(err).Msg("failed validating flags")
 	}
+
 	return flags
 }
 
 func initFileService(dataStorePath string) portainer.FileService {
 	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
-		logrus.Fatalf("Failed creating file service: %v", err)
+		log.Fatal().Err(err).Msg("failed creating file service")
 	}
+
 	return fileService
 }
 
 func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
 	connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
 	if err != nil {
-		logrus.Fatalf("failed creating database connection: %s", err)
+		log.Fatal().Err(err).Msg("failed creating database connection")
 	}
 
 	if bconn, ok := connection.(*boltdb.DbConnection); ok {
@@ -80,30 +81,31 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 		bconn.MaxBatchDelay = *flags.MaxBatchDelay
 		bconn.InitialMmapSize = *flags.InitialMmapSize
 	} else {
-		logrus.Fatalf("failed creating database connection: expecting a boltdb database type but a different one was received")
+		log.Fatal().Msg("failed creating database connection: expecting a boltdb database type but a different one was received")
 	}
 
 	store := datastore.NewStore(*flags.Data, fileService, connection)
 	isNew, err := store.Open()
 	if err != nil {
-		logrus.Fatalf("Failed opening store: %v", err)
+		log.Fatal().Err(err).Msg("failed opening store")
 	}
 
 	if *flags.Rollback {
 		err := store.Rollback(false)
 		if err != nil {
-			logrus.Fatalf("Failed rolling back: %v", err)
+			log.Fatal().Err(err).Msg("failed rolling back")
 		}
 
-		logrus.Println("Exiting rollback")
+		log.Info().Msg("exiting rollback")
 		os.Exit(0)
+
 		return nil
 	}
 
 	// Init sets some defaults - it's basically a migration
 	err = store.Init()
 	if err != nil {
-		logrus.Fatalf("Failed initializing data store: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing data store")
 	}
 
 	if isNew {
@@ -112,24 +114,25 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 
 		err := updateSettingsFromFlags(store, flags)
 		if err != nil {
-			logrus.Fatalf("Failed updating settings from flags: %v", err)
+			log.Fatal().Err(err).Msg("failed updating settings from flags")
 		}
 	} else {
 		storedVersion, err := store.VersionService.DBVersion()
 		if err != nil {
-			logrus.Fatalf("Something Failed during creation of new database: %v", err)
+			log.Fatal().Err(err).Msg("failure during creation of new database")
 		}
+
 		if storedVersion != portainer.DBVersion {
 			err = store.MigrateData()
 			if err != nil {
-				logrus.Fatalf("Failed migration: %v", err)
+				log.Fatal().Err(err).Msg("failed migration")
 			}
 		}
 	}
 
 	err = updateSettingsFromFlags(store, flags)
 	if err != nil {
-		log.Fatalf("Failed updating settings from flags: %v", err)
+		log.Fatal().Err(err).Msg("failed updating settings from flags")
 	}
 
 	// this is for the db restore functionality - needs more tests.
@@ -141,19 +144,19 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 
 		err := store.Export(exportFilename)
 		if err != nil {
-			logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
+			log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
 		} else {
-			logrus.Debugf("exported to %s", exportFilename)
+			log.Debug().Str("filename", exportFilename).Msg("exported")
 		}
-		connection.Close()
 	}()
+
 	return store
 }
 
 func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
 	if err != nil {
-		logrus.Fatalf("Failed creating compose manager: %v", err)
+		log.Fatal().Err(err).Msg("failed creating compose manager")
 	}
 
 	return composeWrapper
@@ -187,6 +190,7 @@ func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore)
 	if err != nil {
 		return nil, err
 	}
+
 	return jwtService, nil
 }
 
@@ -206,8 +210,8 @@ func initOAuthService() portainer.OAuthService {
 	return oauth.NewService()
 }
 
-func initGitService() portainer.GitService {
-	return git.NewService()
+func initGitService(ctx context.Context) portainer.GitService {
+	return git.NewService(ctx)
 }
 
 func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
@@ -231,8 +235,8 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService,
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout)
 }
 
 func initSnapshotService(snapshotIntervalFromFlag string, dataStore dataservices.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
@@ -341,11 +345,7 @@ func enableFeaturesFromFlags(dataStore dataservices.DataStore, flags *portainer.
 			return fmt.Errorf("feature flag's '%s' value should be true or false", feat.Name)
 		}
 
-		if featureState {
-			logrus.Printf("Feature %v : on", *correspondingFeature)
-		} else {
-			logrus.Printf("Feature %v : off", *correspondingFeature)
-		}
+		log.Info().Str("feature", string(*correspondingFeature)).Bool("state", featureState).Msg("")
 
 		settings.FeatureFlagSettings[*correspondingFeature] = featureState
 	}
@@ -373,7 +373,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
 func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	existingKeyPair, err := fileService.KeyPairFilesExist()
 	if err != nil {
-		logrus.Fatalf("Failed checking for existing key pair: %v", err)
+		log.Fatal().Err(err).Msg("failed checking for existing key pair")
 	}
 
 	if existingKeyPair {
@@ -443,7 +443,11 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		logrus.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).
+			Err(err).
+			Msg("environment snapshot error")
 	}
 
 	return dataStore.Endpoint().Create(endpoint)
@@ -488,7 +492,10 @@ func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStor
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		logrus.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).Err(err).
+			Msg("environment snapshot error")
 	}
 
 	return dataStore.Endpoint().Create(endpoint)
@@ -505,7 +512,8 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
 	}
 
 	if len(endpoints) > 0 {
-		logrus.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
+		log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
+
 		return nil
 	}
 
@@ -519,9 +527,9 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
 	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
 	if err != nil {
 		if os.IsNotExist(err) {
-			logrus.Printf("Encryption key file `%s` not present", keyfilename)
+			log.Info().Str("filename", keyfilename).Msg("encryption key file not present")
 		} else {
-			logrus.Printf("Error reading encryption key file: %v", err)
+			log.Info().Err(err).Msg("error reading encryption key file")
 		}
 
 		return nil
@@ -538,38 +546,41 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	fileService := initFileService(*flags.Data)
 	encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
 	if encryptionKey == nil {
-		logrus.Println("Proceeding without encryption key")
+		log.Info().Msg("proceeding without encryption key")
 	}
 
 	dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
-		logrus.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
+
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
-		logrus.Fatalf("Failed getting instance id: %v", err)
+		log.Fatal().Err(err).Msg("failed getting instance id")
 	}
 
 	apiKeyService := initAPIKeyService(dataStore)
 
 	settings, err := dataStore.Settings().Settings()
 	if err != nil {
-		logrus.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
+
 	jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore)
 	if err != nil {
-		logrus.Fatalf("Failed initializing JWT service: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing JWT service")
 	}
 
 	err = enableFeaturesFromFlags(dataStore, flags)
 	if err != nil {
-		logrus.Fatalf("Failed enabling feature flag: %v", err)
+		log.Fatal().Err(err).Msg("failed enabling feature flag")
 	}
 
 	ldapService := initLDAPService()
+
 	oauthService := initOAuthService()
-	gitService := initGitService()
+	gitService := initGitService(shutdownCtx)
 
 	openAMTService := openamt.NewService()
 
@@ -579,27 +590,27 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
 	if err != nil {
-		logrus.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
 
 	sslSettings, err := sslService.GetSSLSettings()
 	if err != nil {
-		logrus.Fatalf("Failed to get ssl settings: %s", err)
+		log.Fatal().Err(err).Msg("failed to get SSL settings")
 	}
 
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
-		logrus.Fatalf("Failed initializing key pair: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
 	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
+	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
-		logrus.Fatalf("Failed initializing snapshot service: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
 	}
 	snapshotService.Start()
 
@@ -620,19 +631,19 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
-		logrus.Fatalf("Failed initializing swarm stack manager: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
 	}
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
-		logrus.Fatalf("Failed initializing helm package manager: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing helm package manager")
 	}
 
 	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
 	if err != nil {
-		logrus.Fatalf("Failed loading edge jobs from database: %v", err)
+		log.Fatal().Err(err).Msg("failed loading edge jobs from database")
 	}
 
 	applicationStatus := initStatus(instanceID)
@@ -641,24 +652,25 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	if *flags.DemoEnvironment {
 		err := demoService.Init(dataStore, cryptoService)
 		if err != nil {
-			log.Fatalf("failed initializing demo environment: %v", err)
+			log.Fatal().Err(err).Msg("failed initializing demo environment")
 		}
 	}
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		logrus.Fatalf("Failed initializing environment: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing environment")
 	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
 		content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
 		if err != nil {
-			logrus.Fatalf("Failed getting admin password file: %v", err)
+			log.Fatal().Err(err).Msg("failed getting admin password file")
 		}
+
 		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
-			logrus.Fatalf("Failed hashing admin password: %v", err)
+			log.Fatal().Err(err).Msg("failed hashing admin password")
 		}
 	} else if *flags.AdminPassword != "" {
 		adminPasswordHash = *flags.AdminPassword
@@ -667,39 +679,57 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	if adminPasswordHash != "" {
 		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
-			logrus.Fatalf("Failed getting admin user: %v", err)
+			log.Fatal().Err(err).Msg("failed getting admin user")
 		}
 
 		if len(users) == 0 {
-			logrus.Println("Created admin user with the given password.")
+			log.Info().Msg("created admin user with the given password.")
 			user := &portainer.User{
 				Username: "admin",
 				Role:     portainer.AdministratorRole,
 				Password: adminPasswordHash,
 			}
+
 			err := dataStore.User().Create(user)
 			if err != nil {
-				logrus.Fatalf("Failed creating admin user: %v", err)
+				log.Fatal().Err(err).Msg("failed creating admin user")
 			}
 		} else {
-			logrus.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
+			log.Info().Msg("instance already has an administrator user defined, skipping admin password related flags.")
 		}
 	}
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		logrus.Fatalf("Failed starting tunnel server: %v", err)
+		log.Fatal().Err(err).Msg("failed starting tunnel server")
 	}
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
 	if err != nil {
-		logrus.Fatalf("Failed to fetch ssl settings from DB")
+		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
 	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
 	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
+	// FIXME: In 2.16 we changed the way ingress controller permissions are
+	// stored. Instead of being stored as annotation on an ingress rule, we keep
+	// them in our database. However, in order to run the migration we need an
+	// admin kube client to run lookup the old ingress rules and compare them
+	// with the current existing ingress classes.
+	//
+	// Unfortunately, our migrations run as part of the database initialization
+	// and our kubeclients require an initialized database. So it is not
+	// possible to do this migration as part of our normal flow. We DO have a
+	// migration which toggles a boolean in kubernetes configuration that
+	// indicated that this "post init" migration should be run. If/when this is
+	// resolved we can remove this function.
+	err = kubernetesClientFactory.PostInitMigrateIngresses()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failure during creation of new database")
+	}
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
@@ -738,22 +768,27 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 }
 
 func main() {
+	configureLogger()
+	setLoggingMode("PRETTY")
+
 	flags := initCLI()
 
-	configureLogger()
+	setLoggingLevel(*flags.LogLevel)
+	setLoggingMode(*flags.LogMode)
 
 	for {
 		server := buildServer(flags)
-		logrus.WithFields(logrus.Fields{
-			"Version":        portainer.APIVersion,
-			"BuildNumber":    build.BuildNumber,
-			"ImageTag":       build.ImageTag,
-			"NodejsVersion":  build.NodejsVersion,
-			"YarnVersion":    build.YarnVersion,
-			"WebpackVersion": build.WebpackVersion,
-			"GoVersion":      build.GoVersion},
-		).Print("[INFO] [cmd,main] Starting Portainer")
+		log.Info().
+			Str("version", portainer.APIVersion).
+			Str("build_number", build.BuildNumber).
+			Str("image_tag", build.ImageTag).
+			Str("nodejs_version", build.NodejsVersion).
+			Str("yarn_version", build.YarnVersion).
+			Str("webpack_version", build.WebpackVersion).
+			Str("go_version", build.GoVersion).
+			Msg("starting Portainer")
+
 		err := server.Start()
-		logrus.Printf("[INFO] [cmd,main] Http server exited: %v\n", err)
+		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/cmd/portainer/main_test.go

@@ -21,7 +21,7 @@ func (m mockKingpinSetting) SetValue(value kingpin.Value) {
 func Test_enableFeaturesFromFlags(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	tests := []struct {
@@ -76,7 +76,7 @@ func Test_optionalFeature(t *testing.T) {
 
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	// Enable the test feature

api/crypto/aes_test.go

@@ -7,13 +7,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")
@@ -52,8 +50,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 }
 
 func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")
@@ -92,8 +89,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 }
 
 func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")

api/database/boltdb/db.go

@@ -11,7 +11,8 @@ import (
 	"time"
 
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -120,7 +121,7 @@ func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
 // Open opens and initializes the BoltDB database.
 func (connection *DbConnection) Open() error {
 
-	logrus.Infof("Loading PortainerDB: %s", connection.GetDatabaseFileName())
+	log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
 
 	// Now we open the db
 	databasePath := connection.GetDatabaseFilePath()
@@ -348,6 +349,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
 func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
 	err := connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
+
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			err := connection.UnmarshalObject(v, obj)
@@ -362,6 +364,7 @@ func (connection *DbConnection) GetAll(bucketName string, obj interface{}, appen
 
 		return nil
 	})
+
 	return err
 }
 
@@ -411,7 +414,7 @@ func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error
 	for bucketName, v := range s {
 		id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
 		if !ok {
-			logrus.Errorf("Failed to restore metadata to bucket %s, skipped", bucketName)
+			log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
 			continue
 		}
 
@@ -420,6 +423,7 @@ func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error
 			if err != nil {
 				return err
 			}
+
 			return bucket.SetSequence(uint64(id))
 		})
 	}

api/database/boltdb/export.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/sirupsen/logrus"
+	"github.com/rs/zerolog/log"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -28,11 +28,11 @@ func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
 
 // ExportJSON creates a JSON representation from a DbConnection. You can include
 // the database's metadata or ignore it. Ensure the database is closed before
-// using this function
+// using this function.
 // inspired by github.com/konoui/boltdb-exporter (which has no license)
 // but very much simplified, based on how we use boltdb
 func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, error) {
-	logrus.WithField("databasePath", databasePath).Infof("exportJson")
+	log.Debug().Str("databasePath", databasePath).Msg("exportJson")
 
 	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
 	if err != nil {
@@ -44,8 +44,9 @@ func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, e
 	if metadata {
 		meta, err := backupMetadata(connection)
 		if err != nil {
-			logrus.WithError(err).Errorf("Failed exporting metadata: %v", err)
+			log.Error().Err(err).Msg("failed exporting metadata")
 		}
+
 		backup["__metadata"] = meta
 	}
 
@@ -59,22 +60,31 @@ func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, e
 				if v == nil {
 					continue
 				}
+
 				var obj interface{}
 				err := c.UnmarshalObject(v, &obj)
 				if err != nil {
-					logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v))
+					log.Error().
+						Str("bucket", bucketName).
+						Str("object", string(v)).
+						Err(err).
+						Msg("failed to unmarshal")
+
 					obj = v
 				}
+
 				if bucketName == "version" {
 					version[string(k)] = string(v)
 				} else {
 					list = append(list, obj)
 				}
 			}
+
 			if bucketName == "version" {
 				backup[bucketName] = version
 				return nil
 			}
+
 			if len(list) > 0 {
 				if bucketName == "ssl" ||
 					bucketName == "settings" ||
@@ -91,8 +101,10 @@ func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, e
 
 			return nil
 		})
+
 		return err
 	})
+
 	if err != nil {
 		return []byte("{}"), err
 	}

api/database/models/configmaps_and_secrets.go

@@ -0,0 +1,17 @@
+package models
+
+type (
+	K8sConfigMapOrSecret struct {
+		UID          string            `json:"UID"`
+		Name         string            `json:"Name"`
+		Namespace    string            `json:"Namespace"`
+		CreationDate string            `json:"CreationDate"`
+		Annotations  map[string]string `json:"Annotations"`
+		Data         map[string]string `json:"Data"`
+		Applications []string          `json:"Applications"`
+		IsSecret     bool              `json:"IsSecret"`
+
+		// SecretType will be an empty string for config maps.
+		SecretType string `json:"SecretType"`
+	}
+)

api/database/models/ingress.go

@@ -0,0 +1,75 @@
+package models
+
+import (
+	"errors"
+	"net/http"
+)
+
+type (
+	K8sIngressController struct {
+		Name         string `json:"Name"`
+		ClassName    string `json:"ClassName"`
+		Type         string `json:"Type"`
+		Availability bool   `json:"Availability"`
+		New          bool   `json:"New"`
+		Used         bool   `json:"Used"`
+	}
+
+	K8sIngressControllers []K8sIngressController
+
+	K8sIngressInfo struct {
+		Name        string            `json:"Name"`
+		UID         string            `json:"UID"`
+		Type        string            `json:"Type"`
+		Namespace   string            `json:"Namespace"`
+		ClassName   string            `json:"ClassName"`
+		Annotations map[string]string `json:"Annotations"`
+		Hosts       []string          `json:"Hosts"`
+		Paths       []K8sIngressPath  `json:"Paths"`
+		TLS         []K8sIngressTLS   `json:"TLS"`
+	}
+
+	K8sIngressTLS struct {
+		Hosts      []string `json:"Hosts"`
+		SecretName string   `json:"SecretName"`
+	}
+
+	K8sIngressPath struct {
+		IngressName string `json:"IngressName"`
+		Host        string `json:"Host"`
+		ServiceName string `json:"ServiceName"`
+		Port        int    `json:"Port"`
+		Path        string `json:"Path"`
+		PathType    string `json:"PathType"`
+	}
+
+	// K8sIngressDeleteRequests is a mapping of namespace names to a slice of
+	// ingress names.
+	K8sIngressDeleteRequests map[string][]string
+)
+
+func (r K8sIngressControllers) Validate(request *http.Request) error {
+	return nil
+}
+
+func (r K8sIngressInfo) Validate(request *http.Request) error {
+	if r.Name == "" {
+		return errors.New("missing ingress name from the request payload")
+	}
+	if r.Namespace == "" {
+		return errors.New("missing ingress Namespace from the request payload")
+	}
+	return nil
+}
+
+func (r K8sIngressDeleteRequests) Validate(request *http.Request) error {
+	if len(r) == 0 {
+		return errors.New("missing deletion request list in payload")
+	}
+	for ns := range r {
+		if len(ns) == 0 {
+			return errors.New("deletion given with empty namespace")
+		}
+	}
+	return nil
+}

api/database/models/namespaces.go

@@ -0,0 +1,12 @@
+package models
+
+import "net/http"
+
+type K8sNamespaceDetails struct {
+	Name        string            `json:"Name"`
+	Annotations map[string]string `json:"Annotations"`
+}
+
+func (r *K8sNamespaceDetails) Validate(request *http.Request) error {
+	return nil
+}

api/database/models/services.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"errors"
+	"net/http"
+)
+
+type (
+	K8sServiceInfo struct {
+		Name                          string              `json:"Name"`
+		UID                           string              `json:"UID"`
+		Type                          string              `json:"Type"`
+		Namespace                     string              `json:"Namespace"`
+		Annotations                   map[string]string   `json:"Annotations"`
+		CreationTimestamp             string              `json:"CreationTimestamp"`
+		Labels                        map[string]string   `json:"Labels"`
+		AllocateLoadBalancerNodePorts *bool               `json:"AllocateLoadBalancerNodePorts,omitempty"`
+		Ports                         []K8sServicePort    `json:"Ports"`
+		Selector                      map[string]string   `json:"Selector"`
+		IngressStatus                 []K8sServiceIngress `json:"IngressStatus"`
+	}
+
+	K8sServicePort struct {
+		Name       string `json:"Name"`
+		NodePort   int    `json:"NodePort"`
+		Port       int    `json:"Port"`
+		Protocol   string `json:"Protocol"`
+		TargetPort int    `json:"TargetPort"`
+	}
+
+	K8sServiceIngress struct {
+		IP   string `json:"IP"`
+		Host string `json:"Host"`
+	}
+
+	// K8sServiceDeleteRequests is a mapping of namespace names to a slice of
+	// service names.
+	K8sServiceDeleteRequests map[string][]string
+)
+
+func (s *K8sServiceInfo) Validate(request *http.Request) error {
+	if s.Name == "" {
+		return errors.New("missing service name from the request payload")
+	}
+	if s.Namespace == "" {
+		return errors.New("missing service namespace from the request payload")
+	}
+	if s.Ports == nil {
+		return errors.New("missing service ports from the request payload")
+	}
+	return nil
+}
+
+func (r K8sServiceDeleteRequests) Validate(request *http.Request) error {
+	if len(r) == 0 {
+		return errors.New("missing deletion request list in payload")
+	}
+	for ns := range r {
+		if len(ns) == 0 {
+			return errors.New("deletion given with empty namespace")
+		}
+	}
+	return nil
+}

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -6,7 +6,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -41,12 +42,14 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.APIKey)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to APIKey object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
 				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
+
 			if record.UserID == userID {
 				result = append(result, *record)
 			}
+
 			return &portainer.APIKey{}, nil
 		})
 
@@ -64,18 +67,21 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 		func(obj interface{}) (interface{}, error) {
 			key, ok := obj.(*portainer.APIKey)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to APIKey object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
 				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
 			if bytes.Equal(key.Digest, digest) {
 				k = key
 				return nil, stop
 			}
+
 			return &portainer.APIKey{}, nil
 		})
+
 	if err == stop {
 		return k, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}

api/dataservices/customtemplate/customtemplate.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -44,10 +45,11 @@ func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 			//var tag portainer.Tag
 			customTemplate, ok := obj.(*portainer.CustomTemplate)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to CustomTemplate object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
 				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
 			}
 			customTemplates = append(customTemplates, *customTemplate)
+
 			return &portainer.CustomTemplate{}, nil
 		})
 

api/dataservices/edgegroup/edgegroup.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -43,10 +44,11 @@ func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 		func(obj interface{}) (interface{}, error) {
 			group, ok := obj.(*portainer.EdgeGroup)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeGroup object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
 				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
 			}
 			groups = append(groups, *group)
+
 			return &portainer.EdgeGroup{}, nil
 		})
 

api/dataservices/edgejob/edgejob.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -41,13 +42,14 @@ func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 		BucketName,
 		&portainer.EdgeJob{},
 		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
 			job, ok := obj.(*portainer.EdgeJob)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeJob object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
 				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
 			}
+
 			edgeJobs = append(edgeJobs, *job)
+
 			return &portainer.EdgeJob{}, nil
 		})
 

api/dataservices/edgestack/edgestack.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -44,10 +45,12 @@ func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 			//var tag portainer.Tag
 			stack, ok := obj.(*portainer.EdgeStack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeStack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
 				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
 			}
+
 			stacks = append(stacks, *stack)
+
 			return &portainer.EdgeStack{}, nil
 		})
 

api/dataservices/edgeupdateschedule/edgeupdateschedule.go

@@ -0,0 +1,185 @@
+package edgeupdateschedule
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/edgetypes"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edge_update_schedule"
+)
+
+// Service represents a service for managing Edge Update Schedule data.
+type Service struct {
+	connection portainer.Connection
+
+	mu                 sync.Mutex
+	idxActiveSchedules map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	service := &Service{
+		connection: connection,
+	}
+
+	service.idxActiveSchedules = map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation{}
+
+	schedules, err := service.List()
+	if err != nil {
+		return nil, errors.WithMessage(err, "Unable to list schedules")
+	}
+
+	for _, schedule := range schedules {
+		service.setRelation(&schedule)
+	}
+
+	return service, nil
+}
+
+func (service *Service) ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	return service.idxActiveSchedules[environmentID]
+}
+
+func (service *Service) ActiveSchedules(environmentsIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	schedules := []edgetypes.EndpointUpdateScheduleRelation{}
+
+	for _, environmentID := range environmentsIDs {
+		if s, ok := service.idxActiveSchedules[environmentID]; ok {
+			schedules = append(schedules, *s)
+		}
+	}
+
+	return schedules
+}
+
+// List return an array containing all the items in the bucket.
+func (service *Service) List() ([]edgetypes.UpdateSchedule, error) {
+	var list = make([]edgetypes.UpdateSchedule, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&edgetypes.UpdateSchedule{},
+		func(obj interface{}) (interface{}, error) {
+			item, ok := obj.(*edgetypes.UpdateSchedule)
+			if !ok {
+				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeUpdateSchedule object")
+				return nil, fmt.Errorf("failed to convert to EdgeUpdateSchedule object: %s", obj)
+			}
+			list = append(list, *item)
+			return &edgetypes.UpdateSchedule{}, nil
+		})
+
+	return list, err
+}
+
+// Item returns a item by ID.
+func (service *Service) Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error) {
+	var item edgetypes.UpdateSchedule
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &item)
+	if err != nil {
+		return nil, err
+	}
+
+	return &item, nil
+}
+
+// Create assign an ID to a new object and saves it.
+func (service *Service) Create(item *edgetypes.UpdateSchedule) error {
+	err := service.connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			item.ID = edgetypes.UpdateScheduleID(id)
+			return int(item.ID), item
+		},
+	)
+
+	if err != nil {
+		return err
+	}
+
+	return service.setRelation(item)
+}
+
+// Update updates an item.
+func (service *Service) Update(id edgetypes.UpdateScheduleID, item *edgetypes.UpdateSchedule) error {
+	identifier := service.connection.ConvertToKey(int(id))
+	err := service.connection.UpdateObject(BucketName, identifier, item)
+	if err != nil {
+		return err
+	}
+
+	service.cleanRelation(id)
+
+	return service.setRelation(item)
+}
+
+// Delete deletes an item.
+func (service *Service) Delete(id edgetypes.UpdateScheduleID) error {
+
+	service.cleanRelation(id)
+
+	identifier := service.connection.ConvertToKey(int(id))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+func (service *Service) cleanRelation(id edgetypes.UpdateScheduleID) {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	for _, schedule := range service.idxActiveSchedules {
+		if schedule != nil && schedule.ScheduleID == id {
+			delete(service.idxActiveSchedules, schedule.EnvironmentID)
+		}
+	}
+}
+
+func (service *Service) setRelation(schedule *edgetypes.UpdateSchedule) error {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	for environmentID, environmentStatus := range schedule.Status {
+		if environmentStatus.Status != edgetypes.UpdateScheduleStatusPending {
+			continue
+		}
+
+		// this should never happen
+		if service.idxActiveSchedules[environmentID] != nil && service.idxActiveSchedules[environmentID].ScheduleID != schedule.ID {
+			return errors.New("Multiple schedules are pending for the same environment")
+		}
+
+		service.idxActiveSchedules[environmentID] = &edgetypes.EndpointUpdateScheduleRelation{
+			EnvironmentID: environmentID,
+			ScheduleID:    schedule.ID,
+			TargetVersion: environmentStatus.TargetVersion,
+			Status:        environmentStatus.Status,
+			Error:         environmentStatus.Error,
+			Type:          schedule.Type,
+		}
+	}
+
+	return nil
+}

api/dataservices/endpoint/endpoint.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -68,10 +69,12 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 		func(obj interface{}) (interface{}, error) {
 			endpoint, ok := obj.(*portainer.Endpoint)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Endpoint object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
 				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
 			}
+
 			endpoints = append(endpoints, *endpoint)
+
 			return &portainer.Endpoint{}, nil
 		})
 

api/dataservices/endpointgroup/endpointgroup.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -66,13 +67,14 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 		BucketName,
 		&portainer.EndpointGroup{},
 		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
 			endpointGroup, ok := obj.(*portainer.EndpointGroup)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EndpointGroup object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
 				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
 			}
+
 			endpointGroups = append(endpointGroups, *endpointGroup)
+
 			return &portainer.EndpointGroup{}, nil
 		})
 

api/dataservices/endpointrelation/endpointrelation.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -33,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//EndpointRelations returns an array of all EndpointRelations
+// EndpointRelations returns an array of all EndpointRelations
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
@@ -43,10 +44,12 @@ func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error
 		func(obj interface{}) (interface{}, error) {
 			r, ok := obj.(*portainer.EndpointRelation)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EndpointRelation object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
 				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
 			}
+
 			all = append(all, *r)
+
 			return &portainer.EndpointRelation{}, nil
 		})
 

api/dataservices/extension/extension.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 		func(obj interface{}) (interface{}, error) {
 			extension, ok := obj.(*portainer.Extension)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Extension object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
 				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
 			}
+
 			extensions = append(extensions, *extension)
+
 			return &portainer.Extension{}, nil
 		})
 

api/dataservices/fdoprofile/fdoprofile.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -43,8 +44,9 @@ func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
 		func(obj interface{}) (interface{}, error) {
 			fdoProfile, ok := obj.(*portainer.FDOProfile)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to FDOProfile object")
-				return nil, fmt.Errorf("failed to convert to FDOProfile object: %s", obj)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
+
+				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
 			}
 			fdoProfiles = append(fdoProfiles, *fdoProfile)
 			return &portainer.FDOProfile{}, nil

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -33,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//HelmUserRepository returns an array of all HelmUserRepository
+// HelmUserRepository returns an array of all HelmUserRepository
 func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
 	var repos = make([]portainer.HelmUserRepository, 0)
 
@@ -43,10 +44,12 @@ func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository,
 		func(obj interface{}) (interface{}, error) {
 			r, ok := obj.(*portainer.HelmUserRepository)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to HelmUserRepository object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
 				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
 			}
+
 			repos = append(repos, *r)
+
 			return &portainer.HelmUserRepository{}, nil
 		})
 
@@ -63,12 +66,14 @@ func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]p
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.HelmUserRepository)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to HelmUserRepository object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
 				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
 			}
+
 			if record.UserID == userID {
 				result = append(result, *record)
 			}
+
 			return &portainer.HelmUserRepository{}, nil
 		})
 

api/dataservices/interface.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/edgetypes"
 
 	portainer "github.com/portainer/portainer/api"
 )
@@ -28,6 +29,7 @@ type (
 		EdgeGroup() EdgeGroupService
 		EdgeJob() EdgeJobService
 		EdgeStack() EdgeStackService
+		EdgeUpdateSchedule() EdgeUpdateScheduleService
 		Endpoint() EndpointService
 		EndpointGroup() EndpointGroupService
 		EndpointRelation() EndpointRelationService
@@ -38,6 +40,7 @@ type (
 		Role() RoleService
 		APIKeyRepository() APIKeyRepository
 		Settings() SettingsService
+		Snapshot() SnapshotService
 		SSLSettings() SSLSettingsService
 		Stack() StackService
 		Tag() TagService
@@ -81,6 +84,17 @@ type (
 		BucketName() string
 	}
 
+	EdgeUpdateScheduleService interface {
+		ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation
+		ActiveSchedules(environmentIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation
+		List() ([]edgetypes.UpdateSchedule, error)
+		Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error)
+		Create(edgeUpdateSchedule *edgetypes.UpdateSchedule) error
+		Update(ID edgetypes.UpdateScheduleID, edgeUpdateSchedule *edgetypes.UpdateSchedule) error
+		Delete(ID edgetypes.UpdateScheduleID) error
+		BucketName() string
+	}
+
 	// EdgeStackService represents a service to manage Edge stacks
 	EdgeStackService interface {
 		EdgeStacks() ([]portainer.EdgeStack, error)
@@ -201,6 +215,15 @@ type (
 		BucketName() string
 	}
 
+	SnapshotService interface {
+		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
+		Snapshots() ([]portainer.Snapshot, error)
+		UpdateSnapshot(snapshot *portainer.Snapshot) error
+		DeleteSnapshot(endpointID portainer.EndpointID) error
+		Create(snapshot *portainer.Snapshot) error
+		BucketName() string
+	}
+
 	// SSLSettingsService represents a service for managing application settings
 	SSLSettingsService interface {
 		Settings() (*portainer.SSLSettings, error)

api/dataservices/registry/registry.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 		func(obj interface{}) (interface{}, error) {
 			registry, ok := obj.(*portainer.Registry)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Registry object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
 				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
 			}
+
 			registries = append(registries, *registry)
+
 			return &portainer.Registry{}, nil
 		})
 

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -58,7 +59,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
 				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
 			}
 
@@ -73,6 +74,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 					return nil, stop
 				}
 			}
+
 			return &portainer.ResourceControl{}, nil
 		})
 	if err == stop {
@@ -92,10 +94,12 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
 				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
 			}
+
 			rcs = append(rcs, *rc)
+
 			return &portainer.ResourceControl{}, nil
 		})
 

api/dataservices/role/role.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 		func(obj interface{}) (interface{}, error) {
 			set, ok := obj.(*portainer.Role)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Role object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
 				return nil, fmt.Errorf("Failed to convert to Role object: %s", obj)
 			}
+
 			sets = append(sets, *set)
+
 			return &portainer.Role{}, nil
 		})
 

api/dataservices/schedule/schedule.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -68,10 +69,12 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 		func(obj interface{}) (interface{}, error) {
 			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
 			}
+
 			schedules = append(schedules, *schedule)
+
 			return &portainer.Schedule{}, nil
 		})
 
@@ -89,12 +92,14 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 		func(obj interface{}) (interface{}, error) {
 			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
 			}
+
 			if schedule.JobType == jobType {
 				schedules = append(schedules, *schedule)
 			}
+
 			return &portainer.Schedule{}, nil
 		})
 

api/dataservices/snapshot/snapshot.go

@@ -0,0 +1,76 @@
+package snapshot
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	BucketName = "snapshots"
+)
+
+type Service struct {
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot portainer.Snapshot
+	identifier := service.connection.ConvertToKey(int(endpointID))
+
+	err := service.connection.GetObject(BucketName, identifier, &snapshot)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
+	var snapshots = make([]portainer.Snapshot, 0)
+
+	err := service.connection.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Snapshot{},
+		func(obj interface{}) (interface{}, error) {
+			snapshot, ok := obj.(*portainer.Snapshot)
+			if !ok {
+				logrus.WithField("obj", obj).Errorf("Failed to convert to Snapshot object")
+				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
+			}
+			snapshots = append(snapshots, *snapshot)
+			return &portainer.Snapshot{}, nil
+		})
+
+	return snapshots, err
+}
+
+func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
+	identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
+	return service.connection.UpdateObject(BucketName, identifier, snapshot)
+}
+
+func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
+	identifier := service.connection.ConvertToKey(int(endpointID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+func (service *Service) Create(snapshot *portainer.Snapshot) error {
+	return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+}

api/dataservices/stack/stack.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -60,13 +60,15 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.Name == name {
 				s = stack
 				return nil, stop
 			}
+
 			return &portainer.Stack{}, nil
 		})
 	if err == stop {
@@ -89,12 +91,14 @@ func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.Name == name {
 				stacks = append(stacks, stack)
 			}
+
 			return &portainer.Stack{}, nil
 		})
 
@@ -111,10 +115,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			stacks = append(stacks, *stack)
+
 			return &portainer.Stack{}, nil
 		})
 
@@ -156,13 +162,15 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 			s, ok = obj.(*portainer.Stack)
 
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+
 				return &portainer.Stack{}, nil
 			}
 
 			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
 				return nil, stop
 			}
+
 			return &portainer.Stack{}, nil
 		})
 	if err == stop {
@@ -186,12 +194,14 @@ func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
 				stacks = append(stacks, *stack)
 			}
+
 			return &portainer.Stack{}, nil
 		})
 

api/dataservices/stack/tests/stack_test.go

@@ -29,7 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -87,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}

api/dataservices/tag/tag.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -43,10 +44,12 @@ func (service *Service) Tags() ([]portainer.Tag, error) {
 		func(obj interface{}) (interface{}, error) {
 			tag, ok := obj.(*portainer.Tag)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Tag object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
 				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
 			}
+
 			tags = append(tags, *tag)
+
 			return &portainer.Tag{}, nil
 		})
 

api/dataservices/team/team.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -60,13 +60,15 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 		func(obj interface{}) (interface{}, error) {
 			team, ok := obj.(*portainer.Team)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Team object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
 				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
 			}
+
 			if strings.EqualFold(team.Name, name) {
 				t = team
 				return nil, stop
 			}
+
 			return &portainer.Team{}, nil
 		})
 	if err == stop {
@@ -89,10 +91,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 		func(obj interface{}) (interface{}, error) {
 			team, ok := obj.(*portainer.Team)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Team object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
 				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
 			}
+
 			teams = append(teams, *team)
+
 			return &portainer.Team{}, nil
 		})
 

api/dataservices/team/tests/team_test.go

@@ -10,7 +10,7 @@ import (
 
 func Test_teamByName(t *testing.T) {
 	t.Run("When store is empty should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
 		defer teardown()
 
 		_, err := store.Team().TeamByName("name")
@@ -19,7 +19,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is no object with the same name should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
 		defer teardown()
 
 		teamBuilder := teamBuilder{
@@ -35,7 +35,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is an object with the same name should return the object", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
 		defer teardown()
 
 		teamBuilder := teamBuilder{

api/dataservices/teammembership/teammembership.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			memberships = append(memberships, *membership)
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -76,12 +79,14 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			if membership.UserID == userID {
 				memberships = append(memberships, *membership)
 			}
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -98,12 +103,14 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			if membership.TeamID == teamID {
 				memberships = append(memberships, *membership)
 			}
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -140,13 +147,15 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 				return -1, false
 			}
+
 			if membership.UserID == userID {
 				return int(membership.ID), true
 			}
+
 			return -1, false
 		})
 }
@@ -158,13 +167,15 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 				return -1, false
 			}
+
 			if membership.TeamID == teamID {
 				return int(membership.ID), true
 			}
+
 			return -1, false
 		})
 }

api/dataservices/user/user.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -59,18 +59,23 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			if strings.EqualFold(user.Username, username) {
 				u = user
 				return nil, stop
 			}
+
 			return &portainer.User{}, nil
 		})
+
 	if err == stop {
 		return u, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}
@@ -88,10 +93,13 @@ func (service *Service) Users() ([]portainer.User, error) {
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			users = append(users, *user)
+
 			return &portainer.User{}, nil
 		})
 
@@ -108,12 +116,15 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			if user.Role == role {
 				users = append(users, *user)
 			}
+
 			return &portainer.User{}, nil
 		})
 

api/dataservices/webhook/webhook.go

@@ -5,7 +5,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -34,7 +35,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//Webhooks returns an array of all webhooks
+// Webhooks returns an array of all webhooks
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
@@ -44,10 +45,12 @@ func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			webhooks = append(webhooks, *webhook)
+
 			return &portainer.Webhook{}, nil
 		})
 
@@ -77,18 +80,23 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			if webhook.ResourceID == ID {
 				w = webhook
 				return nil, stop
 			}
+
 			return &portainer.Webhook{}, nil
 		})
+
 	if err == stop {
 		return w, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}
@@ -106,18 +114,23 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			if webhook.Token == token {
 				w = webhook
 				return nil, stop
 			}
+
 			return &portainer.Webhook{}, nil
 		})
+
 	if err == stop {
 		return w, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}

api/datastore/backup.go

@@ -6,7 +6,7 @@ import (
 	"path"
 	"time"
 
-	plog "github.com/portainer/portainer/api/datastore/log"
+	"github.com/rs/zerolog/log"
 )
 
 var backupDefaults = struct {
@@ -17,8 +17,6 @@ var backupDefaults = struct {
 	"common",
 }
 
-var backupLog = plog.NewScopedLog("database, backup")
-
 //
 // Backup Helpers
 //
@@ -29,7 +27,7 @@ func (store *Store) createBackupFolders() {
 	commonDir := store.commonBackupDir()
 	if exists, _ := store.fileService.FileExists(commonDir); !exists {
 		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			backupLog.Error("Error while creating common backup folder", err)
+			log.Error().Err(err).Msg("error while creating common backup folder")
 		}
 	}
 }
@@ -43,11 +41,13 @@ func (store *Store) commonBackupDir() string {
 }
 
 func (store *Store) copyDBFile(from string, to string) error {
-	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
+	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
+
 	err := store.fileService.Copy(from, to, true)
 	if err != nil {
-		backupLog.Error("Failed", err)
+		log.Error().Err(err).Msg("failed")
 	}
+
 	return err
 }
 
@@ -99,7 +99,8 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 
 // BackupWithOptions backup current database with options
 func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
-	backupLog.Info("creating db backup")
+	log.Info().Msg("creating DB backup")
+
 	store.createBackupFolders()
 
 	options = store.setupOptions(options)
@@ -122,6 +123,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 			err,
 		)
 	}
+
 	return options.BackupPath, nil
 }
 
@@ -135,17 +137,19 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 	// Check if backup file exist before restoring
 	_, err := os.Stat(options.BackupPath)
 	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
+
 		return err
 	}
 
 	err = store.Close()
 	if err != nil {
-		backupLog.Error("Error while closing store before restore", err)
+		log.Error().Err(err).Msg("error while closing store before restore")
+
 		return err
 	}
 
-	backupLog.Info("Restoring db backup")
+	log.Info().Msg("restoring DB backup")
 	err = store.copyDBFile(options.BackupPath, store.databasePath())
 	if err != nil {
 		return err
@@ -157,20 +161,22 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 
 // RemoveWithOptions removes backup database based on supplied options
 func (store *Store) removeWithOptions(options *BackupOptions) error {
-	backupLog.Info("Removing db backup")
+	log.Info().Msg("removing DB backup")
 
 	options = store.setupOptions(options)
 	_, err := os.Stat(options.BackupPath)
 
 	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
+
 		return err
 	}
 
-	backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
+	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
 	err = os.Remove(options.BackupPath)
 	if err != nil {
-		backupLog.Error("Failed", err)
+		log.Error().Err(err).Msg("failed")
+
 		return err
 	}
 

api/datastore/backup_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false, true)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	connection := store.GetConnection()
@@ -27,7 +27,7 @@ func TestCreateBackupFolders(t *testing.T) {
 }
 
 func TestStoreCreation(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true, true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	if store == nil {
@@ -40,7 +40,7 @@ func TestStoreCreation(t *testing.T) {
 }
 
 func TestBackup(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true, true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	connection := store.GetConnection()
 	defer teardown()
 
@@ -67,7 +67,7 @@ func TestBackup(t *testing.T) {
 }
 
 func TestRemoveWithOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true, true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	t.Run("successfully removes file if existent", func(t *testing.T) {

api/datastore/datastore.go

@@ -9,7 +9,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (store *Store) version() (int, error) {
@@ -73,7 +74,8 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	if version > 0 {
-		logrus.WithField("version", version).Infof("Opened existing store")
+		log.Debug().Int("version", version).Msg("opened existing store")
+
 		return false, nil
 	}
 
@@ -121,19 +123,21 @@ func (store *Store) encryptDB() error {
 
 	// The DB is not currently encrypted.  First save the encrypted db filename
 	oldFilename := store.connection.GetDatabaseFilePath()
-	logrus.Infof("Encrypting database")
+	log.Info().Msg("encrypting database")
 
 	// export file path for backup
 	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
 
-	logrus.Infof("Exporting database backup to %s", exportFilename)
+	log.Info().Str("filename", exportFilename).Msg("exporting database backup")
+
 	err = store.Export(exportFilename)
 	if err != nil {
-		logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
+		log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
+
 		return err
 	}
 
-	logrus.Infof("Database backup exported")
+	log.Info().Msg("database backup exported")
 
 	// Close existing un-encrypted db so that we can delete the file later
 	store.connection.Close()
@@ -152,22 +156,23 @@ func (store *Store) encryptDB() error {
 	if err != nil {
 		// Remove the new encrypted file that we failed to import
 		os.Remove(store.connection.GetDatabaseFilePath())
-		logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
+		log.Fatal().Err(portainerErrors.ErrDBImportFailed).Msg("")
 	}
 
 	err = os.Remove(oldFilename)
 	if err != nil {
-		logrus.Errorf("Failed to remove the un-encrypted db file")
+		log.Error().Msg("failed to remove the un-encrypted db file")
 	}
 
 	err = os.Remove(exportFilename)
 	if err != nil {
-		logrus.Errorf("Failed to remove the json backup file")
+		log.Error().Msg("failed to remove the json backup file")
 	}
 
 	// Close db connection
 	store.connection.Close()
 
-	logrus.Info("Database successfully encrypted")
+	log.Info().Msg("database successfully encrypted")
+
 	return nil
 }

api/datastore/datastore_test.go

@@ -27,7 +27,7 @@ const (
 // TestStoreFull an eventually comprehensive set of tests for the Store.
 // The idea is what we write to the store, we should read back.
 func TestStoreFull(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true, true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	testCases := map[string]func(t *testing.T){

api/datastore/init.go

@@ -44,7 +44,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 	settings, err := store.SettingsService.Settings()
 	if store.IsErrObjectNotFound(err) {
 		defaultSettings := &portainer.Settings{
-			EnableTelemetry:      true,
+			EnableTelemetry:      false,
 			AuthenticationMethod: portainer.AuthenticationInternal,
 			BlackListedLabels:    make([]portainer.Pair, 0),
 			InternalAuthSettings: portainer.InternalAuthSettings{

api/datastore/log/log.go

@@ -1,51 +0,0 @@
-package log
-
-import (
-	"fmt"
-	"log"
-)
-
-const (
-	INFO  = "INFO"
-	ERROR = "ERROR"
-	DEBUG = "DEBUG"
-	FATAL = "FATAL"
-)
-
-type ScopedLog struct {
-	scope string
-}
-
-func NewScopedLog(scope string) *ScopedLog {
-	return &ScopedLog{scope: scope}
-}
-
-func (slog *ScopedLog) print(kind string, message string) {
-	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
-}
-
-func (slog *ScopedLog) Debug(message string) {
-	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Debugf(message string, vars ...interface{}) {
-	message = fmt.Sprintf(message, vars...)
-	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Info(message string) {
-	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Infof(message string, vars ...interface{}) {
-	message = fmt.Sprintf(message, vars...)
-	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Error(message string, err error) {
-	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
-}
-
-func (slog *ScopedLog) NotImplemented(method string) {
-	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
-}

api/datastore/migrate_data.go

@@ -4,21 +4,18 @@ import (
 	"fmt"
 	"runtime/debug"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	plog "github.com/portainer/portainer/api/datastore/log"
 	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/sirupsen/logrus"
 
 	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
 )
 
 const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
 
-var migrateLog = plog.NewScopedLog("database, migrate")
-
 func (store *Store) MigrateData() error {
 	version, err := store.version()
 	if err != nil {
@@ -43,6 +40,7 @@ func (store *Store) MigrateData() error {
 		RoleService:             store.RoleService,
 		ScheduleService:         store.ScheduleService,
 		SettingsService:         store.SettingsService,
+		SnapshotService:         store.SnapshotService,
 		StackService:            store.StackService,
 		TagService:              store.TagService,
 		TeamMembershipService:   store.TeamMembershipService,
@@ -56,20 +54,19 @@ func (store *Store) MigrateData() error {
 	// restore on error
 	err = store.connectionMigrateData(migratorParams)
 	if err != nil {
-		logrus.Errorf("While DB migration %v. Restoring DB", err)
+		log.Error().Err(err).Msg("while DB migration, restoring DB")
+
 		// Restore options
 		options := BackupOptions{
 			BackupPath: backupPath,
 		}
+
 		err := store.restoreWithOptions(&options)
 		if err != nil {
-			logrus.Fatalf(
-				"Failed restoring the backup. portainer database file needs to restored manually by "+
-					"replacing %s database file with recent backup %s. Error %v",
-				store.databasePath(),
-				options.BackupPath,
-				err,
-			)
+			log.Fatal().
+				Str("database_file", store.databasePath()).
+				Str("backup", options.BackupPath).Err(err).
+				Msg("failed restoring the backup, Portainer database file needs to restored manually by replacing the database file with a recent backup")
 		}
 	}
 
@@ -111,10 +108,15 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
 	}
 
 	if migrator.Version() < portainer.DBVersion {
-		migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
+		log.Info().
+			Int("migrator_version", migrator.Version()).
+			Int("db_version", portainer.DBVersion).
+			Msg("migrating database")
+
 		err = store.FailSafeMigrate(migrator)
 		if err != nil {
-			migrateLog.Error("An error occurred during database migration", err)
+			log.Error().Err(err).Msg("an error occurred during database migration")
+
 			return err
 		}
 	}
@@ -124,17 +126,19 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
 
 // backupVersion will backup the database or panic if any errors occur
 func (store *Store) backupVersion(migrator *migrator.Migrator) error {
-	migrateLog.Info("Backing up database prior to version upgrade...")
+	log.Info().Msg("backing up database prior to version upgrade")
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
 
 	_, err := store.backupWithOptions(options)
 	if err != nil {
-		migrateLog.Error("An error occurred during database backup", err)
+		log.Error().Err(err).Msg("an error occurred during database backup")
+
 		removalErr := store.removeWithOptions(options)
 		if removalErr != nil {
-			migrateLog.Error("An error occurred during store removal prior to backup", err)
+			log.Error().Err(err).Msg("an error occurred during store removal prior to backup")
 		}
+
 		return err
 	}
 

api/datastore/migrate_data_test.go

@@ -5,15 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/rs/zerolog/log"
 )
 
 // testVersion is a helper which tests current store version against wanted version
@@ -53,7 +54,7 @@ func TestMigrateData(t *testing.T) {
 	}
 
 	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store, teardown := MustNewTestStore(false, true)
+		newStore, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		if !newStore {
@@ -80,7 +81,7 @@ func TestMigrateData(t *testing.T) {
 		{version: 21, expectedVersion: portainer.DBVersion},
 	}
 	for _, tc := range tests {
-		_, store, teardown := MustNewTestStore(true, true)
+		_, store, teardown := MustNewTestStore(t, true, true)
 		defer teardown()
 
 		// Setup data
@@ -105,7 +106,7 @@ func TestMigrateData(t *testing.T) {
 	}
 
 	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false, true)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		version := 17
@@ -117,7 +118,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false, true)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(0)
 
@@ -131,7 +132,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false, true)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		store.VersionService.StoreIsUpdating(true)
@@ -146,7 +147,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false, true)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		store.MigrateData()
@@ -157,48 +158,48 @@ func TestMigrateData(t *testing.T) {
 			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
 		}
 	})
-
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false, true)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
 
 	wantDir := store.commonBackupDir()
 	if !strings.HasSuffix(options.BackupDir, wantDir) {
-		log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
+		log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir")
 	}
 
 	wantFilename := "portainer.db.bak"
 	if options.BackupFileName != wantFilename {
-		log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
+		log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file")
 	}
 }
 
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
 		version := 21
-		_, store, teardown := MustNewTestStore(false, true)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(version)
 
 		_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
 		if err != nil {
-			log.Fatal(err)
+			log.Fatal().Err(err).Msg("")
 		}
 
 		// Change the current edition
 		err = store.VersionService.StoreDBVersion(version + 10)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatal().Err(err).Msg("")
 		}
 
 		err = store.Rollback(true)
 		if err != nil {
 			t.Logf("Rollback failed: %s", err)
 			t.Fail()
+
 			return
 		}
 
@@ -226,7 +227,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
 	}
 
 	// Parse source json to db.
-	_, store, teardown := MustNewTestStore(true, false)
+	_, store, teardown := MustNewTestStore(t, true, false)
 	defer teardown()
 	err = importJSON(t, bytes.NewReader(srcJSON), store)
 	if err != nil {

api/datastore/migrate_dbversion29_test.go

@@ -33,7 +33,7 @@ func setup(store *Store) error {
 }
 
 func TestMigrateSettings(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false, true)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	err := setup(store)

api/datastore/migrate_dbversion33_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false, true)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	stackService := store.Stack()

api/datastore/migrator/migrate_ce.go

@@ -5,8 +5,10 @@ import (
 	"reflect"
 	"runtime"
 
-	werrors "github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
+
+	werrors "github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 type migration struct {
@@ -106,6 +108,12 @@ func (m *Migrator) Migrate() error {
 
 		// Portainer 2.15
 		newMigration(60, m.migrateDBVersionToDB60),
+
+		// Portainer 2.16
+		newMigration(70, m.migrateDBVersionToDB70),
+
+		// Portainer 2.16.1
+		newMigration(71, m.migrateDBVersionToDB71),
 	}
 
 	var lastDbVersion int
@@ -114,7 +122,7 @@ func (m *Migrator) Migrate() error {
 
 			// Print the next line only when the version changes
 			if migration.dbversion > lastDbVersion {
-				migrateLog.Infof("Migrating DB to version %d", migration.dbversion)
+				log.Info().Int("to_version", migration.dbversion).Msg("migrating DB")
 			}
 
 			err := migration.migrate()
@@ -125,12 +133,14 @@ func (m *Migrator) Migrate() error {
 		lastDbVersion = migration.dbversion
 	}
 
-	migrateLog.Infof("Setting DB version to %d", portainer.DBVersion)
+	log.Info().Int("version", portainer.DBVersion).Msg("setting DB version")
+
 	err = m.versionService.StoreDBVersion(portainer.DBVersion)
 	if err != nil {
 		return migrationError(err, "StoreDBVersion")
 	}
-	migrateLog.Infof("Updated DB version to %d", portainer.DBVersion)
+
+	log.Info().Int("version", portainer.DBVersion).Msg("updated DB version")
 
 	// reset DB updating status
 	return m.versionService.StoreIsUpdating(false)

api/datastore/migrator/migrate_dbversion17.go

@@ -2,10 +2,13 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateUsersToDBVersion18() error {
-	migrateLog.Info("- updating users")
+	log.Info().Msg("updating users")
+
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err
@@ -40,7 +43,8 @@ func (m *Migrator) updateUsersToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointsToDBVersion18() error {
-	migrateLog.Info("- updating endpoints")
+	log.Info().Msg("updating endpoints")
+
 	legacyEndpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
@@ -71,7 +75,8 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
-	migrateLog.Info("- updating endpoint groups")
+	log.Info().Msg("updating endpoint groups")
+
 	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
@@ -102,7 +107,8 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 }
 
 func (m *Migrator) updateRegistriesToDBVersion18() error {
-	migrateLog.Info("- updating registries")
+	log.Info().Msg("updating registries")
+
 	legacyRegistries, err := m.registryService.Registries()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion18.go

@@ -1,9 +1,14 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) updateSettingsToDBVersion19() error {
-	migrateLog.Info("- updating settings")
+	log.Info().Msg("updating settings")
+
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion19.go

@@ -2,12 +2,15 @@ package migrator
 
 import (
 	"strings"
+
+	"github.com/rs/zerolog/log"
 )
 
 const scheduleScriptExecutionJobType = 1
 
 func (m *Migrator) updateUsersToDBVersion20() error {
-	migrateLog.Info("- updating user authentication")
+	log.Info().Msg("updating user authentication")
+
 	return m.authorizationService.UpdateUsersAuthorizations()
 }
 
@@ -23,7 +26,8 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
 }
 
 func (m *Migrator) updateSchedulesToDBVersion20() error {
-	migrateLog.Info("- updating schedules")
+	log.Info().Msg("updating schedules")
+
 	legacySchedules, err := m.scheduleService.Schedules()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion20.go

@@ -3,10 +3,13 @@ package migrator
 import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/authorization"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateResourceControlsToDBVersion22() error {
-	migrateLog.Info("- updating resource controls")
+	log.Info().Msg("updating resource controls")
+
 	legacyResourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err
@@ -25,7 +28,8 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 }
 
 func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
-	migrateLog.Info("- updating users and roles")
+	log.Info().Msg("updating users and roles")
+
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion22.go

@@ -1,9 +1,14 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) updateTagsToDBVersion23() error {
-	migrateLog.Info("- Updating tags")
+	log.Info().Msg("updating tags")
+
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -21,7 +26,8 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 }
 
 func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
-	migrateLog.Info("- updating endpoints and endpoint groups")
+	log.Info().Msg("updating endpoints and endpoint groups")
+
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -90,5 +96,6 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 			return err
 		}
 	}
+
 	return nil
 }

api/datastore/migrator/migrate_dbversion23.go

@@ -1,9 +1,13 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) updateSettingsToDB24() error {
-	migrateLog.Info("- updating Settings")
+	log.Info().Msg("updating Settings")
 
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
@@ -18,7 +22,8 @@ func (m *Migrator) updateSettingsToDB24() error {
 }
 
 func (m *Migrator) updateStacksToDB24() error {
-	migrateLog.Info("- updating stacks")
+	log.Info().Msg("updating stacks")
+
 	stacks, err := m.stackService.Stacks()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion24.go

@@ -2,10 +2,12 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateSettingsToDB25() error {
-	migrateLog.Info("- updating settings")
+	log.Info().Msg("updating settings")
 
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {

api/datastore/migrator/migrate_dbversion25.go

@@ -2,10 +2,13 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateEndpointSettingsToDB25() error {
-	migrateLog.Info("- updating endpoint settings")
+	log.Info().Msg("updating endpoint settings")
+
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion26.go

@@ -4,10 +4,13 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
 	"github.com/portainer/portainer/api/internal/stackutils"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateStackResourceControlToDB27() error {
-	migrateLog.Info("- updating stack resource controls")
+	log.Info().Msg("updating stack resource controls")
+
 	resourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion29.go

@@ -1,12 +1,11 @@
 package migrator
 
-func (m *Migrator) migrateDBVersionToDB30() error {
-	migrateLog.Info("- updating legacy settings")
-	if err := m.MigrateSettingsToDB30(); err != nil {
-		return err
-	}
+import "github.com/rs/zerolog/log"
 
-	return nil
+func (m *Migrator) migrateDBVersionToDB30() error {
+	log.Info().Msg("updating legacy settings")
+
+	return m.MigrateSettingsToDB30()
 }
 
 // so setting to false and "", is what would happen without this code

api/datastore/migrator/migrate_dbversion31.go

@@ -2,14 +2,14 @@ package migrator
 
 import (
 	"fmt"
-	"log"
-
-	"github.com/docker/docker/api/types/volume"
-	"github.com/portainer/portainer/api/dataservices/errors"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
+
+	"github.com/docker/docker/api/types/volume"
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB32() error {
@@ -39,7 +39,8 @@ func (m *Migrator) migrateDBVersionToDB32() error {
 }
 
 func (m *Migrator) updateRegistriesToDB32() error {
-	migrateLog.Info("- updating registries")
+	log.Info().Msg("updating registries")
+
 	registries, err := m.registryService.Registries()
 	if err != nil {
 		return err
@@ -82,7 +83,8 @@ func (m *Migrator) updateRegistriesToDB32() error {
 }
 
 func (m *Migrator) updateDockerhubToDB32() error {
-	migrateLog.Info("- updating dockerhub")
+	log.Info().Msg("updating dockerhub")
+
 	dockerhub, err := m.dockerhubService.DockerHub()
 	if err == errors.ErrObjectNotFound {
 		return nil
@@ -171,7 +173,8 @@ func (m *Migrator) updateDockerhubToDB32() error {
 }
 
 func (m *Migrator) updateVolumeResourceControlToDB32() error {
-	migrateLog.Info("- updating resource controls")
+	log.Info().Msg("updating resource controls")
+
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return fmt.Errorf("failed fetching environments: %w", err)
@@ -199,7 +202,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		totalSnapshots := len(endpoint.Snapshots)
 		if totalSnapshots == 0 {
-			log.Println("[DEBUG] [volume migration] [message: no snapshot found]")
+			log.Debug().Msg("no snapshot found")
 			continue
 		}
 
@@ -207,13 +210,13 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
 		if err != nil {
-			log.Printf("[WARN] [database,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
+			log.Warn().Err(err).Msg("failed fetching environment docker id")
 			continue
 		}
 
 		volumesData := snapshot.SnapshotRaw.Volumes
 		if volumesData.Volumes == nil {
-			log.Println("[DEBUG] [volume migration] [message: no volume data found]")
+			log.Debug().Msg("no volume data found")
 			continue
 		}
 
@@ -224,17 +227,18 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 	for _, resourceControl := range volumeResourceControls {
 		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
 			resourceControl.ResourceID = newResourceID
+
 			err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
 			if err != nil {
 				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
 			}
-
 		} else {
 			err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
 			if err != nil {
 				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
 			}
-			log.Printf("[DEBUG] [volume migration] [message: legacy resource control(%s) has been deleted]", resourceControl.ResourceID)
+
+			log.Debug().Str("resource_id", resourceControl.ResourceID).Msg("legacy resource control has been deleted")
 		}
 	}
 
@@ -257,21 +261,25 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData volume.VolumeList
 }
 
 func (m *Migrator) kubeconfigExpiryToDB32() error {
-	migrateLog.Info("- updating kubeconfig expiry")
+	log.Info().Msg("updating kubeconfig expiry")
+
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err
 	}
+
 	settings.KubeconfigExpiry = portainer.DefaultKubeconfigExpiry
 	return m.settingsService.UpdateSettings(settings)
 }
 
 func (m *Migrator) helmRepositoryURLToDB32() error {
-	migrateLog.Info("- setting default helm repository URL")
+	log.Info().Msg("setting default helm repository URL")
+
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err
 	}
+
 	settings.HelmRepositoryURL = portainer.DefaultHelmRepositoryURL
 	return m.settingsService.UpdateSettings(settings)
 }

api/datastore/migrator/migrate_dbversion32.go

@@ -2,15 +2,14 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB33() error {
-	migrateLog.Info("- updating settings")
-	if err := m.migrateSettingsToDB33(); err != nil {
-		return err
-	}
+	log.Info().Msg("updating settings")
 
-	return nil
+	return m.migrateSettingsToDB33()
 }
 
 func (m *Migrator) migrateSettingsToDB33() error {
@@ -19,7 +18,8 @@ func (m *Migrator) migrateSettingsToDB33() error {
 		return err
 	}
 
-	migrateLog.Info("- setting default kubectl shell image")
+	log.Info().Msg("setting default kubectl shell image")
 	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
+
 	return m.settingsService.UpdateSettings(settings)
 }

api/datastore/migrator/migrate_dbversion33.go

@@ -2,16 +2,14 @@ package migrator
 
 import (
 	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB34() error {
-	migrateLog.Info("- updating stacks")
-	err := MigrateStackEntryPoint(m.stackService)
-	if err != nil {
-		return err
-	}
+	log.Info().Msg("updating stacks")
 
-	return nil
+	return MigrateStackEntryPoint(m.stackService)
 }
 
 // MigrateStackEntryPoint exported for testing (blah.)
@@ -20,15 +18,18 @@ func MigrateStackEntryPoint(stackService dataservices.StackService) error {
 	if err != nil {
 		return err
 	}
+
 	for i := range stacks {
 		stack := &stacks[i]
 		if stack.GitConfig == nil {
 			continue
 		}
+
 		stack.GitConfig.ConfigFilePath = stack.EntryPoint
 		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
 			return err
 		}
 	}
+
 	return nil
 }

api/datastore/migrator/migrate_dbversion34.go

@@ -1,12 +1,12 @@
 package migrator
 
+import "github.com/rs/zerolog/log"
+
 func (m *Migrator) migrateDBVersionToDB35() error {
 	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
 	// calling it again will now fix the issue as the function has been repaired.
-	migrateLog.Info("- updating dockerhub registries")
-	err := m.updateDockerhubToDB32()
-	if err != nil {
-		return err
-	}
-	return nil
+
+	log.Info().Msg("updating dockerhub registries")
+
+	return m.updateDockerhubToDB32()
 }

api/datastore/migrator/migrate_dbversion35.go

@@ -3,15 +3,14 @@ package migrator
 import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/authorization"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB36() error {
-	migrateLog.Info("Updating user authorizations")
-	if err := m.migrateUsersToDB36(); err != nil {
-		return err
-	}
+	log.Info().Msg("updating user authorizations")
 
-	return nil
+	return m.migrateUsersToDB36()
 }
 
 func (m *Migrator) migrateUsersToDB36() error {

api/datastore/migrator/migrate_dbversion40.go

@@ -1,17 +1,18 @@
 package migrator
 
-import "github.com/portainer/portainer/api/internal/endpointutils"
+import (
+	"github.com/portainer/portainer/api/internal/endpointutils"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) migrateDBVersionToDB40() error {
-	if err := m.trustCurrentEdgeEndpointsDB40(); err != nil {
-		return err
-	}
-
-	return nil
+	return m.trustCurrentEdgeEndpointsDB40()
 }
 
 func (m *Migrator) trustCurrentEdgeEndpointsDB40() error {
-	migrateLog.Info("- trusting current edge endpoints")
+	log.Info().Msg("trusting current edge endpoints")
+
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion50.go

@@ -2,6 +2,7 @@ package migrator
 
 import (
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB50() error {
@@ -9,7 +10,8 @@ func (m *Migrator) migrateDBVersionToDB50() error {
 }
 
 func (m *Migrator) migratePasswordLengthSettings() error {
-	migrateLog.Info("Updating required password length")
+	log.Info().Msg("updating required password length")
+
 	s, err := m.settingsService.Settings()
 	if err != nil {
 		return errors.Wrap(err, "unable to retrieve settings")

api/datastore/migrator/migrate_dbversion60.go

@@ -1,17 +1,18 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) migrateDBVersionToDB60() error {
-	if err := m.addGpuInputFieldDB60(); err != nil {
-		return err
-	}
-
-	return nil
+	return m.addGpuInputFieldDB60()
 }
 
 func (m *Migrator) addGpuInputFieldDB60() error {
-	migrateLog.Info("- add gpu input field")
+	log.Info().Msg("add gpu input field")
+
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion70.go

@@ -0,0 +1,70 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDBVersionToDB70() error {
+	log.Info().Msg("- add IngressAvailabilityPerNamespace field")
+	if err := m.updateIngressFieldsForEnvDB70(); err != nil {
+		return err
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		// copy snapshots to new object
+		log.Info().Msg("moving snapshots from endpoint to new object")
+		snapshot := portainer.Snapshot{EndpointID: endpoint.ID}
+
+		if len(endpoint.Snapshots) > 0 {
+			snapshot.Docker = &endpoint.Snapshots[len(endpoint.Snapshots)-1]
+		}
+
+		if len(endpoint.Kubernetes.Snapshots) > 0 {
+			snapshot.Kubernetes = &endpoint.Kubernetes.Snapshots[len(endpoint.Kubernetes.Snapshots)-1]
+		}
+
+		// save new object
+		err = m.snapshotService.Create(&snapshot)
+		if err != nil {
+			return err
+		}
+
+		// set to nil old fields
+		log.Info().Msg("deleting snapshot from endpoint")
+		endpoint.Snapshots = []portainer.DockerSnapshot{}
+		endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{}
+
+		// update endpoint
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateIngressFieldsForEnvDB70() error {
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		endpoint.Kubernetes.Configuration.IngressAvailabilityPerNamespace = true
+		endpoint.Kubernetes.Configuration.AllowNoneIngressClass = false
+		endpoint.PostInitMigrations.MigrateIngresses = true
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrate_dbversion71.go

@@ -0,0 +1,36 @@
+package migrator
+
+import (
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDBVersionToDB71() error {
+	log.Info().Msg("removing orphaned snapshots")
+
+	snapshots, err := m.snapshotService.Snapshots()
+	if err != nil {
+		return err
+	}
+
+	for _, s := range snapshots {
+		_, err := m.endpointService.Endpoint(s.EndpointID)
+		if err == nil {
+			log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("keeping snapshot")
+			continue
+		} else if err != errors.ErrObjectNotFound {
+			log.Debug().Int("endpoint_id", int(s.EndpointID)).Err(err).Msg("database error")
+			return err
+		}
+
+		log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("removing snapshot")
+
+		err = m.snapshotService.DeleteSnapshot(s.EndpointID)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -13,17 +13,15 @@ import (
 	"github.com/portainer/portainer/api/dataservices/role"
 	"github.com/portainer/portainer/api/dataservices/schedule"
 	"github.com/portainer/portainer/api/dataservices/settings"
+	"github.com/portainer/portainer/api/dataservices/snapshot"
 	"github.com/portainer/portainer/api/dataservices/stack"
 	"github.com/portainer/portainer/api/dataservices/tag"
 	"github.com/portainer/portainer/api/dataservices/teammembership"
 	"github.com/portainer/portainer/api/dataservices/user"
 	"github.com/portainer/portainer/api/dataservices/version"
-	plog "github.com/portainer/portainer/api/datastore/log"
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
-var migrateLog = plog.NewScopedLog("database, migrate")
-
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
@@ -38,6 +36,7 @@ type (
 		roleService             *role.Service
 		scheduleService         *schedule.Service
 		settingsService         *settings.Service
+		snapshotService         *snapshot.Service
 		stackService            *stack.Service
 		tagService              *tag.Service
 		teamMembershipService   *teammembership.Service
@@ -61,6 +60,7 @@ type (
 		RoleService             *role.Service
 		ScheduleService         *schedule.Service
 		SettingsService         *settings.Service
+		SnapshotService         *snapshot.Service
 		StackService            *stack.Service
 		TagService              *tag.Service
 		TeamMembershipService   *teammembership.Service
@@ -86,6 +86,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		roleService:             parameters.RoleService,
 		scheduleService:         parameters.ScheduleService,
 		settingsService:         parameters.SettingsService,
+		snapshotService:         parameters.SnapshotService,
 		tagService:              parameters.TagService,
 		teamMembershipService:   parameters.TeamMembershipService,
 		stackService:            parameters.StackService,

api/datastore/services.go

@@ -2,6 +2,7 @@ package datastore
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strconv"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/edgegroup"
 	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"github.com/portainer/portainer/api/dataservices/edgestack"
+	"github.com/portainer/portainer/api/dataservices/edgeupdateschedule"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
@@ -24,6 +26,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/role"
 	"github.com/portainer/portainer/api/dataservices/schedule"
 	"github.com/portainer/portainer/api/dataservices/settings"
+	"github.com/portainer/portainer/api/dataservices/snapshot"
 	"github.com/portainer/portainer/api/dataservices/ssl"
 	"github.com/portainer/portainer/api/dataservices/stack"
 	"github.com/portainer/portainer/api/dataservices/tag"
@@ -33,7 +36,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices/user"
 	"github.com/portainer/portainer/api/dataservices/version"
 	"github.com/portainer/portainer/api/dataservices/webhook"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 // Store defines the implementation of portainer.DataStore using
@@ -46,6 +50,7 @@ type Store struct {
 	DockerHubService          *dockerhub.Service
 	EdgeGroupService          *edgegroup.Service
 	EdgeJobService            *edgejob.Service
+	EdgeUpdateScheduleService *edgeupdateschedule.Service
 	EdgeStackService          *edgestack.Service
 	EndpointGroupService      *endpointgroup.Service
 	EndpointService           *endpoint.Service
@@ -59,6 +64,7 @@ type Store struct {
 	APIKeyRepositoryService   *apikeyrepository.Service
 	ScheduleService           *schedule.Service
 	SettingsService           *settings.Service
+	SnapshotService           *snapshot.Service
 	SSLSettingsService        *ssl.Service
 	StackService              *stack.Service
 	TagService                *tag.Service
@@ -89,6 +95,12 @@ func (store *Store) initServices() error {
 	}
 	store.DockerHubService = dockerhubService
 
+	edgeUpdateScheduleService, err := edgeupdateschedule.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeUpdateScheduleService = edgeUpdateScheduleService
+
 	edgeStackService, err := edgestack.NewService(store.connection)
 	if err != nil {
 		return err
@@ -161,6 +173,12 @@ func (store *Store) initServices() error {
 	}
 	store.SettingsService = settingsService
 
+	snapshotService, err := snapshot.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SnapshotService = snapshotService
+
 	sslSettingsService, err := ssl.NewService(store.connection)
 	if err != nil {
 		return err
@@ -245,6 +263,11 @@ func (store *Store) EdgeJob() dataservices.EdgeJobService {
 	return store.EdgeJobService
 }
 
+// EdgeUpdateSchedule gives access to the EdgeUpdateSchedule data management layer
+func (store *Store) EdgeUpdateSchedule() dataservices.EdgeUpdateScheduleService {
+	return store.EdgeUpdateScheduleService
+}
+
 // EdgeStack gives access to the EdgeStack data management layer
 func (store *Store) EdgeStack() dataservices.EdgeStackService {
 	return store.EdgeStackService
@@ -300,6 +323,10 @@ func (store *Store) Settings() dataservices.SettingsService {
 	return store.SettingsService
 }
 
+func (store *Store) Snapshot() dataservices.SnapshotService {
+	return store.SnapshotService
+}
+
 // SSLSettings gives access to the SSL Settings data management layer
 func (store *Store) SSLSettings() dataservices.SSLSettingsService {
 	return store.SSLSettingsService
@@ -360,6 +387,7 @@ type storeExport struct {
 	Role               []portainer.Role               `json:"roles,omitempty"`
 	Schedules          []portainer.Schedule           `json:"schedules,omitempty"`
 	Settings           portainer.Settings             `json:"settings,omitempty"`
+	Snapshot           []portainer.Snapshot           `json:"snapshots,omitempty"`
 	SSLSettings        portainer.SSLSettings          `json:"ssl,omitempty"`
 	Stack              []portainer.Stack              `json:"stacks,omitempty"`
 	Tag                []portainer.Tag                `json:"tags,omitempty"`
@@ -378,7 +406,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Custom Templates")
+			log.Error().Err(err).Msg("exporting Custom Templates")
 		}
 	} else {
 		backup.CustomTemplate = c
@@ -386,7 +414,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Edge Groups")
+			log.Error().Err(err).Msg("exporting Edge Groups")
 		}
 	} else {
 		backup.EdgeGroup = e
@@ -394,7 +422,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if e, err := store.EdgeJob().EdgeJobs(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Edge Jobs")
+			log.Error().Err(err).Msg("exporting Edge Jobs")
 		}
 	} else {
 		backup.EdgeJob = e
@@ -402,7 +430,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if e, err := store.EdgeStack().EdgeStacks(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Edge Stacks")
+			log.Error().Err(err).Msg("exporting Edge Stacks")
 		}
 	} else {
 		backup.EdgeStack = e
@@ -410,7 +438,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if e, err := store.Endpoint().Endpoints(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Endpoints")
+			log.Error().Err(err).Msg("exporting Endpoints")
 		}
 	} else {
 		backup.Endpoint = e
@@ -418,7 +446,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Endpoint Groups")
+			log.Error().Err(err).Msg("exporting Endpoint Groups")
 		}
 	} else {
 		backup.EndpointGroup = e
@@ -426,7 +454,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if r, err := store.EndpointRelation().EndpointRelations(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Endpoint Relations")
+			log.Error().Err(err).Msg("exporting Endpoint Relations")
 		}
 	} else {
 		backup.EndpointRelation = r
@@ -434,7 +462,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if r, err := store.ExtensionService.Extensions(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Extensions")
+			log.Error().Err(err).Msg("exporting Extensions")
 		}
 	} else {
 		backup.Extensions = r
@@ -442,7 +470,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Helm User Repositories")
+			log.Error().Err(err).Msg("exporting Helm User Repositories")
 		}
 	} else {
 		backup.HelmUserRepository = r
@@ -450,7 +478,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if r, err := store.Registry().Registries(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Registries")
+			log.Error().Err(err).Msg("exporting Registries")
 		}
 	} else {
 		backup.Registry = r
@@ -458,7 +486,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if c, err := store.ResourceControl().ResourceControls(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Resource Controls")
+			log.Error().Err(err).Msg("exporting Resource Controls")
 		}
 	} else {
 		backup.ResourceControl = c
@@ -466,7 +494,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if role, err := store.Role().Roles(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Roles")
+			log.Error().Err(err).Msg("exporting Roles")
 		}
 	} else {
 		backup.Role = role
@@ -474,7 +502,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if r, err := store.ScheduleService.Schedules(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Schedules")
+			log.Error().Err(err).Msg("exporting Schedules")
 		}
 	} else {
 		backup.Schedules = r
@@ -482,15 +510,23 @@ func (store *Store) Export(filename string) (err error) {
 
 	if settings, err := store.Settings().Settings(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Settings")
+			log.Error().Err(err).Msg("exporting Settings")
 		}
 	} else {
 		backup.Settings = *settings
 	}
 
+	if snapshot, err := store.Snapshot().Snapshots(); err != nil {
+		if !store.IsErrObjectNotFound(err) {
+			log.Err(err).Msg("Exporting Snapshots")
+		}
+	} else {
+		backup.Snapshot = snapshot
+	}
+
 	if settings, err := store.SSLSettings().Settings(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting SSL Settings")
+			log.Error().Err(err).Msg("exporting SSL Settings")
 		}
 	} else {
 		backup.SSLSettings = *settings
@@ -498,7 +534,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if t, err := store.Stack().Stacks(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Stacks")
+			log.Error().Err(err).Msg("exporting Stacks")
 		}
 	} else {
 		backup.Stack = t
@@ -506,7 +542,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if t, err := store.Tag().Tags(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Tags")
+			log.Error().Err(err).Msg("exporting Tags")
 		}
 	} else {
 		backup.Tag = t
@@ -514,7 +550,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if t, err := store.TeamMembership().TeamMemberships(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Team Memberships")
+			log.Error().Err(err).Msg("exporting Team Memberships")
 		}
 	} else {
 		backup.TeamMembership = t
@@ -522,7 +558,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if t, err := store.Team().Teams(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Teams")
+			log.Error().Err(err).Msg("exporting Teams")
 		}
 	} else {
 		backup.Team = t
@@ -530,7 +566,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if info, err := store.TunnelServer().Info(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Tunnel Server")
+			log.Error().Err(err).Msg("exporting Tunnel Server")
 		}
 	} else {
 		backup.TunnelServer = *info
@@ -538,7 +574,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if users, err := store.User().Users(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Users")
+			log.Error().Err(err).Msg("exporting Users")
 		}
 	} else {
 		backup.User = users
@@ -546,7 +582,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if webhooks, err := store.Webhook().Webhooks(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			logrus.WithError(err).Errorf("Exporting Webhooks")
+			log.Error().Err(err).Msg("exporting Webhooks")
 		}
 	} else {
 		backup.Webhook = webhooks
@@ -554,7 +590,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	v, err := store.Version().DBVersion()
 	if err != nil && !store.IsErrObjectNotFound(err) {
-		logrus.WithError(err).Errorf("Exporting DB version")
+		log.Error().Err(err).Msg("exporting DB version")
 	}
 	instance, _ := store.Version().InstanceID()
 	backup.Version = map[string]string{
@@ -564,7 +600,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	backup.Metadata, err = store.connection.BackupMetadata()
 	if err != nil {
-		logrus.WithError(err).Errorf("Exporting Metadata")
+		log.Error().Err(err).Msg("exporting Metadata")
 	}
 
 	b, err := json.MarshalIndent(backup, "", "  ")
@@ -575,7 +611,6 @@ func (store *Store) Export(filename string) (err error) {
 }
 
 func (store *Store) Import(filename string) (err error) {
-
 	backup := storeExport{}
 
 	s, err := ioutil.ReadFile(filename)
@@ -591,13 +626,13 @@ func (store *Store) Import(filename string) (err error) {
 	if dbversion, ok := backup.Version["DB_VERSION"]; ok {
 		if v, err := strconv.Atoi(dbversion); err == nil {
 			if err := store.Version().StoreDBVersion(v); err != nil {
-				logrus.WithError(err).Errorf("DB_VERSION import issue")
+				log.Error().Err(err).Msg("DB_VERSION import issue")
 			}
 		}
 	}
 	if instanceID, ok := backup.Version["INSTANCE_ID"]; ok {
 		if err := store.Version().StoreInstanceID(instanceID); err != nil {
-			logrus.WithError(err).Errorf("INSTANCE_ID import issue")
+			log.Error().Err(err).Msg("INSTANCE_ID import issue")
 		}
 	}
 
@@ -648,6 +683,10 @@ func (store *Store) Import(filename string) (err error) {
 	store.Settings().UpdateSettings(&backup.Settings)
 	store.SSLSettings().UpdateSettings(&backup.SSLSettings)
 
+	for _, v := range backup.Snapshot {
+		store.Snapshot().UpdateSnapshot(&v)
+	}
+
 	for _, v := range backup.Stack {
 		store.Stack().UpdateStack(v.ID, &v)
 	}
@@ -668,7 +707,7 @@ func (store *Store) Import(filename string) (err error) {
 
 	for _, user := range backup.User {
 		if err := store.User().UpdateUser(user.ID, &user); err != nil {
-			logrus.WithField("user", user).WithError(err).Errorf("User: Failed to Update Database")
+			log.Debug().Str("user", fmt.Sprintf("%+v", user)).Err(err).Msg("user: failed to Update Database")
 		}
 	}
 

api/datastore/test_data/output_24_to_latest.json

@@ -52,16 +52,23 @@
       "IsEdgeDevice": false,
       "Kubernetes": {
         "Configuration": {
+          "AllowNoneIngressClass": false,
+          "EnableResourceOverCommit": false,
+          "IngressAvailabilityPerNamespace": true,
           "IngressClasses": null,
+          "ResourceOverCommitPercentage": 0,
           "RestrictDefaultNamespace": false,
           "StorageClasses": null,
           "UseLoadBalancer": false,
           "UseServerMetrics": false
         },
-        "Snapshots": null
+        "Snapshots": []
       },
       "LastCheckInDate": 0,
       "Name": "local",
+      "PostInitMigrations": {
+        "MigrateIngresses": true
+      },
       "PublicURL": "",
       "QueryDate": 0,
       "SecuritySettings": {
@@ -75,127 +82,7 @@
         "allowVolumeBrowserForRegularUsers": false,
         "enableHostManagementFeatures": false
       },
-      "Snapshots": [
-        {
-          "DockerSnapshotRaw": {
-            "Containers": null,
-            "Images": null,
-            "Info": {
-              "Architecture": "",
-              "BridgeNfIp6tables": false,
-              "BridgeNfIptables": false,
-              "CPUSet": false,
-              "CPUShares": false,
-              "CgroupDriver": "",
-              "ContainerdCommit": {
-                "Expected": "",
-                "ID": ""
-              },
-              "Containers": 0,
-              "ContainersPaused": 0,
-              "ContainersRunning": 0,
-              "ContainersStopped": 0,
-              "CpuCfsPeriod": false,
-              "CpuCfsQuota": false,
-              "Debug": false,
-              "DefaultRuntime": "",
-              "DockerRootDir": "",
-              "Driver": "",
-              "DriverStatus": null,
-              "ExperimentalBuild": false,
-              "GenericResources": null,
-              "HttpProxy": "",
-              "HttpsProxy": "",
-              "ID": "",
-              "IPv4Forwarding": false,
-              "Images": 0,
-              "IndexServerAddress": "",
-              "InitBinary": "",
-              "InitCommit": {
-                "Expected": "",
-                "ID": ""
-              },
-              "Isolation": "",
-              "KernelMemory": false,
-              "KernelMemoryTCP": false,
-              "KernelVersion": "",
-              "Labels": null,
-              "LiveRestoreEnabled": false,
-              "LoggingDriver": "",
-              "MemTotal": 0,
-              "MemoryLimit": false,
-              "NCPU": 0,
-              "NEventsListener": 0,
-              "NFd": 0,
-              "NGoroutines": 0,
-              "Name": "",
-              "NoProxy": "",
-              "OSType": "",
-              "OSVersion": "",
-              "OomKillDisable": false,
-              "OperatingSystem": "",
-              "PidsLimit": false,
-              "Plugins": {
-                "Authorization": null,
-                "Log": null,
-                "Network": null,
-                "Volume": null
-              },
-              "RegistryConfig": null,
-              "RuncCommit": {
-                "Expected": "",
-                "ID": ""
-              },
-              "Runtimes": null,
-              "SecurityOptions": null,
-              "ServerVersion": "",
-              "SwapLimit": false,
-              "Swarm": {
-                "ControlAvailable": false,
-                "Error": "",
-                "LocalNodeState": "",
-                "NodeAddr": "",
-                "NodeID": "",
-                "RemoteManagers": null
-              },
-              "SystemTime": "",
-              "Warnings": null
-            },
-            "Networks": null,
-            "Version": {
-              "ApiVersion": "",
-              "Arch": "",
-              "GitCommit": "",
-              "GoVersion": "",
-              "Os": "",
-              "Platform": {
-                "Name": ""
-              },
-              "Version": ""
-            },
-            "Volumes": {
-              "Volumes": null,
-              "Warnings": null
-            }
-          },
-          "DockerVersion": "20.10.13",
-          "GpuUseAll": false,
-          "GpuUseList": null,
-          "HealthyContainerCount": 0,
-          "ImageCount": 9,
-          "NodeCount": 0,
-          "RunningContainerCount": 5,
-          "ServiceCount": 0,
-          "StackCount": 2,
-          "StoppedContainerCount": 0,
-          "Swarm": false,
-          "Time": 1648610112,
-          "TotalCPU": 8,
-          "TotalMemory": 25098706944,
-          "UnhealthyContainerCount": 0,
-          "VolumeCount": 10
-        }
-      ],
+      "Snapshots": [],
       "Status": 1,
       "TLSConfig": {
         "TLS": false,
@@ -775,6 +662,131 @@
       "mpsUser": ""
     }
   },
+  "snapshots": [
+    {
+      "Docker": {
+        "DockerSnapshotRaw": {
+          "Containers": null,
+          "Images": null,
+          "Info": {
+            "Architecture": "",
+            "BridgeNfIp6tables": false,
+            "BridgeNfIptables": false,
+            "CPUSet": false,
+            "CPUShares": false,
+            "CgroupDriver": "",
+            "ContainerdCommit": {
+              "Expected": "",
+              "ID": ""
+            },
+            "Containers": 0,
+            "ContainersPaused": 0,
+            "ContainersRunning": 0,
+            "ContainersStopped": 0,
+            "CpuCfsPeriod": false,
+            "CpuCfsQuota": false,
+            "Debug": false,
+            "DefaultRuntime": "",
+            "DockerRootDir": "",
+            "Driver": "",
+            "DriverStatus": null,
+            "ExperimentalBuild": false,
+            "GenericResources": null,
+            "HttpProxy": "",
+            "HttpsProxy": "",
+            "ID": "",
+            "IPv4Forwarding": false,
+            "Images": 0,
+            "IndexServerAddress": "",
+            "InitBinary": "",
+            "InitCommit": {
+              "Expected": "",
+              "ID": ""
+            },
+            "Isolation": "",
+            "KernelMemory": false,
+            "KernelMemoryTCP": false,
+            "KernelVersion": "",
+            "Labels": null,
+            "LiveRestoreEnabled": false,
+            "LoggingDriver": "",
+            "MemTotal": 0,
+            "MemoryLimit": false,
+            "NCPU": 0,
+            "NEventsListener": 0,
+            "NFd": 0,
+            "NGoroutines": 0,
+            "Name": "",
+            "NoProxy": "",
+            "OSType": "",
+            "OSVersion": "",
+            "OomKillDisable": false,
+            "OperatingSystem": "",
+            "PidsLimit": false,
+            "Plugins": {
+              "Authorization": null,
+              "Log": null,
+              "Network": null,
+              "Volume": null
+            },
+            "RegistryConfig": null,
+            "RuncCommit": {
+              "Expected": "",
+              "ID": ""
+            },
+            "Runtimes": null,
+            "SecurityOptions": null,
+            "ServerVersion": "",
+            "SwapLimit": false,
+            "Swarm": {
+              "ControlAvailable": false,
+              "Error": "",
+              "LocalNodeState": "",
+              "NodeAddr": "",
+              "NodeID": "",
+              "RemoteManagers": null
+            },
+            "SystemTime": "",
+            "Warnings": null
+          },
+          "Networks": null,
+          "Version": {
+            "ApiVersion": "",
+            "Arch": "",
+            "GitCommit": "",
+            "GoVersion": "",
+            "Os": "",
+            "Platform": {
+              "Name": ""
+            },
+            "Version": ""
+          },
+          "Volumes": {
+            "Volumes": null,
+            "Warnings": null
+          }
+        },
+        "DockerVersion": "20.10.13",
+        "GpuUseAll": false,
+        "GpuUseList": null,
+        "HealthyContainerCount": 0,
+        "ImageCount": 9,
+        "NodeCount": 0,
+        "RunningContainerCount": 5,
+        "ServiceCount": 0,
+        "StackCount": 2,
+        "StoppedContainerCount": 0,
+        "Swarm": false,
+        "Time": 1648610112,
+        "TotalCPU": 8,
+        "TotalMemory": 25098706944,
+        "UnhealthyContainerCount": 0,
+        "VolumeCount": 10
+      },
+      "EndpointId": 1,
+      "Kubernetes": null
+    }
+  ],
   "ssl": {
     "certPath": "",
     "httpEnabled": true,
@@ -919,7 +931,7 @@
   ],
   "version": {
     "DB_UPDATING": "false",
-    "DB_VERSION": "70",
+    "DB_VERSION": "72",
     "INSTANCE_ID": "null"
   }
 }

api/datastore/teststore.go

@@ -1,15 +1,14 @@
 package datastore
 
 import (
-	"io/ioutil"
-	"log"
-	"os"
+	"testing"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database"
+	"github.com/portainer/portainer/api/filesystem"
 
 	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/filesystem"
+	"github.com/rs/zerolog/log"
 )
 
 var errTempDir = errors.New("can't create a temp dir")
@@ -18,25 +17,22 @@ func (store *Store) GetConnection() portainer.Connection {
 	return store.connection
 }
 
-func MustNewTestStore(init, secure bool) (bool, *Store, func()) {
-	newStore, store, teardown, err := NewTestStore(init, secure)
+func MustNewTestStore(t *testing.T, init, secure bool) (bool, *Store, func()) {
+	newStore, store, teardown, err := NewTestStore(t, init, secure)
 	if err != nil {
 		if !errors.Is(err, errTempDir) {
 			teardown()
 		}
-		log.Fatal(err)
+
+		log.Fatal().Err(err).Msg("")
 	}
 
 	return newStore, store, teardown
 }
 
-func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
+func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error) {
 	// Creates unique temp directory in a concurrency friendly manner.
-	storePath, err := ioutil.TempDir("", "test-store")
-	if err != nil {
-		return false, nil, nil, errors.Wrap(errTempDir, err.Error())
-	}
-
+	storePath := t.TempDir()
 	fileService, err := filesystem.NewService(storePath, "")
 	if err != nil {
 		return false, nil, nil, err
@@ -51,12 +47,15 @@ func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
 	if err != nil {
 		panic(err)
 	}
+
 	store := NewStore(storePath, fileService, connection)
 	newStore, err := store.Open()
 	if err != nil {
 		return newStore, nil, nil, err
 	}
 
+	log.Debug().Msg("opened")
+
 	if init {
 		err = store.Init()
 		if err != nil {
@@ -64,6 +63,8 @@ func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
 		}
 	}
 
+	log.Debug().Msg("initialised")
+
 	if newStore {
 		// from MigrateData
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
@@ -73,20 +74,15 @@ func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
 	}
 
 	teardown := func() {
-		teardown(store, storePath)
+		teardown(store)
 	}
 
 	return newStore, store, teardown, nil
 }
 
-func teardown(store *Store, storePath string) {
+func teardown(store *Store) {
 	err := store.Close()
 	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = os.RemoveAll(storePath)
-	if err != nil {
-		log.Fatalln(err)
+		log.Fatal().Err(err).Msg("")
 	}
 }

api/demo/demo.go

@@ -1,11 +1,11 @@
 package demo
 
 import (
-	"log"
-
-	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 type EnvironmentDetails struct {
@@ -27,7 +27,7 @@ func (service *Service) Details() EnvironmentDetails {
 }
 
 func (service *Service) Init(store dataservices.DataStore, cryptoService portainer.CryptoService) error {
-	log.Print("[INFO] [main] Starting demo environment")
+	log.Info().Msg("starting demo environment")
 
 	isClean, err := isCleanStore(store)
 	if err != nil {

api/demo/init.go

@@ -60,6 +60,15 @@ func initDemoLocalEndpoint(store dataservices.DataStore) (portainer.EndpointID,
 	}
 
 	err := store.Endpoint().Create(localEndpoint)
+	if err != nil {
+		return id, errors.WithMessage(err, "failed creating local endpoint")
+	}
+
+	err = store.Snapshot().Create(&portainer.Snapshot{EndpointID: id})
+	if err != nil {
+		return id, errors.WithMessage(err, "failed creating snapshot")
+	}
+
 	return id, errors.WithMessage(err, "failed creating local endpoint")
 }
 

api/docker/snapshot.go

@@ -2,15 +2,16 @@ package docker
 
 import (
 	"context"
-	"log"
 	"strings"
 	"time"
 
+	portainer "github.com/portainer/portainer/api"
+
 	"github.com/docker/docker/api/types"
 	_container "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
 )
 
 // Snapshotter represents a service used to create environment(endpoint) snapshots
@@ -48,44 +49,44 @@ func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Dock
 
 	err = snapshotInfo(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine information")
 	}
 
 	if snapshot.Swarm {
 		err = snapshotSwarmServices(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [environment: %s] [err: %s]", endpoint.Name, err)
+			log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm services")
 		}
 
 		err = snapshotNodes(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [environment: %s] [err: %s]", endpoint.Name, err)
+			log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm nodes")
 		}
 	}
 
 	err = snapshotContainers(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot containers")
 	}
 
 	err = snapshotImages(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot images")
 	}
 
 	err = snapshotVolumes(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot volumes")
 	}
 
 	err = snapshotNetworks(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot networks")
 	}
 
 	err = snapshotVersion(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine version")
 	}
 
 	snapshot.Time = time.Now().Unix()

api/edgetypes/edgetypes.go

@@ -0,0 +1,102 @@
+package edgetypes
+
+import portainer "github.com/portainer/portainer/api"
+
+const (
+	// PortainerAgentUpdateScheduleIDHeader represents the name of the header containing the update schedule id
+	PortainerAgentUpdateScheduleIDHeader = "X-Portainer-Update-Schedule-ID"
+	// PortainerAgentUpdateStatusHeader is the name of the header that will have the update status
+	PortainerAgentUpdateStatusHeader = "X-Portainer-Update-Status"
+	// PortainerAgentUpdateErrorHeader is the name of the header that will have the update error
+	PortainerAgentUpdateErrorHeader = "X-Portainer-Update-Error"
+)
+
+type (
+
+	// UpdateScheduleID represents an Edge schedule identifier
+	UpdateScheduleID int
+
+	// UpdateSchedule represents a schedule for update/rollback of edge devices
+	UpdateSchedule struct {
+		// EdgeUpdateSchedule Identifier
+		ID UpdateScheduleID `json:"id" example:"1"`
+		// Name of the schedule
+		Name string `json:"name" example:"Update Schedule"`
+		// Type of the schedule
+		Time int64 `json:"time" example:"1564897200"`
+		// EdgeGroups to be updated
+		GroupIDs []portainer.EdgeGroupID `json:"groupIds" example:"1"`
+		// Type of the update (1 - update, 2 - rollback)
+		Type UpdateScheduleType `json:"type" example:"1" enums:"1,2"`
+		// Status of the schedule, grouped by environment id
+		Status map[portainer.EndpointID]UpdateScheduleStatus `json:"status"`
+		// Created timestamp
+		Created int64 `json:"created" example:"1564897200"`
+		// Created by user id
+		CreatedBy portainer.UserID `json:"createdBy" example:"1"`
+	}
+
+	// UpdateScheduleType represents type of an Edge update schedule
+	UpdateScheduleType int
+
+	// UpdateScheduleStatus represents status of an Edge update schedule
+	UpdateScheduleStatus struct {
+		// Status of the schedule (0 - pending, 1 - failed, 2 - success)
+		Status UpdateScheduleStatusType `json:"status" example:"1" enums:"1,2,3"`
+		// Error message if status is failed
+		Error string `json:"error" example:""`
+		// Target version of the edge agent
+		TargetVersion string `json:"targetVersion" example:"1"`
+		// Current version of the edge agent
+		CurrentVersion string `json:"currentVersion" example:"1"`
+	}
+
+	// UpdateScheduleStatusType represents status type of an Edge update schedule
+	UpdateScheduleStatusType int
+
+	VersionUpdateRequest struct {
+		// Target version
+		Version string
+		// Scheduled time
+		ScheduledTime int64
+		// If need to update
+		Active bool
+		// Update schedule ID
+		ScheduleID UpdateScheduleID
+	}
+
+	// VersionUpdateStatus represents the status of an agent version update
+	VersionUpdateStatus struct {
+		Status     UpdateScheduleStatusType
+		ScheduleID UpdateScheduleID
+		Error      string
+	}
+
+	// EndpointUpdateScheduleRelation represents the relation between an environment(endpoint) and an update schedule
+	EndpointUpdateScheduleRelation struct {
+		EnvironmentID portainer.EndpointID     `json:"environmentId"`
+		ScheduleID    UpdateScheduleID         `json:"scheduleId"`
+		TargetVersion string                   `json:"targetVersion"`
+		Status        UpdateScheduleStatusType `json:"status"`
+		Error         string                   `json:"error"`
+		Type          UpdateScheduleType       `json:"type"`
+		ScheduledTime int64                    `json:"scheduledTime"`
+	}
+)
+
+const (
+	_ UpdateScheduleType = iota
+	// UpdateScheduleUpdate represents an edge device scheduled for an update
+	UpdateScheduleUpdate
+	// UpdateScheduleRollback represents an edge device scheduled for a rollback
+	UpdateScheduleRollback
+)
+
+const (
+	// UpdateScheduleStatusPending represents a pending edge update schedule
+	UpdateScheduleStatusPending UpdateScheduleStatusType = iota
+	// UpdateScheduleStatusError represents a failed edge update schedule
+	UpdateScheduleStatusError
+	// UpdateScheduleStatusSuccess represents a successful edge update schedule
+	UpdateScheduleStatusSuccess
+)

api/exec/compose_stack.go

@@ -85,6 +85,27 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 	return errors.Wrap(err, "failed to remove a stack")
 }
 
+// Pull an image associated with a service defined in a docker-compose.yml or docker-stack.yml file,
+// but does not start containers based on those images.
+func (manager *ComposeStackManager) Pull(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	url, proxy, err := manager.fetchEndpointProxy(endpoint)
+	if err != nil {
+		return err
+	}
+	if proxy != nil {
+		defer proxy.Close()
+	}
+
+	envFile, err := createEnvFile(stack)
+	if err != nil {
+		return errors.Wrap(err, "failed to create env file")
+	}
+
+	filePaths := stackutils.GetStackFilePaths(stack)
+	err = manager.deployer.Pull(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
+	return errors.Wrap(err, "failed to pull images of the stack")
+}
+
 // NormalizeStackName returns a new stack name with unsupported characters replaced
 func (manager *ComposeStackManager) NormalizeStackName(name string) string {
 	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")

api/exec/compose_stack_integration_test.go

@@ -3,7 +3,6 @@ package exec
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -12,6 +11,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+
+	"github.com/rs/zerolog/log"
 )
 
 const composeFile = `version: "3.9"
@@ -77,7 +78,7 @@ func containerExists(containerName string) bool {
 
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to list containers: %s", err)
+		log.Fatal().Err(err).Msg("failed to list containers")
 	}
 
 	return strings.Contains(string(out), containerName)

api/exec/swarm_stack.go

@@ -89,7 +89,7 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 }
 
 // Deploy executes the docker stack deploy command.
-func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
+func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, pullImage bool, endpoint *portainer.Endpoint) error {
 	filePaths := stackutils.GetStackFilePaths(stack)
 	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	if err != nil {
@@ -101,6 +101,9 @@ func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, end
 	} else {
 		args = append(args, "stack", "deploy", "--with-registry-auth")
 	}
+	if !pullImage {
+		args = append(args, "--resolve-image=never")
+	}
 
 	args = configureFilePaths(args, filePaths)
 	args = append(args, stack.Name)

api/filesystem/copy_test.go

@@ -11,17 +11,13 @@ import (
 )
 
 func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	err := copyFile("does-not-exist", tmpdir)
 	assert.Error(t, err)
 }
 
 func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
 
@@ -33,8 +29,7 @@ func Test_copyFile_shouldMakeAbackup(t *testing.T) {
 }
 
 func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
+	destination := t.TempDir()
 	err := CopyDir("./testdata/copy_test", destination, true)
 	assert.NoError(t, err)
 
@@ -44,8 +39,7 @@ func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
 }
 
 func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
+	destination := t.TempDir()
 	err := CopyDir("./testdata/copy_test", destination, false)
 	assert.NoError(t, err)
 
@@ -55,9 +49,7 @@ func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
 }
 
 func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	err := CopyPath("does-not-exists", tmpdir)
 	assert.NoError(t, err)
 
@@ -65,9 +57,7 @@ func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
 }
 
 func Test_CopyPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
 
@@ -81,8 +71,7 @@ func Test_CopyPath_shouldCopyFile(t *testing.T) {
 }
 
 func Test_CopyPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
+	destination := t.TempDir()
 	err := CopyPath("./testdata/copy_test", destination)
 	assert.NoError(t, err)
 

api/filesystem/filesystem.go

@@ -234,6 +234,58 @@ func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string
 	return service.wrapFileStore(stackStorePath), nil
 }
 
+// UpdateStoreStackFileFromBytes makes stack file backup and updates a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) UpdateStoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	err := service.createBackupFileInStore(composeFilePath)
+	if err != nil {
+		return "", err
+	}
+
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return service.wrapFileStore(stackStorePath), nil
+}
+
+// RemoveStackFileBackup removes the stack file backup in the ComposeStorePath.
+func (service *Service) RemoveStackFileBackup(stackIdentifier, fileName string) error {
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+
+	return service.removeBackupFileInStore(composeFilePath)
+}
+
+// RollbackStackFile rollbacks the stack file backup in the ComposeStorePath.
+func (service *Service) RollbackStackFile(stackIdentifier, fileName string) error {
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	path := service.wrapFileStore(composeFilePath)
+	backupPath := fmt.Sprintf("%s.bak", path)
+
+	exists, err := service.FileExists(backupPath)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		// keep the updated/failed stack file
+		return nil
+	}
+
+	err = service.Copy(backupPath, path, true)
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(backupPath)
+}
+
 // GetEdgeStackProjectPath returns the absolute path on the FS for a edge stack based
 // on its identifier.
 func (service *Service) GetEdgeStackProjectPath(edgeStackIdentifier string) string {
@@ -447,6 +499,31 @@ func (service *Service) createFileInStore(filePath string, r io.Reader) error {
 	return err
 }
 
+// createBackupFileInStore makes a copy in the file store.
+func (service *Service) createBackupFileInStore(filePath string) error {
+	path := service.wrapFileStore(filePath)
+	backupPath := fmt.Sprintf("%s.bak", path)
+
+	return service.Copy(path, backupPath, true)
+}
+
+// removeBackupFileInStore removes the copy in the file store.
+func (service *Service) removeBackupFileInStore(filePath string) error {
+	path := service.wrapFileStore(filePath)
+	backupPath := fmt.Sprintf("%s.bak", path)
+
+	exists, err := service.FileExists(backupPath)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return os.Remove(backupPath)
+	}
+
+	return nil
+}
+
 func (service *Service) createPEMFileInStore(content []byte, fileType, filePath string) error {
 	path := service.wrapFileStore(filePath)
 	block := &pem.Block{Type: fileType, Bytes: content}

api/filesystem/filesystem_fileexists_test.go

@@ -43,7 +43,7 @@ func testHelperFileExists_fileExists(t *testing.T, checker func(path string) (bo
 }
 
 func testHelperFileExists_fileNotExists(t *testing.T, checker func(path string) (bool, error)) {
-	filePath := path.Join(os.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int()))
+	filePath := path.Join(t.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int()))
 
 	err := os.RemoveAll(filePath)
 	assert.NoError(t, err, "RemoveAll should not fail")

api/filesystem/filesystem_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func createService(t *testing.T) *Service {
-	dataStorePath := path.Join(os.TempDir(), t.Name())
+	dataStorePath := path.Join(t.TempDir(), t.Name())
 
 	service, err := NewService(dataStorePath, "")
 	assert.NoError(t, err, "NewService should not fail")

api/git/azure.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +11,10 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 )
@@ -32,20 +36,47 @@ type azureOptions struct {
 	username, password string
 }
 
-type azureDownloader struct {
+// azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs
+type azureRef struct {
+	Name     string `json:"name"`
+	ObjectID string `json:"objectId"`
+}
+
+// azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download
+type azureItem struct {
+	ObjectID string `json:"objectId"`
+	CommitId string `json:"commitId"`
+	Path     string `json:"path"`
+}
+
+type azureClient struct {
 	client  *http.Client
 	baseUrl string
 }
 
-func NewAzureDownloader(client *http.Client) *azureDownloader {
-	return &azureDownloader{
-		client:  client,
+func NewAzureClient() *azureClient {
+	httpsCli := newHttpClientForAzure()
+	return &azureClient{
+		client:  httpsCli,
 		baseUrl: "https://dev.azure.com",
 	}
 }
 
-func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
-	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
+func newHttpClientForAzure() *http.Client {
+	httpsCli := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			Proxy:           http.ProxyFromEnvironment,
+		},
+		Timeout: 300 * time.Second,
+	}
+
+	client.InstallProtocol("https", githttp.NewClient(httpsCli))
+	return httpsCli
+}
+
+func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error {
+	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt)
 	if err != nil {
 		return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
 	}
@@ -59,12 +90,12 @@ func (a *azureDownloader) download(ctx context.Context, destination string, opti
 	return nil
 }
 
-func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
-	config, err := parseUrl(options.repositoryUrl)
+func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) {
+	config, err := parseUrl(opt.repositoryUrl)
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to parse url")
 	}
-	downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
+	downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName)
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to build download url")
 	}
@@ -75,8 +106,8 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	defer zipFile.Close()
 
 	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
-	if options.username != "" || options.password != "" {
-		req.SetBasicAuth(options.username, options.password)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
 	} else if config.username != "" || config.password != "" {
 		req.SetBasicAuth(config.username, config.password)
 	}
@@ -102,53 +133,58 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	return zipFile.Name(), nil
 }
 
-func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
-	config, err := parseUrl(options.repositoryUrl)
+func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
+	rootItem, err := a.getRootItem(ctx, opt)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to parse url")
+		return "", err
+	}
+	return rootItem.CommitId, nil
+}
+
+func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) {
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
 	}
 
-	rootItemUrl, err := a.buildRootItemUrl(config, options.referenceName)
+	rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to build azure root item url")
+		return nil, errors.WithMessage(err, "failed to build azure root item url")
 	}
 
 	req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
-	if options.username != "" || options.password != "" {
-		req.SetBasicAuth(options.username, options.password)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
 	} else if config.username != "" || config.password != "" {
 		req.SetBasicAuth(config.username, config.password)
 	}
 
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
 	resp, err := a.client.Do(req)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to make an HTTP request")
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status)
+		return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode)
 	}
 
 	var items struct {
-		Value []struct {
-			CommitId string `json:"commitId"`
-		}
+		Value []azureItem
 	}
 
 	if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
-		return "", errors.Wrap(err, "could not parse Azure items response")
+		return nil, errors.Wrap(err, "could not parse Azure items response")
 	}
 
 	if len(items.Value) == 0 || items.Value[0].CommitId == "" {
-		return "", errors.Errorf("failed to get latest commitID in the repository")
+		return nil, errors.Errorf("failed to get latest commitID in the repository")
 	}
-
-	return items.Value[0].CommitId, nil
+	return &items.Value[0], nil
 }
 
 func parseUrl(rawUrl string) (*azureOptions, error) {
@@ -219,7 +255,7 @@ func parseHttpUrl(rawUrl string) (*azureOptions, error) {
 	return &opt, nil
 }
 
-func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
+func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
 	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
 		a.baseUrl,
 		url.PathEscape(config.organisation),
@@ -246,7 +282,7 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
-func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
+func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
 	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
 		a.baseUrl,
 		url.PathEscape(config.organisation),
@@ -270,6 +306,49 @@ func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
+func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) {
+	// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl)
+	}
+
+	q := u.Query()
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
+func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) {
+	// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository),
+		url.PathEscape(rootObjectHash),
+	)
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
+	}
+	q := u.Query()
+	// projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0
+	q.Set("recursive", "true")
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
 const (
 	branchPrefix = "refs/heads/"
 	tagPrefix    = "refs/tags/"
@@ -294,3 +373,119 @@ func getVersionType(name string) string {
 	}
 	return "commit"
 }
+
+func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
+	}
+
+	listRefsUrl, err := a.buildRefsUrl(config)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to build list refs url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode)
+	}
+
+	var refs struct {
+		Value []azureRef
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
+		return nil, errors.Wrap(err, "could not parse Azure refs response")
+	}
+
+	var ret []string
+	for _, value := range refs.Value {
+		if value.Name == "HEAD" {
+			continue
+		}
+		ret = append(ret, value.Name)
+	}
+
+	return ret, nil
+}
+
+// listFiles list all filenames under the specific repository
+func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
+	rootItem, err := a.getRootItem(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
+	}
+
+	listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to build list tree url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status)
+	}
+
+	var tree struct {
+		TreeEntries []struct {
+			RelativePath string `json:"relativePath"`
+		} `json:"treeEntries"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil {
+		return nil, errors.Wrap(err, "could not parse Azure tree response")
+	}
+
+	var allPaths []string
+	for _, treeEntry := range tree.TreeEntries {
+		allPaths = append(allPaths, treeEntry.RelativePath)
+	}
+
+	return allPaths, nil
+}
+
+func checkAzureStatusCode(err error, code int) error {
+	if code == http.StatusNotFound {
+		return ErrIncorrectRepositoryURL
+	} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
+		return ErrAuthenticationFailure
+	}
+	return err
+}

api/git/azure_integration_test.go

@@ -1,21 +1,26 @@
 package git
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
-	"github.com/docker/docker/pkg/ioutils"
 	_ "github.com/joho/godotenv/autoload"
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
+)
+
 func TestService_ClonePublicRepository_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
 	type args struct {
 		repositoryURLFormat string
@@ -31,7 +36,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 		{
 			name: "Clone Azure DevOps repo branch",
 			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
 				referenceName:       "refs/heads/main",
 				username:            "",
 				password:            pat,
@@ -41,8 +46,8 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 		{
 			name: "Clone Azure DevOps repo tag",
 			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
-				referenceName:       "refs/tags/v1.1",
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
+				referenceName:       "refs/heads/tags/v1.1",
 				username:            "",
 				password:            pat,
 			},
@@ -51,11 +56,9 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			dst, err := ioutils.TempDir("", "clone")
-			assert.NoError(t, err)
-			defer os.RemoveAll(dst)
+			dst := t.TempDir()
 			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
-			err = service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "")
+			err := service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "")
 			assert.NoError(t, err)
 			assert.FileExists(t, filepath.Join(dst, "README.md"))
 		})
@@ -66,14 +69,11 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
-	dst, err := ioutils.TempDir("", "clone")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dst)
+	dst := t.TempDir()
 
-	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
-	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat)
+	err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
@@ -82,14 +82,200 @@ func TestService_LatestCommitID_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
-	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
-	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
+	id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
 
+func TestService_ListRefs_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := NewService(context.TODO())
+
+	refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+}
+
+func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_ListFiles_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+	service := newService(context.TODO(), 0, 0)
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+
+	tests := []struct {
+		name       string
+		args       fetchOption
+		extensions []string
+		expect     expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 19,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"yml"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and non-existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"hcl"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}
+
+func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
+
+	time.Sleep(2 * time.Second)
+}
+
 func getRequiredValue(t *testing.T, name string) string {
 	value, ok := os.LookupEnv(name)
 	if !ok {

api/git/azure_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func Test_buildDownloadUrl(t *testing.T) {
-	a := NewAzureDownloader(nil)
+	a := NewAzureClient()
 	u, err := a.buildDownloadUrl(&azureOptions{
 		organisation: "organisation",
 		project:      "project",
@@ -29,7 +29,7 @@ func Test_buildDownloadUrl(t *testing.T) {
 }
 
 func Test_buildRootItemUrl(t *testing.T) {
-	a := NewAzureDownloader(nil)
+	a := NewAzureClient()
 	u, err := a.buildRootItemUrl(&azureOptions{
 		organisation: "organisation",
 		project:      "project",
@@ -45,6 +45,40 @@ func Test_buildRootItemUrl(t *testing.T) {
 	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
 }
 
+func Test_buildRefsUrl(t *testing.T) {
+	a := NewAzureClient()
+	u, err := a.buildRefsUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	})
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?api-version=6.0")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
+func Test_buildTreeUrl(t *testing.T) {
+	a := NewAzureClient()
+	u, err := a.buildTreeUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "sha1")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/trees/sha1?api-version=6.0&recursive=true")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
 func Test_parseAzureUrl(t *testing.T) {
 	type args struct {
 		url string
@@ -200,7 +234,7 @@ func Test_isAzureUrl(t *testing.T) {
 
 func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 	type args struct {
-		options cloneOptions
+		options baseOption
 	}
 	type basicAuth struct {
 		username, password string
@@ -213,7 +247,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "username, password embedded",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
 				},
 			},
@@ -225,7 +259,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "username, password embedded, clone options take precedence",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
 					username:      "u",
 					password:      "p",
@@ -239,7 +273,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "no credentials",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
 				},
 			},
@@ -256,11 +290,17 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 			}))
 			defer server.Close()
 
-			a := &azureDownloader{
+			a := &azureClient{
 				client:  server.Client(),
 				baseUrl: server.URL,
 			}
-			_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
+
+			option := cloneOption{
+				fetchOption: fetchOption{
+					baseOption: tt.args.options,
+				},
+			}
+			_, err := a.downloadZipFromAzureDevOps(context.Background(), option)
 			assert.Error(t, err)
 			assert.Equal(t, tt.want, zipRequestAuth)
 		})
@@ -287,22 +327,25 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
 	}))
 	defer server.Close()
 
-	a := &azureDownloader{
+	a := &azureClient{
 		client:  server.Client(),
 		baseUrl: server.URL,
 	}
 
 	tests := []struct {
 		name    string
-		args    fetchOptions
+		args    fetchOption
 		want    string
 		wantErr bool
 	}{
 		{
 			name: "should be able to parse response",
-			args: fetchOptions{
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
+				},
 				referenceName: "",
-				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			},
 			want:    "27104ad7549d9e66685e115a497533f18024be9c",
 			wantErr: false,
 		},
@@ -319,3 +362,262 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
 		})
 	}
 }
+
+type testRepoManager struct {
+	called bool
+}
+
+func (t *testRepoManager) download(_ context.Context, _ string, _ cloneOption) error {
+	t.called = true
+	return nil
+}
+
+func (t *testRepoManager) latestCommitID(_ context.Context, _ fetchOption) (string, error) {
+	return "", nil
+}
+
+func (t *testRepoManager) listRefs(_ context.Context, _ baseOption) ([]string, error) {
+	return nil, nil
+}
+
+func (t *testRepoManager) listFiles(_ context.Context, _ fetchOption) ([]string, error) {
+	return nil, nil
+}
+func Test_cloneRepository_azure(t *testing.T) {
+	tests := []struct {
+		name   string
+		url    string
+		called bool
+	}{
+		{
+			name:   "Azure HTTP URL",
+			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			called: true,
+		},
+		{
+			name:   "Azure SSH URL",
+			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			called: true,
+		},
+		{
+			name:   "Something else",
+			url:    "https://example.com",
+			called: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azure := &testRepoManager{}
+			git := &testRepoManager{}
+
+			s := &Service{azure: azure, git: git}
+			s.cloneRepository("", cloneOption{
+				fetchOption: fetchOption{
+					baseOption: baseOption{
+
+						repositoryUrl: tt.url,
+					},
+				},
+				depth: 1,
+			})
+
+			// if azure API is called, git isn't and vice versa
+			assert.Equal(t, tt.called, azure.called)
+			assert.Equal(t, tt.called, !git.called)
+		})
+	}
+}
+
+func Test_listRefs_azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewAzureClient()
+
+	type expectResult struct {
+		err       error
+		refsCount int
+	}
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	tests := []struct {
+		name   string
+		args   baseOption
+		expect expectResult
+	}{
+		{
+			name: "list refs of a real repository",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err:       nil,
+				refsCount: 2,
+			},
+		},
+		{
+			name: "list refs of a real repository with incorrect credential",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      "test-username",
+				password:      "test-token",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a real repository without providing credential",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      "",
+				password:      "",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a fake repository",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL + "fake",
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err: ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			refs, err := client.listRefs(context.TODO(), tt.args)
+			if tt.expect.err == nil {
+				assert.NoError(t, err)
+				if tt.expect.refsCount > 0 {
+					assert.Greater(t, len(refs), 0)
+				}
+			} else {
+				assert.Error(t, err)
+				assert.Equal(t, tt.expect.err, err)
+			}
+		})
+	}
+
+}
+
+func Test_listFiles_azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewAzureClient()
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	tests := []struct {
+		name   string
+		args   fetchOption
+		expect expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 19,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := client.listFiles(context.TODO(), tt.args)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}

api/git/git.go

@@ -2,48 +2,31 @@ package git
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
 )
 
-type fetchOptions struct {
-	repositoryUrl string
-	username      string
-	password      string
-	referenceName string
-}
-
-type cloneOptions struct {
-	repositoryUrl string
-	username      string
-	password      string
-	referenceName string
-	depth         int
-}
-
-type downloader interface {
-	download(ctx context.Context, dst string, opt cloneOptions) error
-	latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
-}
-
 type gitClient struct {
 	preserveGitDirectory bool
 }
 
-func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
+func NewGitClient(preserveGitDir bool) *gitClient {
+	return &gitClient{
+		preserveGitDirectory: preserveGitDir,
+	}
+}
+
+func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error {
 	gitOptions := git.CloneOptions{
 		URL:   opt.repositoryUrl,
 		Depth: opt.depth,
@@ -57,6 +40,9 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	_, err := git.PlainCloneContext(ctx, dst, false, &gitOptions)
 
 	if err != nil {
+		if err.Error() == "authentication required" {
+			return ErrAuthenticationFailure
+		}
 		return errors.Wrap(err, "failed to clone git repository")
 	}
 
@@ -67,7 +53,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	return nil
 }
 
-func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
+func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
 	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
 		Name: "origin",
 		URLs: []string{opt.repositoryUrl},
@@ -79,6 +65,9 @@ func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string
 
 	refs, err := remote.List(listOptions)
 	if err != nil {
+		if err.Error() == "authentication required" {
+			return "", ErrAuthenticationFailure
+		}
 		return "", errors.Wrap(err, "failed to list repository refs")
 	}
 
@@ -114,66 +103,78 @@ func getAuth(username, password string) *githttp.BasicAuth {
 	return nil
 }
 
-// Service represents a service for managing Git.
-type Service struct {
-	httpsCli *http.Client
-	azure    downloader
-	git      downloader
+func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
+	rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
+		Name: "origin",
+		URLs: []string{opt.repositoryUrl},
+	})
+
+	listOptions := &git.ListOptions{
+		Auth: getAuth(opt.username, opt.password),
+	}
+
+	refs, err := rem.List(listOptions)
+	if err != nil {
+		return nil, checkGitError(err)
+	}
+
+	var ret []string
+	for _, ref := range refs {
+		if ref.Name().String() == "HEAD" {
+			continue
+		}
+		ret = append(ret, ref.Name().String())
+	}
+
+	return ret, nil
 }
 
-// NewService initializes a new service.
-func NewService() *Service {
-	httpsCli := &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			Proxy:           http.ProxyFromEnvironment,
-		},
-		Timeout: 300 * time.Second,
+// listFiles list all filenames under the specific repository
+func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
+	cloneOption := &git.CloneOptions{
+		URL:           opt.repositoryUrl,
+		NoCheckout:    true,
+		Depth:         1,
+		SingleBranch:  true,
+		ReferenceName: plumbing.ReferenceName(opt.referenceName),
+		Auth:          getAuth(opt.username, opt.password),
 	}
 
-	client.InstallProtocol("https", githttp.NewClient(httpsCli))
-
-	return &Service{
-		httpsCli: httpsCli,
-		azure:    NewAzureDownloader(httpsCli),
-		git:      gitClient{},
+	repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
+	if err != nil {
+		return nil, checkGitError(err)
 	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return nil, err
+	}
+
+	commit, err := repo.CommitObject(head.Hash())
+	if err != nil {
+		return nil, err
+	}
+
+	tree, err := commit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	var allPaths []string
+	tree.Files().ForEach(func(f *object.File) error {
+		allPaths = append(allPaths, f.Name)
+		return nil
+	})
+
+	return allPaths, nil
 }
 
-// CloneRepository clones a git repository using the specified URL in the specified
-// destination folder.
-func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
-	options := cloneOptions{
-		repositoryUrl: repositoryURL,
-		username:      username,
-		password:      password,
-		referenceName: referenceName,
-		depth:         1,
+func checkGitError(err error) error {
+	errMsg := err.Error()
+	if errMsg == "repository not found" {
+		return ErrIncorrectRepositoryURL
+	} else if errMsg == "authentication required" {
+		return ErrAuthenticationFailure
 	}
-
-	return service.cloneRepository(destination, options)
-}
-
-func (service *Service) cloneRepository(destination string, options cloneOptions) error {
-	if isAzureUrl(options.repositoryUrl) {
-		return service.azure.download(context.TODO(), destination, options)
-	}
-
-	return service.git.download(context.TODO(), destination, options)
-}
-
-// LatestCommitID returns SHA1 of the latest commit of the specified reference
-func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	options := fetchOptions{
-		repositoryUrl: repositoryURL,
-		username:      username,
-		password:      password,
-		referenceName: referenceName,
-	}
-
-	if isAzureUrl(options.repositoryUrl) {
-		return service.azure.latestCommitID(context.TODO(), options)
-	}
-
-	return service.git.latestCommitID(context.TODO(), options)
+	return err
 }

api/git/git_integration_test.go

@@ -1,27 +1,29 @@
 package git
 
 import (
-	"os"
+	"context"
 	"path/filepath"
 	"testing"
+	"time"
 
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	privateGitRepoURL string = "https://github.com/portainer/private-test-repository.git"
+)
+
 func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
-	service := NewService()
+	service := newService(context.TODO(), 0, 0)
 
-	dst, err := ioutils.TempDir("", "clone")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dst)
+	dst := t.TempDir()
 
-	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
-	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
+	repositoryUrl := privateGitRepoURL
+	err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
@@ -31,10 +33,321 @@ func TestService_LatestCommitID_GitHub(t *testing.T) {
 
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
-	service := NewService()
+	service := newService(context.TODO(), 0, 0)
 
-	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	repositoryUrl := privateGitRepoURL
 	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
+
+func TestService_ListRefs_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 0, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+}
+
+func TestService_ListRefs_Github_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	repositoryUrl := privateGitRepoURL
+	go service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_ListFiles_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+	service := newService(context.TODO(), 0, 0)
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+
+	tests := []struct {
+		name       string
+		args       fetchOption
+		extensions []string
+		expect     expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 15,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"yml"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and non-existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"hcl"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}
+
+func TestService_ListFiles_Github_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_purgeCache_Github(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService(context.TODO())
+
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+
+	assert.Equal(t, 1, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	service.purgeCache()
+	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_purgeCacheByTTL_Github(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	timeout := 100 * time.Millisecond
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
+	service := newService(context.TODO(), 2, 40*timeout)
+
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.Equal(t, 1, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	// 40*timeout is designed for giving enough time for TTL being activated
+	time.Sleep(40 * timeout)
+	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) {
+	timeout := 10 * time.Millisecond
+	deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout))
+
+	service := NewService(deadlineCtx)
+	assert.False(t, service.timerHasStopped(), "timer should not be stopped")
+
+	<-time.After(20 * timeout)
+
+	assert.True(t, service.timerHasStopped(), "timer should be stopped")
+}
+
+func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 2, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	assert.Error(t, err)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+}
+
+func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 2, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 2, service.repoFileCache.Len())
+
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	assert.Error(t, err)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	assert.Error(t, err)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+	// The relevant file caches should be removed too
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	service := newService(context.TODO(), 2, 0)
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	repositoryUrl := privateGitRepoURL
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}

api/git/git_test.go

@@ -2,106 +2,76 @@ package git
 
 import (
 	"context"
-	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/portainer/portainer/api/archive"
+
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
 	"github.com/stretchr/testify/assert"
 )
 
-var bareRepoDir string
-
-func TestMain(m *testing.M) {
-	if err := testMain(m); err != nil {
-		log.Fatal(err)
-	}
-}
-
-// testMain does extra setup/teardown before/after testing.
-// The function is separated from TestMain due to necessity to call os.Exit/log.Fatal in the latter.
-func testMain(m *testing.M) error {
-	dir, err := ioutil.TempDir("", "git-repo-")
-	if err != nil {
-		return errors.Wrap(err, "failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
-
-	bareRepoDir = filepath.Join(dir, "test-clone.git")
+func setup(t *testing.T) string {
+	dir := t.TempDir()
+	bareRepoDir := filepath.Join(dir, "test-clone.git")
 
 	file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0755)
 	if err != nil {
-		return errors.Wrap(err, "failed to open an archive")
+		t.Fatal(errors.Wrap(err, "failed to open an archive"))
 	}
+
 	err = archive.ExtractTarGz(file, dir)
 	if err != nil {
-		return errors.Wrapf(err, "failed to extract file from the archive to a folder %s\n", dir)
+		t.Fatal(errors.Wrapf(err, "failed to extract file from the archive to a folder %s", dir))
 	}
 
-	m.Run()
-
-	return nil
+	return bareRepoDir
 }
 
 func Test_ClonePublicRepository_Shallow(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
-	repositoryURL := bareRepoDir
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
+	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
-	destination := "shallow"
 
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
-	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "")
 	assert.NoError(t, err)
 	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
 }
 
 func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
-	repositoryURL := bareRepoDir
+	service := Service{git: NewGitClient(false)} // no need for http client since the test access the repo via file system.
+	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
-	destination := "shallow"
-
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-
-	defer os.RemoveAll(dir)
 
+	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
-	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	err := service.CloneRepository(dir, repositoryURL, referenceName, "", "")
 	assert.NoError(t, err)
 	assert.NoDirExists(t, filepath.Join(dir, ".git"))
 }
 
 func Test_cloneRepository(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
 
-	repositoryURL := bareRepoDir
+	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
-	destination := "shallow"
 
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
 
-	err = service.cloneRepository(dir, cloneOptions{
-		repositoryUrl: repositoryURL,
-		referenceName: referenceName,
-		depth:         10,
+	err := service.cloneRepository(dir, cloneOption{
+		fetchOption: fetchOption{
+			baseOption: baseOption{
+				repositoryUrl: repositoryURL,
+			},
+			referenceName: referenceName,
+		},
+		depth: 10,
 	})
 
 	assert.NoError(t, err)
@@ -109,9 +79,9 @@ func Test_cloneRepository(t *testing.T) {
 }
 
 func Test_latestCommitID(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
 
-	repositoryURL := bareRepoDir
+	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
 
 	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "")
@@ -140,53 +110,196 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int {
 	return count
 }
 
-type testDownloader struct {
-	called bool
-}
+func Test_listRefsPrivateRepository(t *testing.T) {
+	ensureIntegrationTest(t)
 
-func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
-	t.called = true
-	return nil
-}
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
 
-func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
-	return "", nil
-}
+	client := NewGitClient(false)
+
+	type expectResult struct {
+		err       error
+		refsCount int
+	}
 
-func Test_cloneRepository_azure(t *testing.T) {
 	tests := []struct {
 		name   string
-		url    string
-		called bool
+		args   baseOption
+		expect expectResult
 	}{
 		{
-			name:   "Azure HTTP URL",
-			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
-			called: true,
+			name: "list refs of a real private repository",
+			args: baseOption{
+				repositoryUrl: privateGitRepoURL,
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err:       nil,
+				refsCount: 2,
+			},
 		},
 		{
-			name:   "Azure SSH URL",
-			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
-			called: true,
+			name: "list refs of a real private repository with incorrect credential",
+			args: baseOption{
+				repositoryUrl: privateGitRepoURL,
+				username:      "test-username",
+				password:      "test-token",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
 		},
 		{
-			name:   "Something else",
-			url:    "https://example.com",
-			called: false,
+			name: "list refs of a fake repository without providing credential",
+			args: baseOption{
+				repositoryUrl: privateGitRepoURL + "fake",
+				username:      "",
+				password:      "",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a fake repository",
+			args: baseOption{
+				repositoryUrl: privateGitRepoURL + "fake",
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err: ErrIncorrectRepositoryURL,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			azure := &testDownloader{}
-			git := &testDownloader{}
-
-			s := &Service{azure: azure, git: git}
-			s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
-
-			// if azure API is called, git isn't and vice versa
-			assert.Equal(t, tt.called, azure.called)
-			assert.Equal(t, tt.called, !git.called)
+			refs, err := client.listRefs(context.TODO(), tt.args)
+			if tt.expect.err == nil {
+				assert.NoError(t, err)
+				if tt.expect.refsCount > 0 {
+					assert.Greater(t, len(refs), 0)
+				}
+			} else {
+				assert.Error(t, err)
+				assert.Equal(t, tt.expect.err, err)
+			}
+		})
+	}
+}
+
+func Test_listFilesPrivateRepository(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewGitClient(false)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+
+	tests := []struct {
+		name   string
+		args   fetchOption
+		expect expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 15,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := client.listFiles(context.TODO(), tt.args)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
 		})
 	}
 }

api/git/service.go

@@ -0,0 +1,321 @@
+package git
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
+	ErrAuthenticationFailure  = errors.New("Authentication failed, please ensure that the git credentials are correct.")
+
+	REPOSITORY_CACHE_SIZE = 4
+	REPOSITORY_CACHE_TTL  = 5 * time.Minute
+)
+
+// baseOption provides a minimum group of information to operate a git repository, like git-remote
+type baseOption struct {
+	repositoryUrl string
+	username      string
+	password      string
+}
+
+// fetchOption allows to specify the reference name of the target repository
+type fetchOption struct {
+	baseOption
+	referenceName string
+}
+
+// cloneOption allows to add a history truncated to the specified number of commits
+type cloneOption struct {
+	fetchOption
+	depth int
+}
+
+type repoManager interface {
+	download(ctx context.Context, dst string, opt cloneOption) error
+	latestCommitID(ctx context.Context, opt fetchOption) (string, error)
+	listRefs(ctx context.Context, opt baseOption) ([]string, error)
+	listFiles(ctx context.Context, opt fetchOption) ([]string, error)
+}
+
+// Service represents a service for managing Git.
+type Service struct {
+	shutdownCtx  context.Context
+	azure        repoManager
+	git          repoManager
+	timerStopped bool
+	mut          sync.Mutex
+
+	cacheEnabled bool
+	// Cache the result of repository refs, key is repository URL
+	repoRefCache *lru.Cache
+	// Cache the result of repository file tree, key is the concatenated string of repository URL and ref value
+	repoFileCache *lru.Cache
+}
+
+// NewService initializes a new service.
+func NewService(ctx context.Context) *Service {
+	return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL)
+}
+
+func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {
+	service := &Service{
+		shutdownCtx:  ctx,
+		azure:        NewAzureClient(),
+		git:          NewGitClient(false),
+		timerStopped: false,
+		cacheEnabled: cacheSize > 0,
+	}
+
+	if service.cacheEnabled {
+		var err error
+		service.repoRefCache, err = lru.New(cacheSize)
+		if err != nil {
+			log.Debug().Err(err).Msg("failed to create ref cache")
+		}
+
+		service.repoFileCache, err = lru.New(cacheSize)
+		if err != nil {
+			log.Debug().Err(err).Msg("failed to create file cache")
+		}
+
+		if cacheTTL > 0 {
+			go service.startCacheCleanTimer(cacheTTL)
+		}
+	}
+
+	return service
+}
+
+// startCacheCleanTimer starts a timer to purge caches periodically
+func (service *Service) startCacheCleanTimer(d time.Duration) {
+	ticker := time.NewTicker(d)
+
+	for {
+		select {
+		case <-ticker.C:
+			service.purgeCache()
+
+		case <-service.shutdownCtx.Done():
+			ticker.Stop()
+			service.mut.Lock()
+			service.timerStopped = true
+			service.mut.Unlock()
+			return
+		}
+	}
+}
+
+// timerHasStopped shows the CacheClean timer state with thread-safe way
+func (service *Service) timerHasStopped() bool {
+	service.mut.Lock()
+	defer service.mut.Unlock()
+	ret := service.timerStopped
+	return ret
+}
+
+// CloneRepository clones a git repository using the specified URL in the specified
+// destination folder.
+func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	options := cloneOption{
+		fetchOption: fetchOption{
+			baseOption: baseOption{
+				repositoryUrl: repositoryURL,
+				username:      username,
+				password:      password,
+			},
+			referenceName: referenceName,
+		},
+		depth: 1,
+	}
+
+	return service.cloneRepository(destination, options)
+}
+
+func (service *Service) cloneRepository(destination string, options cloneOption) error {
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.download(context.TODO(), destination, options)
+	}
+
+	return service.git.download(context.TODO(), destination, options)
+}
+
+// LatestCommitID returns SHA1 of the latest commit of the specified reference
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	options := fetchOption{
+		baseOption: baseOption{
+			repositoryUrl: repositoryURL,
+			username:      username,
+			password:      password,
+		},
+		referenceName: referenceName,
+	}
+
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.latestCommitID(context.TODO(), options)
+	}
+
+	return service.git.latestCommitID(context.TODO(), options)
+}
+
+// ListRefs will list target repository's references without cloning the repository
+func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	refCacheKey := generateCacheKey(repositoryURL, password)
+	if service.cacheEnabled && hardRefresh {
+		// Should remove the cache explicitly, so that the following normal list can show the correct result
+		service.repoRefCache.Remove(refCacheKey)
+		// Remove file caches pointed to the same repository
+		for _, fileCacheKey := range service.repoFileCache.Keys() {
+			key, ok := fileCacheKey.(string)
+			if ok {
+				if strings.HasPrefix(key, repositoryURL) {
+					service.repoFileCache.Remove(key)
+				}
+			}
+		}
+	}
+
+	if service.repoRefCache != nil {
+		// Lookup the refs cache first
+		cache, ok := service.repoRefCache.Get(refCacheKey)
+		if ok {
+			refs, success := cache.([]string)
+			if success {
+				return refs, nil
+			}
+		}
+	}
+
+	options := baseOption{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+	}
+
+	var (
+		refs []string
+		err  error
+	)
+	if isAzureUrl(options.repositoryUrl) {
+		refs, err = service.azure.listRefs(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		refs, err = service.git.listRefs(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if service.cacheEnabled && service.repoRefCache != nil {
+		service.repoRefCache.Add(refCacheKey, refs)
+	}
+	return refs, nil
+}
+
+// ListFiles will list all the files of the target repository with specific extensions.
+// If extension is not provided, it will list all the files under the target repository
+func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
+	repoKey := generateCacheKey(repositoryURL, referenceName)
+
+	if service.cacheEnabled && hardRefresh {
+		// Should remove the cache explicitly, so that the following normal list can show the correct result
+		service.repoFileCache.Remove(repoKey)
+	}
+
+	if service.repoFileCache != nil {
+		// lookup the files cache first
+		cache, ok := service.repoFileCache.Get(repoKey)
+		if ok {
+			files, success := cache.([]string)
+			if success {
+				// For the case while searching files in a repository without include extensions for the first time,
+				// but with include extensions for the second time
+				includedFiles := filterFiles(files, includedExts)
+				return includedFiles, nil
+			}
+		}
+	}
+
+	options := fetchOption{
+		baseOption: baseOption{
+			repositoryUrl: repositoryURL,
+			username:      username,
+			password:      password,
+		},
+		referenceName: referenceName,
+	}
+
+	var (
+		files []string
+		err   error
+	)
+	if isAzureUrl(options.repositoryUrl) {
+		files, err = service.azure.listFiles(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		files, err = service.git.listFiles(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	includedFiles := filterFiles(files, includedExts)
+	if service.cacheEnabled && service.repoFileCache != nil {
+		service.repoFileCache.Add(repoKey, includedFiles)
+		return includedFiles, nil
+	}
+	return includedFiles, nil
+}
+
+func (service *Service) purgeCache() {
+	if service.repoRefCache != nil {
+		service.repoRefCache.Purge()
+	}
+
+	if service.repoFileCache != nil {
+		service.repoFileCache.Purge()
+	}
+}
+
+func generateCacheKey(names ...string) string {
+	return strings.Join(names, "-")
+}
+
+func matchExtensions(target string, exts []string) bool {
+	if len(exts) == 0 {
+		return true
+	}
+
+	for _, ext := range exts {
+		if strings.HasSuffix(target, ext) {
+			return true
+		}
+	}
+	return false
+}
+
+func filterFiles(paths []string, includedExts []string) []string {
+	if len(includedExts) == 0 {
+		return paths
+	}
+
+	var includedFiles []string
+	for _, filename := range paths {
+		// filter out the filenames with non-included extension
+		if matchExtensions(filename, includedExts) {
+			includedFiles = append(includedFiles, filename)
+		}
+	}
+	return includedFiles
+}

api/git/types/types.go

@@ -17,4 +17,8 @@ type RepoConfig struct {
 type GitAuthentication struct {
 	Username string
 	Password string
+	// Git credentials identifier when the value is not 0
+	// When the value is 0, Username and Password are set without using saved credential
+	// This is introduced since 2.15.0
+	GitCredentialID int `example:"0"`
 }