fix swagger param

* feat(k8s/node): display realtime node metrics GH#4612

* feat(k8s): show observation timestamp instead of real timestamp GH#4612

Co-authored-by: Simon Meng <simon.meng@portainer.io>

.gitignore

@@ -11,3 +11,5 @@ api/cmd/portainer/portainer*
 __debug_bin
 
 api/docs
+.idea
+.env

ATTRIBUTIONS.md

@@ -0,0 +1,32 @@
+# Open Source License Attribution
+
+This application uses Open Source components. You can find the source
+code of their open source projects along with license information below.
+We acknowledge and are grateful to these developers for their contributions
+to open source.
+
+### [angular-json-tree](https://github.com/awendland/angular-json-tree)
+
+by [Alex Wendland](https://github.com/awendland) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
+
+### [caniuse-db](https://github.com/Fyrd/caniuse)
+
+by [caniuse.com](caniuse.com) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
+
+### [caniuse-lite](https://github.com/ben-eb/caniuse-lite)
+
+by [caniuse.com](caniuse.com) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
+
+### [spdx-exceptions](https://github.com/jslicense/spdx-exceptions.json)
+
+by Kyle Mitchell using [SPDX](https://spdx.dev/) from Linux Foundation licensed under [CC BY 3.0 License](https://creativecommons.org/licenses/by/3.0/)
+
+### [fontawesome-free](https://github.com/FortAwesome/Font-Awesome) Icons
+
+by [Fort Awesome](https://fortawesome.com/) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
+
+Portainer also contains the following code, which is licensed under the [MIT license](https://opensource.org/licenses/MIT):
+
+UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
+
+rdash-angular: Copyright (c) [2014][elliot hesp]

CONTRIBUTING.md

@@ -127,3 +127,9 @@ When adding a new route to an existing handler use the following as a template (
 ```
 
 explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]
+
+## Licensing
+
+See the [LICENSE](https://github.com/portainer/portainer/blob/develop/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+
+We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.

README.md

@@ -65,8 +65,4 @@ Portainer supports "Current - 2 docker versions only. Prior versions may operate
 
 Portainer is licensed under the zlib license. See [LICENSE](./LICENSE) for reference.
 
-Portainer also contains the following code, which is licensed under the [MIT license](https://opensource.org/licenses/MIT):
-
-UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
-
-rdash-angular: Copyright (c) [2014][elliot hesp]
+Portainer also contains code from open source projects. See [ATTRIBUTIONS.md](./ATTRIBUTIONS.md) for a list.

api/adminmonitor/admin_monitor.go

@@ -0,0 +1,69 @@
+package adminmonitor
+
+import (
+	"context"
+	"log"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+var logFatalf = log.Fatalf
+
+type Monitor struct {
+	timeout          time.Duration
+	datastore        portainer.DataStore
+	shutdownCtx      context.Context
+	cancellationFunc context.CancelFunc
+}
+
+// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
+func New(timeout time.Duration, datastore portainer.DataStore, shutdownCtx context.Context) *Monitor {
+	return &Monitor{
+		timeout:     timeout,
+		datastore:   datastore,
+		shutdownCtx: shutdownCtx,
+	}
+}
+
+// Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
+func (m *Monitor) Start() {
+	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
+	m.cancellationFunc = cancellationFunc
+
+	go func() {
+		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
+		select {
+		case <-time.After(m.timeout):
+			initialized, err := m.WasInitialized()
+			if err != nil {
+				logFatalf("%s", err)
+			}
+			if !initialized {
+				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
+			}
+		case <-cancellationCtx.Done():
+			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
+		case <-m.shutdownCtx.Done():
+			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
+		}
+	}()
+}
+
+// Stop stops monitor. Safe to call even if monitor wasn't started.
+func (m *Monitor) Stop() {
+	if m.cancellationFunc == nil {
+		return
+	}
+	m.cancellationFunc()
+	m.cancellationFunc = nil
+}
+
+// WasInitialized is a system initialization check
+func (m *Monitor) WasInitialized() (bool, error) {
+	users, err := m.datastore.User().UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		return false, err
+	}
+	return len(users) > 0, nil
+}

api/adminmonitor/admin_monitor_test.go

@@ -0,0 +1,50 @@
+package adminmonitor
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	i "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_stopWithoutStarting(t *testing.T) {
+	monitor := New(1*time.Minute, nil, nil)
+	monitor.Stop()
+}
+
+func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
+	monitor := New(1*time.Minute, nil, nil)
+	monitor.Stop()
+	monitor.Stop()
+}
+
+func Test_canStopStartedMonitor(t *testing.T) {
+	monitor := New(1*time.Minute, nil, context.Background())
+	monitor.Start()
+	assert.NotNil(t, monitor.cancellationFunc, "cancellation function is missing in started monitor")
+
+	monitor.Stop()
+	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
+}
+
+func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
+	timeout := 10 * time.Millisecond
+
+	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
+
+	var fataled bool
+	origLogFatalf := logFatalf
+	logFatalf = func(s string, v ...interface{}) { fataled = true }
+	defer func() {
+		logFatalf = origLogFatalf
+	}()
+
+	monitor := New(timeout, datastore, context.Background())
+	monitor.Start()
+	<-time.After(2 * timeout)
+
+	assert.True(t, fataled, "monitor should been timeout and fatal")
+}

api/archive/targz.go

@@ -0,0 +1,119 @@
+package archive
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// TarGzDir creates a tar.gz archive and returns it's path.
+// abosolutePath should be an absolute path to a directory.
+// Archive name will be <directoryName>.tar.gz and will be placed next to the directory.
+func TarGzDir(absolutePath string) (string, error) {
+	targzPath := filepath.Join(absolutePath, fmt.Sprintf("%s.tar.gz", filepath.Base(absolutePath)))
+	outFile, err := os.Create(targzPath)
+	if err != nil {
+		return "", err
+	}
+	defer outFile.Close()
+
+	zipWriter := gzip.NewWriter(outFile)
+	defer zipWriter.Close()
+	tarWriter := tar.NewWriter(zipWriter)
+	defer tarWriter.Close()
+
+	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == targzPath {
+			return nil // skip archive file
+		}
+
+		pathInArchive := filepath.Clean(strings.TrimPrefix(path, absolutePath))
+		if pathInArchive == "" {
+			return nil // skip root dir
+		}
+
+		return addToArchive(tarWriter, pathInArchive, path, info)
+	})
+
+	return targzPath, err
+}
+
+func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info os.FileInfo) error {
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return err
+	}
+
+	header.Name = pathInArchive // use relative paths in archive
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	if info.IsDir() {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(tarWriter, file)
+	return err
+}
+
+// ExtractTarGz reads a .tar.gz archive from the reader and extracts it into outputDirPath directory
+func ExtractTarGz(r io.Reader, outputDirPath string) error {
+	zipReader, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer zipReader.Close()
+
+	tarReader := tar.NewReader(zipReader)
+
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			// skip, dir will be created with a file
+		case tar.TypeReg:
+			p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
+			if err := os.MkdirAll(filepath.Dir(p), 0744); err != nil {
+				return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
+			}
+			outFile, err := os.Create(p)
+			if err != nil {
+				return fmt.Errorf("Failed to create file %s", header.Name)
+			}
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				return fmt.Errorf("Failed to extract file %s", header.Name)
+			}
+			outFile.Close()
+		default:
+			return fmt.Errorf("Tar: uknown type: %v in %s",
+				header.Typeflag,
+				header.Name)
+		}
+	}
+
+	return nil
+}

api/archive/targz_test.go

@@ -0,0 +1,99 @@
+package archive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func listFiles(dir string) []string {
+	items := make([]string, 0)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		items = append(items, path)
+		return nil
+	})
+
+	return items
+}
+
+func Test_shouldCreateArhive(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+
+	gzPath, err := TarGzDir(tmpdir)
+	assert.Nil(t, err)
+	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
+
+	extractionDir, _ := ioutils.TempDir("", "extract")
+	defer os.RemoveAll(extractionDir)
+
+	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
+	err = cmd.Run()
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+	extractedFiles := listFiles(extractionDir)
+
+	wasExtracted := func(p string) {
+		fullpath := path.Join(extractionDir, p)
+		assert.Contains(t, extractedFiles, fullpath)
+		copyContent, _ := ioutil.ReadFile(fullpath)
+		assert.Equal(t, content, copyContent)
+	}
+
+	wasExtracted("outer")
+	wasExtracted("dir/inner")
+	wasExtracted("dir/.dotfile")
+}
+
+func Test_shouldCreateArhiveXXXXX(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+
+	gzPath, err := TarGzDir(tmpdir)
+	assert.Nil(t, err)
+	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
+
+	extractionDir, _ := ioutils.TempDir("", "extract")
+	defer os.RemoveAll(extractionDir)
+
+	r, _ := os.Open(gzPath)
+	ExtractTarGz(r, extractionDir)
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+	extractedFiles := listFiles(extractionDir)
+
+	wasExtracted := func(p string) {
+		fullpath := path.Join(extractionDir, p)
+		assert.Contains(t, extractedFiles, fullpath)
+		copyContent, _ := ioutil.ReadFile(fullpath)
+		assert.Equal(t, content, copyContent)
+	}
+
+	wasExtracted("outer")
+	wasExtracted("dir/inner")
+	wasExtracted("dir/.dotfile")
+}

api/archive/zip.go

@@ -3,10 +3,13 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
+	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -52,3 +55,60 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 
 	return outFile.Close()
 }
+
+// UnzipFile will decompress a zip archive, moving all files and folders
+// within the zip file (parameter 1) to an output directory (parameter 2).
+func UnzipFile(src string, dest string) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		p := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+		if !strings.HasPrefix(p, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: illegal file path", p)
+		}
+
+		if f.FileInfo().IsDir() {
+			// Make Folder
+			os.MkdirAll(p, os.ModePerm)
+			continue
+		}
+
+		err = unzipFile(f, p)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func unzipFile(f *zip.File, p string) error {
+	// Make File
+	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
+		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
+	}
+	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
+	}
+	defer outFile.Close()
+	rc, err := f.Open()
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
+	}
+	defer rc.Close()
+
+	_, err = io.Copy(outFile, rc)
+
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
+	}
+
+	return nil
+}

api/archive/zip_test.go

@@ -0,0 +1,32 @@
+package archive
+
+import (
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestUnzipFile(t *testing.T) {
+	dir, err := ioutil.TempDir("", "unzip-test-")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+	/*
+		Archive structure.
+		├── 0
+		│	├── 1
+		│	│	└── 2.txt
+		│	└── 1.txt
+		└── 0.txt
+	*/
+
+	err = UnzipFile("./testdata/sample_archive.zip", dir)
+
+	assert.NoError(t, err)
+	archiveDir := dir + "/sample_archive"
+	assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
+
+}

api/backup/backup.go

@@ -0,0 +1,83 @@
+package backup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/offlinegate"
+)
+
+const rwxr__r__ os.FileMode = 0744
+
+var filesToBackup = []string{"compose", "config.json", "custom_templates", "edge_jobs", "edge_stacks", "extensions", "portainer.key", "portainer.pub", "tls"}
+
+// Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
+func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
+	unlock := gate.Lock()
+	defer unlock()
+
+	backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
+	if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
+		return "", errors.Wrap(err, "Failed to create backup dir")
+	}
+
+	if err := backupDb(backupDirPath, datastore); err != nil {
+		return "", errors.Wrap(err, "Failed to backup database")
+	}
+
+	for _, filename := range filesToBackup {
+		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
+		if err != nil {
+			return "", errors.Wrap(err, "Failed to create backup file")
+		}
+	}
+
+	archivePath, err := archive.TarGzDir(backupDirPath)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to make an archive")
+	}
+
+	if password != "" {
+		archivePath, err = encrypt(archivePath, password)
+		if err != nil {
+			return "", errors.Wrap(err, "Failed to encrypt backup with the password")
+		}
+	}
+
+	return archivePath, nil
+}
+
+func backupDb(backupDirPath string, datastore portainer.DataStore) error {
+	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
+	if err != nil {
+		return err
+	}
+	if err = datastore.BackupTo(backupWriter); err != nil {
+		return err
+	}
+	return backupWriter.Close()
+}
+
+func encrypt(path string, passphrase string) (string, error) {
+	in, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer in.Close()
+
+	outFileName := fmt.Sprintf("%s.encrypted", path)
+	out, err := os.Create(outFileName)
+	if err != nil {
+		return "", err
+	}
+
+	err = crypto.AesEncrypt(in, out, []byte(passphrase))
+
+	return outFileName, err
+}

api/backup/copy.go

@@ -0,0 +1,68 @@
+package backup
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func copyPath(path string, toDir string) error {
+	info, err := os.Stat(path)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		// skip copy if file does not exist
+		return nil
+	}
+
+	if !info.IsDir() {
+		destination := filepath.Join(toDir, info.Name())
+		return copyFile(path, destination)
+	}
+
+	return copyDir(path, toDir)
+}
+
+func copyDir(fromDir, toDir string) error {
+	cleanedSourcePath := filepath.Clean(fromDir)
+	parentDirectory := filepath.Dir(cleanedSourcePath)
+	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		if info.IsDir() {
+			return nil // skip directory creations
+		}
+
+		if info.Mode()&os.ModeSymlink != 0 { // entry is a symlink
+			return nil // don't copy symlinks
+		}
+
+		return copyFile(path, destination)
+	})
+
+	return err
+}
+
+// copies regular a file from src to dst
+func copyFile(src, dst string) error {
+	from, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer from.Close()
+
+	// has to include 'execute' bit, otherwise fails. MkdirAll follows `mkdir -m` restrictions
+	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
+		return err
+	}
+	to, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer to.Close()
+
+	_, err = io.Copy(to, from)
+	return err
+}

api/backup/copy_test.go

@@ -0,0 +1,105 @@
+package backup
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func listFiles(dir string) []string {
+	items := make([]string, 0)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		items = append(items, path)
+		return nil
+	})
+
+	return items
+}
+
+func contains(t *testing.T, list []string, path string) {
+	assert.Contains(t, list, path)
+	copyContent, _ := ioutil.ReadFile(path)
+	assert.Equal(t, "content\n", string(copyContent))
+}
+
+func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyFile("does-not-exist", tmpdir)
+	assert.NotNil(t, err)
+}
+
+func Test_copyFile_shouldMakeAbackup(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+
+	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
+	assert.Nil(t, err)
+
+	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
+	destination, _ := ioutils.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := copyDir("./test_assets/copy_test", destination)
+	assert.Nil(t, err)
+
+	createdFiles := listFiles(destination)
+
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
+}
+
+func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyPath("does-not-exists", tmpdir)
+	assert.Nil(t, err)
+
+	assert.Empty(t, listFiles(tmpdir))
+}
+
+func Test_backupPath_shouldCopyFile(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+
+	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
+	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
+	assert.Nil(t, err)
+
+	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	assert.Nil(t, err)
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_backupPath_shouldCopyDir(t *testing.T) {
+	destination, _ := ioutils.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := copyPath("./test_assets/copy_test", destination)
+	assert.Nil(t, err)
+
+	createdFiles := listFiles(destination)
+
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
+}

api/backup/restore.go

@@ -0,0 +1,68 @@
+package backup
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/offlinegate"
+)
+
+var filesToRestore = append(filesToBackup, "portainer.db")
+
+// Restores system state from backup archive, will trigger system shutdown, when finished.
+func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, shutdownTrigger context.CancelFunc) error {
+	var err error
+	if password != "" {
+		archive, err = decrypt(archive, password)
+		if err != nil {
+			return errors.Wrap(err, "failed to decrypt the archive")
+		}
+	}
+
+	restorePath := filepath.Join(filestorePath, "restore", time.Now().Format("20060102150405"))
+	defer os.RemoveAll(filepath.Dir(restorePath))
+
+	err = extractArchive(archive, restorePath)
+	if err != nil {
+		return errors.Wrap(err, "cannot extract files from the archive. Please ensure the password is correct and try again")
+	}
+
+	unlock := gate.Lock()
+	defer unlock()
+
+	if err = datastore.Close(); err != nil {
+		return errors.Wrap(err, "Failed to stop db")
+	}
+
+	if err = restoreFiles(restorePath, filestorePath); err != nil {
+		return errors.Wrap(err, "failed to restore the system state")
+	}
+
+	shutdownTrigger()
+	return nil
+}
+
+func decrypt(r io.Reader, password string) (io.Reader, error) {
+	return crypto.AesDecrypt(r, []byte(password))
+}
+
+func extractArchive(r io.Reader, destinationDirPath string) error {
+	return archive.ExtractTarGz(r, destinationDirPath)
+}
+
+func restoreFiles(srcDir string, destinationDir string) error {
+	for _, filename := range filesToRestore {
+		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/backup/test_assets/copy_test/dir/.dotfile

@@ -0,0 +1 @@
+content

api/backup/test_assets/copy_test/dir/inner

@@ -0,0 +1 @@
+content

api/backup/test_assets/copy_test/outer

@@ -0,0 +1 @@
+content

api/bolt/bolttest/datastore.go

@@ -0,0 +1,73 @@
+package bolttest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+var errTempDir = errors.New("can't create a temp dir")
+
+func MustNewTestStore(init bool) (*bolt.Store, func()) {
+	store, teardown, err := NewTestStore(init)
+	if err != nil {
+		if !errors.Is(err, errTempDir) {
+			teardown()
+		}
+		log.Fatal(err)
+	}
+
+	return store, teardown
+}
+
+func NewTestStore(init bool) (*bolt.Store, func(), error) {
+	// Creates unique temp directory in a concurrency friendly manner.
+	dataStorePath, err := ioutil.TempDir("", "boltdb")
+	if err != nil {
+		return nil, nil, errors.Wrap(errTempDir, err.Error())
+	}
+
+	fileService, err := filesystem.NewService(dataStorePath, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	store, err := bolt.NewStore(dataStorePath, fileService)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = store.Open()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if init {
+		err = store.Init()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	teardown := func() {
+		teardown(store, dataStorePath)
+	}
+
+	return store, teardown, nil
+}
+
+func teardown(store *bolt.Store, dataStorePath string) {
+	err := store.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	err = os.RemoveAll(dataStorePath)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

api/bolt/customtemplate/customtemplate.go

@@ -2,7 +2,7 @@ package customtemplate
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 	var customTemplates = make([]portainer.CustomTemplate, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 	var customTemplate portainer.CustomTemplate
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &customTemplate)
+	err := internal.GetObject(service.connection, BucketName, identifier, &customTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 // UpdateCustomTemplate updates an custom template.
 func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, customTemplate)
+	return internal.UpdateObject(service.connection, BucketName, identifier, customTemplate)
 }
 
 // DeleteCustomTemplate deletes an custom template.
 func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // CreateCustomTemplate assign an ID to a new custom template and saves it.
 func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTemplate) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(customTemplate)
@@ -92,5 +92,5 @@ func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTem
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/datastore.go

@@ -1,6 +1,7 @@
 package bolt
 
 import (
+	"io"
 	"log"
 	"path"
 	"time"
@@ -17,6 +18,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/internal"
 	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
@@ -42,7 +44,7 @@ const (
 // BoltDB as the storage system.
 type Store struct {
 	path                    string
-	db                      *bolt.DB
+	connection              *internal.DbConnection
 	isNew                   bool
 	fileService             portainer.FileService
 	CustomTemplateService   *customtemplate.Service
@@ -83,6 +85,7 @@ func NewStore(storePath string, fileService portainer.FileService) (*Store, erro
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
+		connection:  &internal.DbConnection{},
 	}
 
 	databasePath := path.Join(storePath, databaseFileName)
@@ -105,15 +108,15 @@ func (store *Store) Open() error {
 	if err != nil {
 		return err
 	}
-	store.db = db
+	store.connection.DB = db
 
 	return store.initServices()
 }
 
 // Close closes the BoltDB database.
 func (store *Store) Close() error {
-	if store.db != nil {
-		return store.db.Close()
+	if store.connection.DB != nil {
+		return store.connection.Close()
 	}
 	return nil
 }
@@ -134,8 +137,9 @@ func (store *Store) CheckCurrentEdition() error {
 
 // MigrateData automatically migrate the data based on the DBVersion.
 // This process is only triggered on an existing database, not if the database was just created.
-func (store *Store) MigrateData() error {
-	if store.isNew {
+// if force is true, then migrate regardless.
+func (store *Store) MigrateData(force bool) error {
+	if store.isNew && !force {
 		return store.VersionService.StoreDBVersion(portainer.DBVersion)
 	}
 
@@ -148,7 +152,7 @@ func (store *Store) MigrateData() error {
 
 	if version < portainer.DBVersion {
 		migratorParams := &migrator.Parameters{
-			DB:                      store.db,
+			DB:                      store.connection.DB,
 			DatabaseVersion:         version,
 			EndpointGroupService:    store.EndpointGroupService,
 			EndpointService:         store.EndpointService,
@@ -180,238 +184,11 @@ func (store *Store) MigrateData() error {
 	return nil
 }
 
-func (store *Store) initServices() error {
-	authorizationsetService, err := role.NewService(store.db)
-	if err != nil {
+// BackupTo backs up db to a provided writer.
+// It does hot backup and doesn't block other database reads and writes
+func (store *Store) BackupTo(w io.Writer) error {
+	return store.connection.View(func(tx *bolt.Tx) error {
+		_, err := tx.WriteTo(w)
 		return err
-	}
-	store.RoleService = authorizationsetService
-
-	customTemplateService, err := customtemplate.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.CustomTemplateService = customTemplateService
-
-	dockerhubService, err := dockerhub.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.DockerHubService = dockerhubService
-
-	edgeStackService, err := edgestack.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EdgeStackService = edgeStackService
-
-	edgeGroupService, err := edgegroup.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EdgeGroupService = edgeGroupService
-
-	edgeJobService, err := edgejob.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EdgeJobService = edgeJobService
-
-	endpointgroupService, err := endpointgroup.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointGroupService = endpointgroupService
-
-	endpointService, err := endpoint.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointService = endpointService
-
-	endpointRelationService, err := endpointrelation.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointRelationService = endpointRelationService
-
-	extensionService, err := extension.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ExtensionService = extensionService
-
-	registryService, err := registry.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.RegistryService = registryService
-
-	resourcecontrolService, err := resourcecontrol.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ResourceControlService = resourcecontrolService
-
-	settingsService, err := settings.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.SettingsService = settingsService
-
-	stackService, err := stack.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.StackService = stackService
-
-	tagService, err := tag.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TagService = tagService
-
-	teammembershipService, err := teammembership.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TeamMembershipService = teammembershipService
-
-	teamService, err := team.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TeamService = teamService
-
-	tunnelServerService, err := tunnelserver.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TunnelServerService = tunnelServerService
-
-	userService, err := user.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.UserService = userService
-
-	versionService, err := version.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.VersionService = versionService
-
-	webhookService, err := webhook.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.WebhookService = webhookService
-
-	scheduleService, err := schedule.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ScheduleService = scheduleService
-
-	return nil
-}
-
-// CustomTemplate gives access to the CustomTemplate data management layer
-func (store *Store) CustomTemplate() portainer.CustomTemplateService {
-	return store.CustomTemplateService
-}
-
-// DockerHub gives access to the DockerHub data management layer
-func (store *Store) DockerHub() portainer.DockerHubService {
-	return store.DockerHubService
-}
-
-// EdgeGroup gives access to the EdgeGroup data management layer
-func (store *Store) EdgeGroup() portainer.EdgeGroupService {
-	return store.EdgeGroupService
-}
-
-// EdgeJob gives access to the EdgeJob data management layer
-func (store *Store) EdgeJob() portainer.EdgeJobService {
-	return store.EdgeJobService
-}
-
-// EdgeStack gives access to the EdgeStack data management layer
-func (store *Store) EdgeStack() portainer.EdgeStackService {
-	return store.EdgeStackService
-}
-
-// Endpoint gives access to the Endpoint data management layer
-func (store *Store) Endpoint() portainer.EndpointService {
-	return store.EndpointService
-}
-
-// EndpointGroup gives access to the EndpointGroup data management layer
-func (store *Store) EndpointGroup() portainer.EndpointGroupService {
-	return store.EndpointGroupService
-}
-
-// EndpointRelation gives access to the EndpointRelation data management layer
-func (store *Store) EndpointRelation() portainer.EndpointRelationService {
-	return store.EndpointRelationService
-}
-
-// Registry gives access to the Registry data management layer
-func (store *Store) Registry() portainer.RegistryService {
-	return store.RegistryService
-}
-
-// ResourceControl gives access to the ResourceControl data management layer
-func (store *Store) ResourceControl() portainer.ResourceControlService {
-	return store.ResourceControlService
-}
-
-// Role gives access to the Role data management layer
-func (store *Store) Role() portainer.RoleService {
-	return store.RoleService
-}
-
-// Settings gives access to the Settings data management layer
-func (store *Store) Settings() portainer.SettingsService {
-	return store.SettingsService
-}
-
-// Stack gives access to the Stack data management layer
-func (store *Store) Stack() portainer.StackService {
-	return store.StackService
-}
-
-// Tag gives access to the Tag data management layer
-func (store *Store) Tag() portainer.TagService {
-	return store.TagService
-}
-
-// TeamMembership gives access to the TeamMembership data management layer
-func (store *Store) TeamMembership() portainer.TeamMembershipService {
-	return store.TeamMembershipService
-}
-
-// Team gives access to the Team data management layer
-func (store *Store) Team() portainer.TeamService {
-	return store.TeamService
-}
-
-// TunnelServer gives access to the TunnelServer data management layer
-func (store *Store) TunnelServer() portainer.TunnelServerService {
-	return store.TunnelServerService
-}
-
-// User gives access to the User data management layer
-func (store *Store) User() portainer.UserService {
-	return store.UserService
-}
-
-// Version gives access to the Version data management layer
-func (store *Store) Version() portainer.VersionService {
-	return store.VersionService
-}
-
-// Webhook gives access to the Webhook data management layer
-func (store *Store) Webhook() portainer.WebhookService {
-	return store.WebhookService
+	})
 }

api/bolt/dockerhub/dockerhub.go

@@ -1,10 +1,8 @@
 package dockerhub
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing Dockerhub data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
-	err := internal.GetObject(service.db, BucketName, []byte(dockerHubKey), &dockerhub)
+	err := internal.GetObject(service.connection, BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(dockerHubKey), dockerhub)
+	return internal.UpdateObject(service.connection, BucketName, []byte(dockerHubKey), dockerhub)
 }

api/bolt/edgegroup/edgegroup.go

@@ -2,7 +2,7 @@ package edgegroup
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 	var groups = make([]portainer.EdgeGroup, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 	var group portainer.EdgeGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &group)
+	err := internal.GetObject(service.connection, BucketName, identifier, &group)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 // UpdateEdgeGroup updates an Edge group.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, group)
+	return internal.UpdateObject(service.connection, BucketName, identifier, group)
 }
 
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) CreateEdgeGroup(group *portainer.EdgeGroup) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/edgejob/edgejob.go

@@ -2,7 +2,7 @@ package edgejob
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 	var edgeJob portainer.EdgeJob
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &edgeJob)
+	err := internal.GetObject(service.connection, BucketName, identifier, &edgeJob)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 
 // CreateEdgeJob creates a new Edge job
 func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeJob.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
 // UpdateEdgeJob updates an Edge job by ID
 func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, edgeJob)
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeJob)
 }
 
 // DeleteEdgeJob deletes an Edge job
 func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/edgestack/edgestack.go

@@ -2,7 +2,7 @@ package edgestack
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 	var stack portainer.EdgeStack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &stack)
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 
 // CreateEdgeStack assign an ID to a new Edge stack and saves it.
 func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeStack.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
 // UpdateEdgeStack updates an Edge stack.
 func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, edgeStack)
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/endpoint/endpoint.go

@@ -2,7 +2,7 @@ package endpoint
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 	var endpoint portainer.Endpoint
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpoint)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -44,20 +44,20 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 // UpdateEndpoint updates an endpoint.
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpoint)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpoint)
 }
 
 // DeleteEndpoint deletes an endpoint.
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // Endpoints return an array containing all the endpoints.
 func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -78,7 +78,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 
 // CreateEndpoint assign an ID to a new endpoint and saves it.
 func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for endpoints
@@ -98,12 +98,12 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }
 
 // Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		for _, endpoint := range toCreate {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,7 +1,7 @@
 package endpointgroup
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 	var endpointGroup portainer.EndpointGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpointGroup)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointGroup)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 // UpdateEndpointGroup updates an endpoint group.
 func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpointGroup)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointGroup)
 }
 
 // DeleteEndpointGroup deletes an endpoint group.
 func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // EndpointGroups return an array containing all the endpoint groups.
 func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -79,7 +79,7 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 
 // CreateEndpointGroup assign an ID to a new endpoint group and saves it.
 func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/endpointrelation/endpointrelation.go

@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint relation data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 	var endpointRelation portainer.EndpointRelation
 	identifier := internal.Itob(int(endpointID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpointRelation)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointRelation)
 	if err != nil {
 		return nil, err
 	}
@@ -43,7 +43,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 
 // CreateEndpointRelation saves endpointRelation
 func (service *Service) CreateEndpointRelation(endpointRelation *portainer.EndpointRelation) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(endpointRelation)
@@ -58,11 +58,11 @@ func (service *Service) CreateEndpointRelation(endpointRelation *portainer.Endpo
 // UpdateEndpointRelation updates an Endpoint relation object
 func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpointRelation)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointRelation)
 }
 
 // DeleteEndpointRelation deletes an Endpoint relation object
 func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/errors/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 var (
 	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documention to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 )

api/bolt/extension/extension.go

@@ -1,7 +1,7 @@
 package extension
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 	var extension portainer.Extension
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &extension)
+	err := internal.GetObject(service.connection, BucketName, identifier, &extension)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(extension)
@@ -82,5 +82,5 @@ func (service *Service) Persist(extension *portainer.Extension) error {
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/internal/db.go

@@ -7,6 +7,10 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
+type DbConnection struct {
+	*bolt.DB
+}
+
 // Itob returns an 8-byte big endian representation of v.
 // This function is typically used for encoding integer IDs to byte slices
 // so that they can be used as BoltDB keys.
@@ -17,8 +21,8 @@ func Itob(v int) []byte {
 }
 
 // CreateBucket is a generic function used to create a bucket inside a bolt database.
-func CreateBucket(db *bolt.DB, bucketName string) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func CreateBucket(connection *DbConnection, bucketName string) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
 		if err != nil {
 			return err
@@ -28,10 +32,10 @@ func CreateBucket(db *bolt.DB, bucketName string) error {
 }
 
 // GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
-func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+func GetObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
 	var data []byte
 
-	err := db.View(func(tx *bolt.Tx) error {
+	err := connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		value := bucket.Get(key)
@@ -52,8 +56,8 @@ func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) e
 }
 
 // UpdateObject is a generic function used to update an object inside a bolt database.
-func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func UpdateObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		data, err := MarshalObject(object)
@@ -71,18 +75,18 @@ func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}
 }
 
 // DeleteObject is a generic function used to delete an object inside a bolt database.
-func DeleteObject(db *bolt.DB, bucketName string, key []byte) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func DeleteObject(connection *DbConnection, bucketName string, key []byte) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
 }
 
 // GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
-func GetNextIdentifier(db *bolt.DB, bucketName string) int {
+func GetNextIdentifier(connection *DbConnection, bucketName string) int {
 	var identifier int
 
-	db.Update(func(tx *bolt.Tx) error {
+	connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {

api/bolt/migrator/migrate_dbversion30.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDB31() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	legacySettings.OAuthSettings.SSO = false
+	legacySettings.OAuthSettings.LogoutURI = ""
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion30_test.go

@@ -0,0 +1,64 @@
+package migrator
+
+import (
+	"os"
+	"testing"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+var (
+	testingDBStorePath string
+	testingDBFileName  string
+	dummyLogoURL       string
+	dbConn             *bolt.DB
+	settingsService    *settings.Service
+)
+
+func setup() error {
+	testingDBStorePath, _ = os.Getwd()
+	testingDBFileName = "portainer-ee-mig-30.db"
+	dummyLogoURL = "example.com"
+	var err error
+	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
+	if err != nil {
+		return err
+	}
+	dummySettingsObj := map[string]interface{}{
+		"LogoURL": dummyLogoURL,
+	}
+	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestUpdateSettingsToDB31(t *testing.T) {
+	if err := setup(); err != nil {
+		t.Errorf("failed to complete testing setups, err: %v", err)
+	}
+	defer dbConn.Close()
+	defer os.Remove(testingDBFileName)
+	m := &Migrator{
+		db:              dbConn,
+		settingsService: settingsService,
+	}
+	if err := m.updateSettingsToDB31(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+	updatedSettings, err := m.settingsService.Settings()
+	if err != nil {
+		t.Errorf("failed to retrieve the updated settings: %v", err)
+	}
+	if updatedSettings.LogoURL != dummyLogoURL {
+		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
+	}
+	if updatedSettings.OAuthSettings.SSO != false {
+		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
+	}
+	if updatedSettings.OAuthSettings.LogoutURI != "" {
+		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
+	}
+}

api/bolt/migrator/migrate_test_helper.go

@@ -0,0 +1,38 @@
+package migrator
+
+import (
+	"path"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+// initTestingDBConn creates a raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
+	databasePath := path.Join(storePath, fileName)
+	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+	return dbConn, nil
+}
+
+// initTestingDBConn creates a settings service with raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
+	internalDBConn := &internal.DbConnection{
+		DB: dbConn,
+	}
+	settingsService, err := settings.NewService(internalDBConn)
+	if err != nil {
+		return nil, err
+	}
+	//insert a obj
+	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
+		return nil, err
+	}
+	return settingsService, nil
+}

api/bolt/migrator/migrator.go

@@ -358,5 +358,13 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// Portainer 2.5.0
+	if m.currentDBVersion < 31 {
+		err := m.updateSettingsToDB31()
+		if err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/registry/registry.go

@@ -1,7 +1,7 @@
 package registry
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 	var registry portainer.Registry
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &registry)
+	err := internal.GetObject(service.connection, BucketName, identifier, &registry)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 func (service *Service) Registries() ([]portainer.Registry, error) {
 	var registries = make([]portainer.Registry, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 
 // CreateRegistry creates a new registry.
 func (service *Service) CreateRegistry(registry *portainer.Registry) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateRegistry(registry *portainer.Registry) error {
 // UpdateRegistry updates an registry.
 func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, registry)
+	return internal.UpdateObject(service.connection, BucketName, identifier, registry)
 }
 
 // DeleteRegistry deletes an registry.
 func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,7 +1,7 @@
 package resourcecontrol
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 	var resourceControl portainer.ResourceControl
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &resourceControl)
+	err := internal.GetObject(service.connection, BucketName, identifier, &resourceControl)
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +48,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -82,7 +82,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -121,11 +121,11 @@ func (service *Service) CreateResourceControl(resourceControl *portainer.Resourc
 // UpdateResourceControl saves a ResourceControl object.
 func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, resourceControl)
+	return internal.UpdateObject(service.connection, BucketName, identifier, resourceControl)
 }
 
 // DeleteResourceControl deletes a ResourceControl object by ID
 func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/role/role.go

@@ -1,7 +1,7 @@
 package role
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &set)
+	err := internal.GetObject(service.connection, BucketName, identifier, &set)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 func (service *Service) Roles() ([]portainer.Role, error) {
 	var sets = make([]portainer.Role, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 
 // CreateRole creates a new Role.
 func (service *Service) CreateRole(role *portainer.Role) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,5 +85,5 @@ func (service *Service) CreateRole(role *portainer.Role) error {
 // UpdateRole updates a role.
 func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, role)
+	return internal.UpdateObject(service.connection, BucketName, identifier, role)
 }

api/bolt/schedule/schedule.go

@@ -1,7 +1,7 @@
 package schedule
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing schedule data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 	var schedule portainer.Schedule
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
+	err := internal.GetObject(service.connection, BucketName, identifier, &schedule)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 // UpdateSchedule updates a schedule.
 func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
+	return internal.UpdateObject(service.connection, BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
 func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -82,7 +82,7 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -105,7 +105,7 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 
 // CreateSchedule assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for schedules
@@ -125,5 +125,5 @@ func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
 
 // GetNextIdentifier returns the next identifier for a schedule.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/services.go

@@ -0,0 +1,263 @@
+package bolt
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/customtemplate"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
+	"github.com/portainer/portainer/api/bolt/edgestack"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
+)
+
+func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
+	customTemplateService, err := customtemplate.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.CustomTemplateService = customTemplateService
+
+	dockerhubService, err := dockerhub.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	edgeStackService, err := edgestack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackService = edgeStackService
+
+	edgeGroupService, err := edgegroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeGroupService = edgeGroupService
+
+	edgeJobService, err := edgejob.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
+	endpointgroupService, err := endpointgroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	endpointRelationService, err := endpointrelation.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	extensionService, err := extension.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
+	registryService, err := registry.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	settingsService, err := settings.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	stackService, err := stack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	tunnelServerService, err := tunnelserver.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
+	userService, err := user.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	webhookService, err := webhook.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
+	return nil
+}
+
+// CustomTemplate gives access to the CustomTemplate data management layer
+func (store *Store) CustomTemplate() portainer.CustomTemplateService {
+	return store.CustomTemplateService
+}
+
+// DockerHub gives access to the DockerHub data management layer
+func (store *Store) DockerHub() portainer.DockerHubService {
+	return store.DockerHubService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Endpoint gives access to the Endpoint data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
+}

api/bolt/settings/settings.go

@@ -1,10 +1,8 @@
 package settings
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings
 
-	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &settings)
+	err := internal.GetObject(service.connection, BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), settings)
+	return internal.UpdateObject(service.connection, BucketName, []byte(settingsKey), settings)
 }

api/bolt/stack/stack.go

@@ -1,7 +1,7 @@
 package stack
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -35,7 +35,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 	var stack portainer.Stack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &stack)
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -47,7 +47,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 	var stack *portainer.Stack
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -78,7 +78,7 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) Stacks() ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -99,12 +99,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) CreateStack(stack *portainer.Stack) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for stacks
@@ -125,11 +125,11 @@ func (service *Service) CreateStack(stack *portainer.Stack) error {
 // UpdateStack updates a stack.
 func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, stack)
+	return internal.UpdateObject(service.connection, BucketName, identifier, stack)
 }
 
 // DeleteStack deletes a stack.
 func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/tag/tag.go

@@ -1,7 +1,7 @@
 package tag
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,7 +57,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 	var tag portainer.Tag
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &tag)
+	err := internal.GetObject(service.connection, BucketName, identifier, &tag)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +67,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 
 // CreateTag creates a new tag.
 func (service *Service) CreateTag(tag *portainer.Tag) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateTag(tag *portainer.Tag) error {
 // UpdateTag updates a tag.
 func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, tag)
+	return internal.UpdateObject(service.connection, BucketName, identifier, tag)
 }
 
 // DeleteTag deletes a tag.
 func (service *Service) DeleteTag(ID portainer.TagID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/team/team.go

@@ -17,18 +17,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -37,7 +37,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 	var team portainer.Team
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &team)
+	err := internal.GetObject(service.connection, BucketName, identifier, &team)
 	if err != nil {
 		return nil, err
 	}
@@ -49,7 +49,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	var team *portainer.Team
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -80,7 +80,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 func (service *Service) Teams() ([]portainer.Team, error) {
 	var teams = make([]portainer.Team, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -102,12 +102,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 // UpdateTeam saves a Team.
 func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, team)
+	return internal.UpdateObject(service.connection, BucketName, identifier, team)
 }
 
 // CreateTeam creates a new Team.
 func (service *Service) CreateTeam(team *portainer.Team) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -125,5 +125,5 @@ func (service *Service) CreateTeam(team *portainer.Team) error {
 // DeleteTeam deletes a Team.
 func (service *Service) DeleteTeam(ID portainer.TeamID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/teammembership/teammembership.go

@@ -1,7 +1,7 @@
 package teammembership
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 	var membership portainer.TeamMembership
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &membership)
+	err := internal.GetObject(service.connection, BucketName, identifier, &membership)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -69,7 +69,7 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -95,7 +95,7 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -120,12 +120,12 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 // UpdateTeamMembership saves a TeamMembership object.
 func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, membership)
+	return internal.UpdateObject(service.connection, BucketName, identifier, membership)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) CreateTeamMembership(membership *portainer.TeamMembership) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -143,12 +143,12 @@ func (service *Service) CreateTeamMembership(membership *portainer.TeamMembershi
 // DeleteTeamMembership deletes a TeamMembership object.
 func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -173,7 +173,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()

api/bolt/tunnelserver/tunnelserver.go

@@ -1,10 +1,8 @@
 package tunnelserver
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 	var info portainer.TunnelServerInfo
 
-	err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
+	err := internal.GetObject(service.connection, BucketName, []byte(infoKey), &info)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 
 // UpdateInfo persists a TunnelServerInfo object.
 func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
+	return internal.UpdateObject(service.connection, BucketName, []byte(infoKey), settings)
 }

api/bolt/user/user.go

@@ -17,18 +17,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -37,7 +37,7 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 	var user portainer.User
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &user)
+	err := internal.GetObject(service.connection, BucketName, identifier, &user)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +51,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 
 	username = strings.ToLower(username)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -81,7 +81,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 func (service *Service) Users() ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) Users() ([]portainer.User, error) {
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -128,12 +128,12 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
 	identifier := internal.Itob(int(ID))
 	user.Username = strings.ToLower(user.Username)
-	return internal.UpdateObject(service.db, BucketName, identifier, user)
+	return internal.UpdateObject(service.connection, BucketName, identifier, user)
 }
 
 // CreateUser creates a new user.
 func (service *Service) CreateUser(user *portainer.User) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -152,5 +152,5 @@ func (service *Service) CreateUser(user *portainer.User) error {
 // DeleteUser deletes a user.
 func (service *Service) DeleteUser(ID portainer.UserID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/version/version.go

@@ -19,18 +19,18 @@ const (
 
 // Service represents a service to manage stored versions.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -38,7 +38,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) DBVersion() (int, error) {
 	var data []byte
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(versionKey))
@@ -75,7 +75,7 @@ func (service *Service) Edition() (portainer.SoftwareEdition, error) {
 
 // StoreDBVersion store the database version.
 func (service *Service) StoreDBVersion(version int) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(strconv.Itoa(version))
@@ -87,7 +87,7 @@ func (service *Service) StoreDBVersion(version int) error {
 func (service *Service) InstanceID() (string, error) {
 	var data []byte
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(instanceKey))
@@ -109,7 +109,7 @@ func (service *Service) InstanceID() (string, error) {
 
 // StoreInstanceID store the instance ID.
 func (service *Service) StoreInstanceID(ID string) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(ID)
@@ -120,7 +120,7 @@ func (service *Service) StoreInstanceID(ID string) error {
 func (service *Service) getKey(key string) ([]byte, error) {
 	var data []byte
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(key))
@@ -142,7 +142,7 @@ func (service *Service) getKey(key string) ([]byte, error) {
 }
 
 func (service *Service) setKey(key string, value string) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(value)

api/bolt/webhook/webhook.go

@@ -1,7 +1,7 @@
 package webhook
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -58,7 +58,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 	var webhook portainer.Webhook
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
+	err := internal.GetObject(service.connection, BucketName, identifier, &webhook)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +70,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -101,7 +101,7 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -131,12 +131,12 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 // DeleteWebhook deletes a webhook.
 func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/chisel/service.go

@@ -1,6 +1,7 @@
 package chisel
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"strconv"
@@ -9,7 +10,7 @@ import (
 	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
 	cmap "github.com/orcaman/concurrent-map"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
@@ -29,13 +30,15 @@ type Service struct {
 	dataStore         portainer.DataStore
 	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
+	shutdownCtx       context.Context
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore portainer.DataStore) *Service {
+func NewService(dataStore portainer.DataStore, shutdownCtx context.Context) *Service {
 	return &Service{
 		tunnelDetailsMap: cmap.New(),
 		dataStore:        dataStore,
+		shutdownCtx:      shutdownCtx,
 	}
 }
 
@@ -83,6 +86,11 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	return nil
 }
 
+// StopTunnelServer stops tunnel http server
+func (service *Service) StopTunnelServer() error {
+	return service.chiselServer.Close()
+}
+
 func (service *Service) retrievePrivateKeySeed() (string, error) {
 	var serverInfo *portainer.TunnelServerInfo
 
@@ -108,13 +116,16 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 func (service *Service) startTunnelVerificationLoop() {
 	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
 	ticker := time.NewTicker(tunnelCleanupInterval)
-	stopSignal := make(chan struct{})
 
 	for {
 		select {
 		case <-ticker.C:
 			service.checkTunnels()
-		case <-stopSignal:
+		case <-service.shutdownCtx.Done():
+			log.Println("[DEBUG] Shutting down tunnel service")
+			if err := service.StopTunnelServer(); err != nil {
+				log.Printf("Stopped tunnel service: %s", err)
+			}
 			ticker.Stop()
 			return
 		}

api/cmd/portainer/main.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"context"
 	"log"
 	"os"
 	"strings"
-	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
@@ -20,6 +20,7 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
@@ -67,7 +68,7 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 		log.Fatalf("failed initializing data store: %v", err)
 	}
 
-	err = store.MigrateData()
+	err = store.MigrateData(false)
 	if err != nil {
 		log.Fatalf("failed migration: %v", err)
 	}
@@ -75,7 +76,7 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 }
 
 func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper := exec.NewComposeWrapper(assetsPath, proxyManager)
+	composeWrapper := exec.NewComposeWrapper(assetsPath, dataStorePath, proxyManager)
 	if composeWrapper != nil {
 		return composeWrapper
 	}
@@ -87,8 +88,8 @@ func initSwarmStackManager(assetsPath string, dataStorePath string, signatureSer
 	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(assetsPath)
+func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath)
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -136,11 +137,11 @@ func initKubernetesClientFactory(signatureService portainer.DigitalSignatureServ
 	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
 }
 
-func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory) (portainer.SnapshotService, error) {
+func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter)
+	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -148,21 +149,6 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
-func loadEdgeJobsFromDatabase(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService) error {
-	edgeJobs, err := dataStore.EdgeJob().EdgeJobs()
-	if err != nil {
-		return err
-	}
-
-	for _, edgeJob := range edgeJobs {
-		for endpointID := range edgeJob.Endpoints {
-			reverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
-		}
-	}
-
-	return nil
-}
-
 func initStatus(flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
 		Version: portainer.APIVersion,
@@ -179,6 +165,7 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 	settings.SnapshotInterval = *flags.SnapshotInterval
 	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
 	settings.EnableTelemetry = true
+	settings.OAuthSettings.SSO = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -254,6 +241,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -315,6 +303,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -353,28 +342,12 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
-func terminateIfNoAdminCreated(dataStore portainer.DataStore) {
-	timer1 := time.NewTimer(5 * time.Minute)
-	<-timer1.C
-
-	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
-	if err != nil {
-		log.Fatalf("failed getting admin user: %v", err)
-	}
-
-	if len(users) == 0 {
-		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
-		return
-	}
-}
-
-func main() {
-	flags := initCLI()
+func buildServer(flags *portainer.CLIFlags) portainer.Server {
+	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
 
 	fileService := initFileService(*flags.Data)
 
 	dataStore := initDataStore(*flags.Data, fileService)
-	defer dataStore.Close()
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -400,7 +373,7 @@ func main() {
 		log.Fatalf("failed initializing key pai: %v", err)
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
 
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
@@ -410,7 +383,7 @@ func main() {
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
 
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory)
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
 		log.Fatalf("failed initializing snapshot service: %v", err)
 	}
@@ -425,7 +398,7 @@ func main() {
 
 	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
 
-	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
@@ -434,7 +407,7 @@ func main() {
 		}
 	}
 
-	err = loadEdgeJobsFromDatabase(dataStore, reverseTunnelService)
+	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
 	if err != nil {
 		log.Fatalf("failed loading edge jobs from database: %v", err)
 	}
@@ -482,14 +455,12 @@ func main() {
 		}
 	}
 
-	go terminateIfNoAdminCreated(dataStore)
-
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting tunnel server: %v", err)
+		log.Fatalf("failed starting license service: %s", err)
 	}
 
-	var server portainer.Server = &http.Server{
+	return &http.Server{
 		ReverseTunnelService:        reverseTunnelService,
 		Status:                      applicationStatus,
 		BindAddress:                 *flags.Addr,
@@ -513,11 +484,18 @@ func main() {
 		SSLKey:                      *flags.SSLKey,
 		DockerClientFactory:         dockerClientFactory,
 		KubernetesClientFactory:     kubernetesClientFactory,
-	}
-
-	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)
-	err = server.Start()
-	if err != nil {
-		log.Fatalf("failed starting server: %v", err)
+		ShutdownCtx:                 shutdownCtx,
+		ShutdownTrigger:             shutdownTrigger,
+	}
+}
+
+func main() {
+	flags := initCLI()
+
+	for {
+		server := buildServer(flags)
+		log.Printf("Starting Portainer %s on %s\n", portainer.APIVersion, *flags.Addr)
+		err := server.Start()
+		log.Printf("Http server exited: %s\n", err)
 	}
 }

api/crypto/aes.go

@@ -0,0 +1,70 @@
+package crypto
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"io"
+
+	"golang.org/x/crypto/scrypt"
+)
+
+// NOTE: has to go with what is considered to be a simplistic in that it omits any
+// authentication of the encrypted data.
+// Person with better knowledge is welcomed to improve it.
+// sourced from https://golang.org/src/crypto/cipher/example_test.go
+
+var emptySalt []byte = make([]byte, 0, 0)
+
+// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
+// passphrase is used to generate an encryption key.
+func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
+	// making a 32 bytes key that would correspond to AES-256
+	// don't necessarily need a salt, so just kept in empty
+	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	if err != nil {
+		return err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	// If the key is unique for each ciphertext, then it's ok to use a zero
+	// IV.
+	var iv [aes.BlockSize]byte
+	stream := cipher.NewOFB(block, iv[:])
+
+	writer := &cipher.StreamWriter{S: stream, W: output}
+	// Copy the input to the output, encrypting as we go.
+	if _, err := io.Copy(writer, input); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// making a 32 bytes key that would correspond to AES-256
+	// don't necessarily need a salt, so just kept in empty
+	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the key is unique for each ciphertext, then it's ok to use a zero
+	// IV.
+	var iv [aes.BlockSize]byte
+	stream := cipher.NewOFB(block, iv[:])
+
+	reader := &cipher.StreamReader{S: stream, R: input}
+
+	return reader, nil
+}

api/crypto/aes_test.go

@@ -0,0 +1,132 @@
+package crypto
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(""))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(""))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
+	tmpdir, _ := ioutils.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+}

api/exec/compose_wrapper.go

@@ -16,17 +16,19 @@ import (
 // ComposeWrapper is a wrapper for docker-compose binary
 type ComposeWrapper struct {
 	binaryPath   string
+	dataPath     string
 	proxyManager *proxy.Manager
 }
 
 // NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeWrapper(binaryPath string, proxyManager *proxy.Manager) *ComposeWrapper {
+func NewComposeWrapper(binaryPath, dataPath string, proxyManager *proxy.Manager) *ComposeWrapper {
 	if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
 		return nil
 	}
 
 	return &ComposeWrapper{
 		binaryPath:   binaryPath,
+		dataPath:     dataPath,
 		proxyManager: proxyManager,
 	}
 }
@@ -36,6 +38,11 @@ func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
 	return portainer.ComposeSyntaxMaxVersion
 }
 
+// NormalizeStackName returns a new stack name with unsupported characters replaced
+func (w *ComposeWrapper) NormalizeStackName(name string) string {
+	return name
+}
+
 // Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
 func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	_, err := w.command([]string{"up", "-d"}, stack, endpoint)
@@ -79,6 +86,8 @@ func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpo
 
 	var stderr bytes.Buffer
 	cmd := exec.Command(program, args...)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_CONFIG=%s", w.dataPath))
 	cmd.Stderr = &stderr
 
 	out, err := cmd.Output()

api/exec/compose_wrapper_integration_test.go

@@ -42,7 +42,7 @@ func Test_UpAndDown(t *testing.T) {
 
 	stack, endpoint := setup(t)
 
-	w := NewComposeWrapper("", nil)
+	w := NewComposeWrapper("", "", nil)
 
 	err := w.Up(stack, endpoint)
 	if err != nil {

api/exec/kubernetes_deploy.go

@@ -2,71 +2,188 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath string
+	binaryPath           string
+	dataStore            portainer.DataStore
+	reverseTunnelService portainer.ReverseTunnelService
+	signatureService     portainer.DigitalSignatureService
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath: binaryPath,
+		binaryPath:           binaryPath,
+		dataStore:            datastore,
+		reverseTunnelService: reverseTunnelService,
+		signatureService:     signatureService,
 	}
 }
 
 // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
-// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
 // Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
-	if composeFormat {
-		convertedData, err := deployer.convertComposeData(data)
+func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
+	if endpoint.Type == portainer.KubernetesLocalEnvironment {
+		token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		data = string(convertedData)
+
+		command := path.Join(deployer.binaryPath, "kubectl")
+		if runtime.GOOS == "windows" {
+			command = path.Join(deployer.binaryPath, "kubectl.exe")
+		}
+
+		args := make([]string, 0)
+		args = append(args, "--server", endpoint.URL)
+		args = append(args, "--insecure-skip-tls-verify")
+		args = append(args, "--token", string(token))
+		args = append(args, "--namespace", namespace)
+		args = append(args, "apply", "-f", "-")
+
+		var stderr bytes.Buffer
+		cmd := exec.Command(command, args...)
+		cmd.Stderr = &stderr
+		cmd.Stdin = strings.NewReader(stackConfig)
+
+		output, err := cmd.Output()
+		if err != nil {
+			return "", errors.New(stderr.String())
+		}
+
+		return string(output), nil
 	}
 
-	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+	// agent
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		if tunnel.Status == portainer.EdgeAgentIdle {
+
+			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+			if err != nil {
+				return "", err
+			}
+
+			settings, err := deployer.dataStore.Settings().Settings()
+			if err != nil {
+				return "", err
+			}
+
+			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
+			time.Sleep(waitForAgentToConnect * 2)
+		}
+
+		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+	}
+
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return "", err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	httpCli := &http.Client{
+		Transport: transport,
+	}
+
+	if !strings.HasPrefix(endpointURL, "http") {
+		endpointURL = fmt.Sprintf("https://%s", endpointURL)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	command := path.Join(deployer.binaryPath, "kubectl")
-	if runtime.GOOS == "windows" {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
-	}
-
-	args := make([]string, 0)
-	args = append(args, "--server", endpoint.URL)
-	args = append(args, "--insecure-skip-tls-verify")
-	args = append(args, "--token", string(token))
-	args = append(args, "--namespace", namespace)
-	args = append(args, "apply", "-f", "-")
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = &stderr
-	cmd.Stdin = strings.NewReader(data)
-
-	output, err := cmd.Output()
+	reqPayload, err := json.Marshal(
+		struct {
+			StackConfig string
+			Namespace   string
+		}{
+			StackConfig: stackConfig,
+			Namespace:   namespace,
+		})
 	if err != nil {
-		return nil, errors.New(stderr.String())
+		return "", err
 	}
 
-	return output, nil
+	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
+	if err != nil {
+		return "", err
+	}
+
+	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
+	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var errorResponseData struct {
+			Message string
+			Details string
+		}
+		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
+		if err != nil {
+			output, parseStringErr := ioutil.ReadAll(resp.Body)
+			if parseStringErr != nil {
+				return "", parseStringErr
+			}
+
+			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
+
+		}
+
+		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
+	}
+
+	var responseData struct{ Output string }
+	err = json.NewDecoder(resp.Body).Decode(&responseData)
+	if err != nil {
+		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
+		if parseStringErr != nil {
+			return "", parseStringErr
+		}
+
+		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
+	}
+
+	return responseData.Output, nil
+
 }
 
-func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
+// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
+func (deployer *KubernetesDeployer) ConvertCompose(data string) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")

api/filesystem/filesystem.go

@@ -9,7 +9,7 @@ import (
 	"io/ioutil"
 
 	"github.com/gofrs/uuid"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 
 	"io"
 	"os"
@@ -505,3 +505,8 @@ func (service *Service) GetTemporaryPath() (string, error) {
 
 	return path.Join(service.fileStorePath, TempPath, uid.String()), nil
 }
+
+// GetDataStorePath returns path to data folder
+func (service *Service) GetDatastorePath() string {
+	return service.dataStorePath
+}

api/git/azure.go

@@ -0,0 +1,219 @@
+package git
+
+import (
+	"context"
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+const (
+	azureDevOpsHost        = "dev.azure.com"
+	visualStudioHostSuffix = ".visualstudio.com"
+)
+
+func isAzureUrl(s string) bool {
+	return strings.Contains(s, azureDevOpsHost) ||
+		strings.Contains(s, visualStudioHostSuffix)
+}
+
+type azureOptions struct {
+	organisation, project, repository string
+	// a user may pass credentials in a repository URL,
+	// for example https://<username>:<password>@<domain>/<path>
+	username, password string
+}
+
+type azureDownloader struct {
+	client  *http.Client
+	baseUrl string
+}
+
+func NewAzureDownloader(client *http.Client) *azureDownloader {
+	return &azureDownloader{
+		client: client,
+		baseUrl: "https://dev.azure.com",
+	}
+}
+
+func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
+	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
+	if err != nil {
+		return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
+	}
+	defer os.Remove(zipFilepath)
+
+	err = archive.UnzipFile(zipFilepath, destination)
+	if err != nil {
+		return errors.Wrap(err, "failed to unzip file")
+	}
+
+	return nil
+}
+
+func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
+	config, err := parseUrl(options.repositoryUrl)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to parse url")
+	}
+	downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to build download url")
+	}
+	zipFile, err := ioutil.TempFile("", "azure-git-repo-*.zip")
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create temp file")
+	}
+	defer zipFile.Close()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
+	if options.username != "" || options.password != "" {
+		req.SetBasicAuth(options.username, options.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	res, err := a.client.Do(req)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to download zip with a status \"%v\"", res.Status)
+	}
+
+	_, err = io.Copy(zipFile, res.Body)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to save HTTP response to a file")
+	}
+	return zipFile.Name(), nil
+}
+
+func parseUrl(rawUrl string) (*azureOptions, error) {
+	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
+		return parseHttpUrl(rawUrl)
+	}
+	if strings.HasPrefix(rawUrl, "git@ssh") {
+		return parseSshUrl(rawUrl)
+	}
+	if strings.HasPrefix(rawUrl, "ssh://") {
+		r := []rune(rawUrl)
+		return parseSshUrl(string(r[6:])) // remove the prefix
+	}
+
+	return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl)
+}
+
+var expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
+
+func parseSshUrl(rawUrl string) (*azureOptions, error) {
+	path := strings.Split(rawUrl, "/")
+
+	unexpectedUrlErr := errors.Errorf("want url %s, got %s", expectedSshUrl, rawUrl)
+	if len(path) != 4 {
+		return nil, unexpectedUrlErr
+	}
+	return &azureOptions{
+		organisation: path[1],
+		project:      path[2],
+		repository:   path[3],
+	}, nil
+}
+
+const expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository"
+const expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository"
+
+func parseHttpUrl(rawUrl string) (*azureOptions, error) {
+	u, err := url.Parse(rawUrl)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse HTTP url")
+	}
+
+	opt := azureOptions{}
+	switch {
+	case u.Host == azureDevOpsHost:
+		path := strings.Split(u.Path, "/")
+		if len(path) != 5 {
+			return nil, errors.Errorf("want url %s, got %s", expectedAzureDevOpsHttpUrl, u)
+		}
+		opt.organisation = path[1]
+		opt.project = path[2]
+		opt.repository = path[4]
+	case strings.HasSuffix(u.Host, visualStudioHostSuffix):
+		path := strings.Split(u.Path, "/")
+		if len(path) != 4 {
+			return nil, errors.Errorf("want url %s, got %s", expectedVisualStudioHttpUrl, u)
+		}
+		opt.organisation = strings.TrimSuffix(u.Host, visualStudioHostSuffix)
+		opt.project = path[1]
+		opt.repository = path[3]
+	default:
+		return nil, errors.Errorf("unknown azure host in url \"%s\"", rawUrl)
+	}
+
+	opt.username = u.User.Username()
+	opt.password, _ = u.User.Password()
+
+	return &opt, nil
+}
+
+func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl)
+	}
+	q := u.Query()
+	// scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0
+	q.Set("scopePath", "/")
+	q.Set("download", "true")
+	q.Set("versionDescriptor.versionType", getVersionType(referenceName))
+	q.Set("versionDescriptor.version", formatReferenceName(referenceName))
+	q.Set("$format", "zip")
+	q.Set("recursionLevel", "full")
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
+const (
+	branchPrefix = "refs/heads/"
+	tagPrefix    = "refs/tags/"
+)
+
+func formatReferenceName(name string) string {
+	if strings.HasPrefix(name, branchPrefix) {
+		return strings.TrimPrefix(name, branchPrefix)
+	}
+	if strings.HasPrefix(name, tagPrefix) {
+		return strings.TrimPrefix(name, tagPrefix)
+	}
+	return name
+}
+
+func getVersionType(name string) string {
+	if strings.HasPrefix(name, branchPrefix) {
+		return "branch"
+	}
+	if strings.HasPrefix(name, tagPrefix) {
+		return "tag"
+	}
+	return "commit"
+}

api/git/azure_integration_test.go

@@ -0,0 +1,92 @@
+package git
+
+import (
+	"fmt"
+	"github.com/docker/docker/pkg/ioutils"
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestService_ClonePublicRepository_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	type args struct {
+		repositoryURLFormat string
+		referenceName       string
+		username            string
+		password            string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Clone Azure DevOps repo branch",
+			args: args{
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				referenceName:       "refs/heads/main",
+				username:            "",
+				password:            pat,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Clone Azure DevOps repo tag",
+			args: args{
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				referenceName:       "refs/tags/v1.1",
+				username:            "",
+				password:            pat,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dst, err := ioutils.TempDir("", "clone")
+			assert.NoError(t, err)
+			defer os.RemoveAll(dst)
+			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
+			err = service.ClonePublicRepository(repositoryUrl, tt.args.referenceName, dst)
+			assert.NoError(t, err)
+			assert.FileExists(t, filepath.Join(dst, "README.md"))
+		})
+	}
+}
+
+func TestService_ClonePrivateRepository_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	dst, err := ioutils.TempDir("", "clone")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dst)
+
+	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, "", pat)
+	assert.NoError(t, err)
+	assert.FileExists(t, filepath.Join(dst, "README.md"))
+}
+
+func getRequiredValue(t *testing.T, name string) string {
+	value, ok := os.LookupEnv(name)
+	if !ok {
+		t.Fatalf("can't find required env var \"%s\"", name)
+	}
+	return value
+}
+
+func ensureIntegrationTest(t *testing.T) {
+	if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
+		t.Skip("skip an integration test")
+	}
+}

api/git/azure_test.go

@@ -0,0 +1,250 @@
+package git
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func Test_buildDownloadUrl(t *testing.T) {
+	a := NewAzureDownloader(nil)
+	u, err := a.buildDownloadUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "refs/heads/main")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0&versionDescriptor.versionType=branch")
+	actualUrl, _ := url.Parse(u)
+	if assert.NoError(t, err) {
+		assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+		assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+		assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+		assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+	}
+}
+
+func Test_parseAzureUrl(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *azureOptions
+		wantErr bool
+	}{
+		{
+			name: "Expected SSH URL format starting with ssh://",
+			args: args{
+				url: "ssh://git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Expected SSH URL format starting with git@ssh",
+			args: args{
+				url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Unexpected SSH URL format",
+			args: args{
+				url: "git@ssh.dev.azure.com:v3/Organisation/Repository",
+			},
+			wantErr: true,
+		},
+		{
+			name: "Expected HTTPS URL format",
+			args: args{
+				url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				username:     "Organisation",
+			},
+			wantErr: false,
+		},
+		{
+			name: "HTTPS URL with credentials",
+			args: args{
+				url: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				username:     "username",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "HTTPS URL with password",
+			args: args{
+				url: "https://:password@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Visual Studio HTTPS URL with credentials",
+			args: args{
+				url: "https://username:password@organisation.visualstudio.com/project/_git/repository",
+			},
+			want: &azureOptions{
+				organisation: "organisation",
+				project:      "project",
+				repository:   "repository",
+				username:     "username",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Unexpected HTTPS URL format",
+			args: args{
+				url: "https://Organisation@dev.azure.com/Project/_git/Repository",
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUrl(tt.args.url)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseUrl() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func Test_isAzureUrl(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Is Azure url",
+			args: args{
+				s: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: true,
+		},
+		{
+			name: "Is Azure url",
+			args: args{
+				s: "https://portainer.visualstudio.com/project/_git/repository",
+			},
+			want: true,
+		},
+		{
+			name: "Is NOT Azure url",
+			args: args{
+				s: "https://github.com/Organisation/Repository",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, isAzureUrl(tt.args.s))
+		})
+	}
+}
+
+func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
+	type args struct {
+		options cloneOptions
+	}
+	type basicAuth struct {
+		username, password string
+	}
+	tests := []struct {
+		name string
+		args args
+		want *basicAuth
+	}{
+		{
+			name: "username, password embedded",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+				},
+			},
+			want: &basicAuth{
+				username: "username",
+				password: "password",
+			},
+		},
+		{
+			name: "username, password embedded, clone options take precedence",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+					username:      "u",
+					password:      "p",
+				},
+			},
+			want: &basicAuth{
+				username: "u",
+				password: "p",
+			},
+		},
+		{
+			name: "no credentials",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var zipRequestAuth *basicAuth
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if username, password, ok := r.BasicAuth(); ok {
+					zipRequestAuth = &basicAuth{username, password}
+				}
+				w.WriteHeader(http.StatusNotFound) // this makes function under test to return an error
+			}))
+			defer server.Close()
+
+			a := &azureDownloader{
+				client:  server.Client(),
+				baseUrl: server.URL,
+			}
+			_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
+			assert.Error(t, err)
+			assert.Equal(t, tt.want, zipRequestAuth)
+		})
+	}
+}

api/git/git.go

@@ -1,21 +1,71 @@
 package git
 
 import (
+	"context"
 	"crypto/tls"
+	"github.com/pkg/errors"
 	"net/http"
-	"net/url"
-	"strings"
+	"os"
+	"path/filepath"
 	"time"
 
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
-	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
+type cloneOptions struct {
+	repositoryUrl string
+	username      string
+	password      string
+	referenceName string
+	depth         int
+}
+
+type downloader interface {
+	download(ctx context.Context, dst string, opt cloneOptions) error
+}
+
+type gitClient struct{
+	preserveGitDirectory bool
+}
+
+func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
+	gitOptions := git.CloneOptions{
+		URL:   opt.repositoryUrl,
+		Depth: opt.depth,
+	}
+
+	if opt.password != "" || opt.username != "" {
+		gitOptions.Auth = &githttp.BasicAuth{
+			Username: opt.username,
+			Password: opt.password,
+		}
+	}
+
+	if opt.referenceName != "" {
+		gitOptions.ReferenceName = plumbing.ReferenceName(opt.referenceName)
+	}
+
+	_, err := git.PlainCloneContext(ctx, dst, false, &gitOptions)
+
+	if err != nil {
+		return errors.Wrap(err, "failed to clone git repository")
+	}
+
+	if !c.preserveGitDirectory {
+		os.RemoveAll(filepath.Join(dst, ".git"))
+	}
+
+	return nil
+}
+
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
+	azure    downloader
+	git      downloader
 }
 
 // NewService initializes a new service.
@@ -31,32 +81,37 @@ func NewService() *Service {
 
 	return &Service{
 		httpsCli: httpsCli,
+		azure:    NewAzureDownloader(httpsCli),
+		git:      gitClient{},
 	}
 }
 
 // ClonePublicRepository clones a public git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
-	return cloneRepository(repositoryURL, referenceName, destination)
+func (service *Service) ClonePublicRepository(repositoryURL, referenceName, destination string) error {
+	return service.cloneRepository(destination, cloneOptions{
+		repositoryUrl: repositoryURL,
+		referenceName: referenceName,
+		depth:         1,
+	})
 }
 
 // ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
-// destination folder. It will use the specified username and password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
-	credentials := username + ":" + url.PathEscape(password)
-	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
-	return cloneRepository(repositoryURL, referenceName, destination)
+// destination folder. It will use the specified Username and Password for basic HTTP authentication.
+func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName, destination, username, password string) error {
+	return service.cloneRepository(destination, cloneOptions{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+		referenceName: referenceName,
+		depth:         1,
+	})
 }
 
-func cloneRepository(repositoryURL, referenceName, destination string) error {
-	options := &git.CloneOptions{
-		URL: repositoryURL,
+func (service *Service) cloneRepository(destination string, options cloneOptions) error {
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.download(context.TODO(), destination, options)
 	}
 
-	if referenceName != "" {
-		options.ReferenceName = plumbing.ReferenceName(referenceName)
-	}
-
-	_, err := git.PlainClone(destination, false, options)
-	return err
+	return service.git.download(context.TODO(), destination, options)
 }

api/git/git_integration_test.go

@@ -0,0 +1,26 @@
+package git
+
+import (
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService()
+
+	dst, err := ioutils.TempDir("", "clone")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dst)
+
+	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, username, pat)
+	assert.NoError(t, err)
+	assert.FileExists(t, filepath.Join(dst, "README.md"))
+}

api/git/git_test.go

@@ -0,0 +1,173 @@
+package git
+
+import (
+	"context"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var bareRepoDir string
+
+func TestMain(m *testing.M) {
+	if err := testMain(m); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// testMain does extra setup/teardown before/after testing.
+// The function is separated from TestMain due to necessity to call os.Exit/log.Fatal in the latter.
+func testMain(m *testing.M) error {
+	dir, err := ioutil.TempDir("", "git-repo-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+
+	bareRepoDir = filepath.Join(dir, "test-clone.git")
+
+	file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0755)
+	if err != nil {
+		return errors.Wrap(err, "failed to open an archive")
+	}
+	err = archive.ExtractTarGz(file, dir)
+	if err != nil {
+		return errors.Wrapf(err, "failed to extract file from the archive to a folder %s\n", dir)
+	}
+
+	m.Run()
+
+	return nil
+}
+
+func Test_ClonePublicRepository_Shallow(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+	t.Logf("Cloning into %s", dir)
+	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+	t.Logf("Cloning into %s", dir)
+	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
+	assert.NoError(t, err)
+	assert.NoDirExists(t, filepath.Join(dir, ".git"))
+}
+
+func Test_cloneRepository(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+	t.Logf("Cloning into %s", dir)
+
+	err = service.cloneRepository(dir, cloneOptions{
+		repositoryUrl: repositoryURL,
+		referenceName: referenceName,
+		depth:         10,
+	})
+
+	assert.NoError(t, err)
+	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func getCommitHistoryLength(t *testing.T, err error, dir string) int {
+	repo, err := git.PlainOpen(dir)
+	if err != nil {
+		t.Fatalf("can't open a git repo at %s with error %v", dir, err)
+	}
+	iter, err := repo.Log(&git.LogOptions{All: true})
+	if err != nil {
+		t.Fatalf("can't get a commit history iterator with error %v", err)
+	}
+	count := 0
+	err = iter.ForEach(func(_ *object.Commit) error {
+		count++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("can't iterate over the commit history with error %v", err)
+	}
+	return count
+}
+
+type testDownloader struct {
+	called bool
+}
+
+func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
+	t.called = true
+	return nil
+}
+
+func Test_cloneRepository_azure(t *testing.T) {
+	tests := []struct {
+		name   string
+		url    string
+		called bool
+	}{
+		{
+			name:   "Azure HTTP URL",
+			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			called: true,
+		},
+		{
+			name:   "Azure SSH URL",
+			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			called: true,
+		},
+		{
+			name:   "Something else",
+			url:    "https://example.com",
+			called: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azure := &testDownloader{}
+			git := &testDownloader{}
+
+			s := &Service{azure: azure, git: git}
+			s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
+
+			// if azure API is called, git isn't and vice versa
+			assert.Equal(t, tt.called, azure.called)
+			assert.Equal(t, tt.called, !git.called)
+		})
+	}
+}

api/go.mod

@@ -3,7 +3,7 @@ module github.com/portainer/portainer/api
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.14
+	github.com/Microsoft/go-winio v0.4.16
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/boltdb/bolt v1.3.1
 	github.com/containerd/containerd v1.3.1 // indirect
@@ -13,30 +13,32 @@ require (
 	github.com/docker/cli v0.0.0-20191126203649-54d085b857e9
 	github.com/docker/docker v0.0.0-00010101000000-000000000000
 	github.com/g07cha/defender v0.0.0-20180505193036-5665c627c814
+	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-ldap/ldap/v3 v3.1.8
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/websocket v1.4.1
-	github.com/imdario/mergo v0.3.8 // indirect
+	github.com/joho/godotenv v1.3.0
 	github.com/jpillora/chisel v0.0.0-20190724232113-f3a8df20e389
 	github.com/json-iterator/go v1.1.8
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
 	github.com/mattn/go-shellwords v1.0.6 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
+	github.com/pkg/errors v0.9.1
 	github.com/portainer/libcompose v0.5.3
 	github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
 	github.com/stretchr/testify v1.6.1
-	golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1
-	golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/src-d/go-git.v4 v4.13.1
 	k8s.io/api v0.17.2
 	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v0.17.2
 )
 
 replace github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
+
+replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456

api/go.sum

@@ -11,10 +11,10 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Microsoft/go-winio v0.3.8 h1:dvxbxtpTIjdAbx2OtL26p4eq0iEvys/U5yrsTJb3NZI=
 github.com/Microsoft/go-winio v0.3.8/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -47,7 +47,7 @@ github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVl
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -92,6 +92,15 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck=
 github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.1.0 h1:4pl5BV4o7ZG/lterP4S6WzJ6xr49Ba5ET9ygheTYahk=
+github.com/go-git/go-billy/v5 v5.1.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
+github.com/go-git/go-git/v5 v5.3.0 h1:8WKMtJR2j8RntEXR/uvTKagfEt4GYlwQ7mntE4+0GWc=
+github.com/go-git/go-git/v5 v5.3.0/go.mod h1:xdX4bWJ48aOrdhnl2XqHYstHbbp6+LFS4r4X+lNVprw=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
 github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
@@ -105,7 +114,6 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -147,11 +155,13 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669 h1:l5rH/CnVVu+HPxjtxjM90nHrm4nov3j3RF9/62UjgLs=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669/go.mod h1:kOeLNvjNBGSV3uYtFjvb72+fnZCMFJF1XDvRIjdom0g=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
@@ -168,8 +178,8 @@ github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46O
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c h1:N7A4JCA2G+j5fuFxCsJqjFU/sZe0mj8H0sSoSwbaikw=
@@ -177,13 +187,14 @@ github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c/go.mod h1:Nn
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v0.0.0-20150511174710-5cf931ef8f76/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
@@ -205,6 +216,7 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -218,11 +230,11 @@ github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c h1:iOMba/KmaXg
 github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
-github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -246,9 +258,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@@ -256,15 +267,10 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
-github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
@@ -272,8 +278,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
-github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
-github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
+github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@@ -286,10 +292,9 @@ golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 h1:anGSYQpPhQwXlwsu5wmfq0nWkCNaMEMUwAv13Y92hd8=
-golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -306,12 +311,10 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -322,27 +325,16 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181019160139-8e24a49d80f8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -354,13 +346,11 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
@@ -371,25 +361,22 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
-gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
-gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
-gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
-gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
-gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=

api/http/handler/auth/authenticate.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -130,19 +131,21 @@ func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, use
 }
 
 func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
-	tokenData := &portainer.TokenData{
-		ID:       user.ID,
-		Username: user.Username,
-		Role:     user.Role,
-	}
+	return handler.persistAndWriteToken(w, composeTokenData(user))
+}
 
-	return handler.persistAndWriteToken(w, tokenData)
+func (handler *Handler) writeTokenForOAuth(w http.ResponseWriter, user *portainer.User, expiryTime *time.Time) *httperror.HandlerError {
+	token, err := handler.JWTService.GenerateTokenForOAuth(composeTokenData(user), expiryTime)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
+	}
+	return response.JSON(w, &authenticateResponse{JWT: token})
 }
 
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
 	}
 
 	return response.JSON(w, &authenticateResponse{JWT: token})
@@ -204,3 +207,11 @@ func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamM
 	}
 	return false
 }
+
+func composeTokenData(user *portainer.User) *portainer.TokenData {
+	return &portainer.TokenData{
+		ID:       user.ID,
+		Username: user.Username,
+		Role:     user.Role,
+	}
+}

api/http/handler/auth/authenticate_oauth.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"net/http"
+	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -25,7 +26,24 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id AuthenticateOauth
+func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, *time.Time, error) {
+	if code == "" {
+		return "", nil, errors.New("Invalid OAuth authorization code")
+	}
+
+	if settings == nil {
+		return "", nil, errors.New("Invalid OAuth configuration")
+	}
+
+	username, expiryTime, err := handler.OAuthService.Authenticate(code, settings)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return username, expiryTime, nil
+}
+
+// @id ValidateOAuth
 // @summary Authenticate with OAuth
 // @tags auth
 // @accept json
@@ -36,52 +54,35 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 // @failure 422 "Invalid Credentials"
 // @failure 500 "Server error"
 // @router /auth/oauth/validate [post]
-func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, error) {
-	if code == "" {
-		return "", errors.New("Invalid OAuth authorization code")
-	}
-
-	if settings == nil {
-		return "", errors.New("Invalid OAuth configuration")
-	}
-
-	username, err := handler.OAuthService.Authenticate(code, settings)
-	if err != nil {
-		return "", err
-	}
-
-	return username, nil
-}
-
 func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	var payload oauthPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve settings from the database", Err: err}
 	}
 
-	if settings.AuthenticationMethod != 3 {
-		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", errors.New("OAuth authentication is not enabled")}
+	if settings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "OAuth authentication is not enabled", Err: errors.New("OAuth authentication is not enabled")}
 	}
 
-	username, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
+	username, expiryTime, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
 	if err != nil {
 		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to authenticate through OAuth", Err: httperrors.ErrUnauthorized}
 	}
 
 	user, err := handler.DataStore.User().UserByUsername(username)
 	if err != nil && err != bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a user with the specified username from the database", Err: err}
 	}
 
 	if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
-		return &httperror.HandlerError{http.StatusForbidden, "Account not created beforehand in Portainer and automatic user provisioning not enabled", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Account not created beforehand in Portainer and automatic user provisioning not enabled", Err: httperrors.ErrUnauthorized}
 	}
 
 	if user == nil {
@@ -92,7 +93,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 		err = handler.DataStore.User().CreateUser(user)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist user inside the database", Err: err}
 		}
 
 		if settings.OAuthSettings.DefaultTeamID != 0 {
@@ -104,11 +105,11 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 			err = handler.DataStore.TeamMembership().CreateTeamMembership(membership)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
+				return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist team membership inside the database", Err: err}
 			}
 		}
 
 	}
 
-	return handler.writeToken(w, user)
+	return handler.writeTokenForOAuth(w, user, expiryTime)
 }

api/http/handler/backup/backup.go

@@ -0,0 +1,53 @@
+package backup
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	operations "github.com/portainer/portainer/api/backup"
+)
+
+type (
+	backupPayload struct {
+		Password string
+	}
+)
+
+func (p *backupPayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// @id Backup
+// @summary Creates an archive with a system data snapshot that could be used to restore the system.
+// @description  Creates an archive with a system data snapshot that could be used to restore the system.
+// @description **Access policy**: admin
+// @tags backup
+// @security jwt
+// @produce octet-stream
+// @param Password body string false "Password to encrypt the backup with"
+// @success 200 "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /backup [post]
+func (h *Handler) backup(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload backupPayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	archivePath, err := operations.CreateBackupArchive(payload.Password, h.gate, h.dataStore, h.filestorePath)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to create backup", Err: err}
+	}
+	defer os.RemoveAll(filepath.Dir(archivePath))
+
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fmt.Sprintf("portainer-backup_%s", filepath.Base(archivePath))))
+	http.ServeFile(w, r, archivePath)
+
+	return nil
+}

api/http/handler/backup/backup_test.go

@@ -0,0 +1,122 @@
+package backup
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/portainer/portainer/api/adminmonitor"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/offlinegate"
+	i "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func listFiles(dir string) []string {
+	items := make([]string, 0)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		items = append(items, path)
+		return nil
+	})
+
+	return items
+}
+
+func contains(t *testing.T, list []string, path string) {
+	assert.Contains(t, list, path)
+	copyContent, _ := ioutil.ReadFile(path)
+	assert.Equal(t, "content\n", string(copyContent))
+}
+
+func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T) {
+	r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":""}`))
+	w := httptest.NewRecorder()
+
+	gate := offlinegate.NewOfflineGate()
+	adminMonitor := adminmonitor.New(time.Hour, nil, context.Background())
+
+	handlerErr := NewHandler(nil, i.NewDatastore(), gate, "./test_assets/handler_test", func() {}, adminMonitor).backup(w, r)
+	assert.Nil(t, handlerErr, "Handler should not fail")
+
+	response := w.Result()
+	body, _ := io.ReadAll(response.Body)
+
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	archivePath := filepath.Join(tmpdir, "archive.tar.gz")
+	err := ioutil.WriteFile(archivePath, body, 0600)
+	if err != nil {
+		t.Fatal("Failed to save downloaded .tar.gz archive: ", err)
+	}
+	cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
+	err = cmd.Run()
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+
+	createdFiles := listFiles(tmpdir)
+
+	contains(t, createdFiles, path.Join(tmpdir, "portainer.key"))
+	contains(t, createdFiles, path.Join(tmpdir, "portainer.pub"))
+	contains(t, createdFiles, path.Join(tmpdir, "tls", "file1"))
+	contains(t, createdFiles, path.Join(tmpdir, "tls", "file2"))
+	assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_file"))
+	assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_folder", "file1"))
+}
+
+func Test_backupHandlerWithPassword_shouldCreateEncryptedATarballArchive(t *testing.T) {
+	r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":"secret"}`))
+	w := httptest.NewRecorder()
+
+	gate := offlinegate.NewOfflineGate()
+	adminMonitor := adminmonitor.New(time.Hour, nil, nil)
+
+	handlerErr := NewHandler(nil, i.NewDatastore(), gate, "./test_assets/handler_test", func() {}, adminMonitor).backup(w, r)
+	assert.Nil(t, handlerErr, "Handler should not fail")
+
+	response := w.Result()
+	body, _ := io.ReadAll(response.Body)
+
+	tmpdir, _ := ioutils.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	dr, err := crypto.AesDecrypt(bytes.NewReader(body), []byte("secret"))
+	if err != nil {
+		t.Fatal("Failed to decrypt archive")
+	}
+
+	archivePath := filepath.Join(tmpdir, "archive.tag.gz")
+	archive, _ := os.Create(archivePath)
+	defer archive.Close()
+	io.Copy(archive, dr)
+
+	cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
+	err = cmd.Run()
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+
+	createdFiles := listFiles(tmpdir)
+
+	contains(t, createdFiles, path.Join(tmpdir, "portainer.key"))
+	contains(t, createdFiles, path.Join(tmpdir, "portainer.pub"))
+	contains(t, createdFiles, path.Join(tmpdir, "tls", "file1"))
+	contains(t, createdFiles, path.Join(tmpdir, "tls", "file2"))
+	assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_file"))
+	assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_folder", "file1"))
+}

api/http/handler/backup/handler.go

@@ -0,0 +1,65 @@
+package backup
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/adminmonitor"
+	"github.com/portainer/portainer/api/http/offlinegate"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// Handler is an http handler responsible for backup and restore portainer state
+type Handler struct {
+	*mux.Router
+	bouncer         *security.RequestBouncer
+	dataStore       portainer.DataStore
+	gate            *offlinegate.OfflineGate
+	filestorePath   string
+	shutdownTrigger context.CancelFunc
+	adminMonitor    *adminmonitor.Monitor
+}
+
+// NewHandler creates an new instance of backup handler
+func NewHandler(bouncer *security.RequestBouncer, dataStore portainer.DataStore, gate *offlinegate.OfflineGate, filestorePath string, shutdownTrigger context.CancelFunc, adminMonitor *adminmonitor.Monitor) *Handler {
+	h := &Handler{
+		Router:          mux.NewRouter(),
+		bouncer:         bouncer,
+		dataStore:       dataStore,
+		gate:            gate,
+		filestorePath:   filestorePath,
+		shutdownTrigger: shutdownTrigger,
+		adminMonitor:    adminMonitor,
+	}
+
+	h.Handle("/backup", bouncer.RestrictedAccess(adminAccess(httperror.LoggerHandler(h.backup)))).Methods(http.MethodPost)
+	h.Handle("/restore", bouncer.PublicAccess(httperror.LoggerHandler(h.restore))).Methods(http.MethodPost)
+
+	return h
+}
+
+func adminAccess(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		securityContext, err := security.RetrieveRestrictedRequestContext(r)
+		if err != nil {
+			httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user info from request context", err)
+		}
+
+		if !securityContext.IsAdmin {
+			httperror.WriteError(w, http.StatusUnauthorized, "User is not authorized to perfom the action", nil)
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+func systemWasInitialized(dataStore portainer.DataStore) (bool, error) {
+	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		return false, err
+	}
+	return len(users) > 0, nil
+}

api/http/handler/backup/restore.go

@@ -0,0 +1,69 @@
+package backup
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	operations "github.com/portainer/portainer/api/backup"
+)
+
+type restorePayload struct {
+	FileContent []byte
+	FileName    string
+	Password    string
+}
+
+// @id Restore
+// @summary Triggers a system restore using provided backup file
+// @description Triggers a system restore using provided backup file
+// @description **Access policy**: public
+// @tags backup
+// @param FileContent body []byte true "Content of the backup"
+// @param FileName body string true "File name"
+// @param Password body string false "Password to decrypt the backup with"
+// @success 200  "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /restore [post]
+func (h *Handler) restore(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	initialized, err := h.adminMonitor.WasInitialized()
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to check system initialization", Err: err}
+	}
+	if initialized {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Cannot restore already initialized instance", Err: errors.New("system already initialized")}
+	}
+	h.adminMonitor.Stop()
+	defer h.adminMonitor.Start()
+
+	var payload restorePayload
+	err = decodeForm(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	var archiveReader io.Reader = bytes.NewReader(payload.FileContent)
+	err = operations.RestoreArchive(archiveReader, payload.Password, h.filestorePath, h.gate, h.dataStore, h.shutdownTrigger)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to restore the backup", Err: err}
+	}
+
+	return nil
+}
+
+func decodeForm(r *http.Request, p *restorePayload) error {
+	content, name, err := request.RetrieveMultiPartFormFile(r, "file")
+	if err != nil {
+		return err
+	}
+	p.FileContent = content
+	p.FileName = name
+
+	password, _ := request.RetrieveMultiPartFormValue(r, "password", true)
+	p.Password = password
+	return nil
+}

api/http/handler/backup/restore_test.go

@@ -0,0 +1,123 @@
+package backup
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/adminmonitor"
+	"github.com/portainer/portainer/api/http/offlinegate"
+	i "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_restoreArchive_usingCombinationOfPasswords(t *testing.T) {
+	tests := []struct {
+		name            string
+		backupPassword  string
+		restorePassword string
+		fails           bool
+	}{
+		{
+			name:            "empty password to both encrypt and decrypt",
+			backupPassword:  "",
+			restorePassword: "",
+			fails:           false,
+		},
+		{
+			name:            "same password to encrypt and decrypt",
+			backupPassword:  "secret",
+			restorePassword: "secret",
+			fails:           false,
+		},
+		{
+			name:            "different passwords to encrypt and decrypt",
+			backupPassword:  "secret",
+			restorePassword: "terces",
+			fails:           true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			datastore := i.NewDatastore(i.WithUsers([]portainer.User{}), i.WithEdgeJobs([]portainer.EdgeJob{}))
+			adminMonitor := adminmonitor.New(time.Hour, datastore, context.Background())
+
+			h := NewHandler(nil, datastore, offlinegate.NewOfflineGate(), "./test_assets/handler_test", func() {}, adminMonitor)
+
+			//backup
+			archive := backup(t, h, test.backupPassword)
+
+			//restore
+			w := httptest.NewRecorder()
+			r, err := prepareMultipartRequest(test.restorePassword, archive)
+			assert.Nil(t, err, "Shouldn't fail to write multipart form")
+
+			restoreErr := h.restore(w, r)
+			assert.Equal(t, test.fails, restoreErr != nil, "Didn't meet expectation of failing restore handler")
+		})
+	}
+}
+
+func Test_restoreArchive_shouldFailIfSystemWasAlreadyInitialized(t *testing.T) {
+	admin := portainer.User{
+		Role: portainer.AdministratorRole,
+	}
+	datastore := i.NewDatastore(i.WithUsers([]portainer.User{admin}), i.WithEdgeJobs([]portainer.EdgeJob{}))
+	adminMonitor := adminmonitor.New(time.Hour, datastore, context.Background())
+
+	h := NewHandler(nil, datastore, offlinegate.NewOfflineGate(), "./test_assets/handler_test", func() {}, adminMonitor)
+
+	//backup
+	archive := backup(t, h, "password")
+
+	//restore
+	w := httptest.NewRecorder()
+	r, err := prepareMultipartRequest("password", archive)
+	assert.Nil(t, err, "Shouldn't fail to write multipart form")
+
+	restoreErr := h.restore(w, r)
+	assert.NotNil(t, restoreErr, "Should fail, because system it already initialized")
+	assert.Equal(t, "Cannot restore already initialized instance", restoreErr.Message, "Should fail with certain error")
+}
+
+func backup(t *testing.T, h *Handler, password string) []byte {
+	r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"password":"%s"}`, password)))
+	w := httptest.NewRecorder()
+
+	backupErr := h.backup(w, r)
+	assert.Nil(t, backupErr, "Backup should not fail")
+
+	response := w.Result()
+	archive, _ := io.ReadAll(response.Body)
+	return archive
+}
+
+func prepareMultipartRequest(password string, file []byte) (*http.Request, error) {
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	err := w.WriteField("password", password)
+	if err != nil {
+		return nil, err
+	}
+	fw, err := w.CreateFormFile("file", "filename")
+	if err != nil {
+		return nil, err
+	}
+	io.Copy(fw, bytes.NewReader(file))
+
+	r := httptest.NewRequest(http.MethodPost, "http://localhost/", &body)
+	r.Header.Set("Content-Type", w.FormDataContentType())
+
+	w.Close()
+
+	return r, nil
+}

api/http/handler/backup/test_assets/handler_test/extra_file

@@ -0,0 +1 @@
+content

api/http/handler/backup/test_assets/handler_test/extra_folder/file1

@@ -0,0 +1 @@
+content

api/http/handler/backup/test_assets/handler_test/portainer.key

@@ -0,0 +1 @@
+content

api/http/handler/backup/test_assets/handler_test/portainer.pub

@@ -0,0 +1 @@
+content

api/http/handler/backup/test_assets/handler_test/tls/file1

@@ -0,0 +1 @@
+content

api/http/handler/backup/test_assets/handler_test/tls/file2

@@ -0,0 +1 @@
+content

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -3,11 +3,12 @@ package endpointproxy
 import (
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 
 	"net/http"
@@ -66,9 +67,15 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 
 	requestPrefix := fmt.Sprintf("/%d/kubernetes", endpointID)
 	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		requestPrefix = fmt.Sprintf("/%d", endpointID)
+		if isKubernetesRequest(strings.TrimPrefix(r.URL.String(), requestPrefix)) {
+			requestPrefix = fmt.Sprintf("/%d", endpointID)
+		}
 	}
 
 	http.StripPrefix(requestPrefix, proxy).ServeHTTP(w, r)
 	return nil
 }
+
+func isKubernetesRequest(requestURL string) bool {
+	return strings.HasPrefix(requestURL, "/api") || strings.HasPrefix(requestURL, "/healthz")
+}

api/http/handler/endpoints/endpoint_create.go

@@ -156,7 +156,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @accept multipart/form-data
 // @produce json
 // @param Name formData string true "Name that will be used to identify this endpoint (example: my-endpoint)"
-// @param EndpointType formData integer true "Environment type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5)
+// @param EndpointCreationType formData integer true "Environment type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5)
 // @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine)"
 // @param PublicURL formData string false "URL or IP address where exposed containers will be reachable. Defaults to URL if not specified (example: docker.mydomain.tld:2375)"
 // @param GroupID formData int false "Endpoint group identifier. If not specified will default to 1 (unassigned)."
@@ -471,6 +471,7 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.
 		AllowVolumeBrowserForRegularUsers: false,
 		EnableHostManagementFeatures:      false,
 
+		AllowSysctlSettingForRegularUsers:         true,
 		AllowBindMountsForRegularUsers:            true,
 		AllowPrivilegedModeForRegularUsers:        true,
 		AllowHostNamespaceForRegularUsers:         true,

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -0,0 +1,140 @@
+package endpoints
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+)
+
+type dockerhubStatusResponse struct {
+	Remaining int `json:"remaining"`
+	Limit     int `json:"limit"`
+}
+
+// GET request on /api/endpoints/{id}/dockerhub/status
+func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if !endpointutils.IsLocalEndpoint(endpoint) {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")}
+	}
+
+	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+	}
+
+	httpClient := client.NewHTTPClient()
+	token, err := getDockerHubToken(httpClient, dockerhub)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err}
+	}
+
+	resp, err := getDockerHubLimits(httpClient, token)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub rate limits from DockerHub", err}
+	}
+
+	return response.JSON(w, resp)
+}
+
+func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) {
+	type dockerhubTokenResponse struct {
+		Token string `json:"token"`
+	}
+
+	requestURL := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull"
+
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	if dockerhub.Authentication {
+		req.SetBasicAuth(dockerhub.Username, dockerhub.Password)
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.New("failed fetching dockerhub token")
+	}
+
+	var data dockerhubTokenResponse
+	err = json.NewDecoder(resp.Body).Decode(&data)
+	if err != nil {
+		return "", err
+	}
+
+	return data.Token, nil
+}
+
+func getDockerHubLimits(httpClient *client.HTTPClient, token string) (*dockerhubStatusResponse, error) {
+
+	requestURL := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest"
+
+	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("failed fetching dockerhub limits")
+	}
+
+	rateLimit, err := parseRateLimitHeader(resp.Header, "RateLimit-Limit")
+	rateLimitRemaining, err := parseRateLimitHeader(resp.Header, "RateLimit-Remaining")
+
+	return &dockerhubStatusResponse{
+		Limit:     rateLimit,
+		Remaining: rateLimitRemaining,
+	}, nil
+}
+
+func parseRateLimitHeader(headers http.Header, headerKey string) (int, error) {
+	headerValue := headers.Get(headerKey)
+	if headerValue == "" {
+		return 0, fmt.Errorf("Missing %s header", headerKey)
+	}
+
+	matches := strings.Split(headerValue, ";")
+	value, err := strconv.Atoi(matches[0])
+	if err != nil {
+		return 0, err
+	}
+
+	return value, nil
+}

api/http/handler/endpoints/endpoint_list.go

@@ -66,6 +66,11 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
 
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+	}
+
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
@@ -108,6 +113,9 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 	for idx := range paginatedEndpoints {
 		hideFields(&paginatedEndpoints[idx])
 		paginatedEndpoints[idx].ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
+		if paginatedEndpoints[idx].EdgeCheckinInterval == 0 {
+			paginatedEndpoints[idx].EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
+		}
 	}
 
 	w.Header().Set("X-Total-Count", strconv.Itoa(filteredEndpointCount))

api/http/handler/endpoints/endpoint_settings_update.go

@@ -25,6 +25,8 @@ type endpointSettingsUpdatePayload struct {
 	AllowStackManagementForRegularUsers *bool `json:"allowStackManagementForRegularUsers" example:"true"`
 	// Whether non-administrator should be able to use container capabilities
 	AllowContainerCapabilitiesForRegularUsers *bool `json:"allowContainerCapabilitiesForRegularUsers" example:"true"`
+	// Whether non-administrator should be able to use sysctl settings
+	AllowSysctlSettingForRegularUsers *bool `json:"allowSysctlSettingForRegularUsers" example:"true"`
 	// Whether host management features are enabled
 	EnableHostManagementFeatures *bool `json:"enableHostManagementFeatures" example:"true"`
 }
@@ -97,6 +99,10 @@ func (handler *Handler) endpointSettingsUpdate(w http.ResponseWriter, r *http.Re
 		securitySettings.AllowVolumeBrowserForRegularUsers = *payload.AllowVolumeBrowserForRegularUsers
 	}
 
+	if payload.AllowSysctlSettingForRegularUsers != nil {
+		securitySettings.AllowSysctlSettingForRegularUsers = *payload.AllowSysctlSettingForRegularUsers
+	}
+
 	if payload.EnableHostManagementFeatures != nil {
 		securitySettings.EnableHostManagementFeatures = *payload.EnableHostManagementFeatures
 	}

api/http/handler/endpoints/handler.go

@@ -51,6 +51,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoints/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoints/{id}/dockerhub",
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
 	h.Handle("/endpoints/{id}/extensions",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)
 	h.Handle("/endpoints/{id}/extensions/{extensionType}",

api/http/handler/handler.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/portainer/portainer/api/http/handler/auth"
+	"github.com/portainer/portainer/api/http/handler/backup"
 	"github.com/portainer/portainer/api/http/handler/customtemplates"
 	"github.com/portainer/portainer/api/http/handler/dockerhub"
 	"github.com/portainer/portainer/api/http/handler/edgegroups"
@@ -36,6 +37,7 @@ import (
 // Handler is a collection of all the service handlers.
 type Handler struct {
 	AuthHandler            *auth.Handler
+	BackupHandler          *backup.Handler
 	CustomTemplatesHandler *customtemplates.Handler
 	DockerHubHandler       *dockerhub.Handler
 	EdgeGroupsHandler      *edgegroups.Handler
@@ -140,6 +142,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch {
 	case strings.HasPrefix(r.URL.Path, "/api/auth"):
 		http.StripPrefix("/api", h.AuthHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/backup"):
+		http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/restore"):
+		http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/dockerhub"):
 		http.StripPrefix("/api", h.DockerHubHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/custom_templates"):

api/http/handler/registries/registry_create.go

@@ -26,6 +26,8 @@ type registryCreatePayload struct {
 	Password string `example:"registry_password"`
 	// Gitlab specific details, required when type = 4
 	Gitlab portainer.GitlabRegistryData
+	// Quay specific details, required when type = 1
+	Quay portainer.QuayRegistryData
 }
 
 func (payload *registryCreatePayload) Validate(r *http.Request) error {
@@ -74,6 +76,7 @@ func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *
 		UserAccessPolicies: portainer.UserAccessPolicies{},
 		TeamAccessPolicies: portainer.TeamAccessPolicies{},
 		Gitlab:             payload.Gitlab,
+		Quay:               payload.Quay,
 	}
 
 	err = handler.DataStore.Registry().CreateRegistry(registry)

api/http/handler/registries/registry_update.go

@@ -24,6 +24,7 @@ type registryUpdatePayload struct {
 	Password           *string `example:"registry_password"`
 	UserAccessPolicies portainer.UserAccessPolicies
 	TeamAccessPolicies portainer.TeamAccessPolicies
+	Quay               *portainer.QuayRegistryData
 }
 
 func (payload *registryUpdatePayload) Validate(r *http.Request) error {
@@ -110,6 +111,10 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 		registry.TeamAccessPolicies = payload.TeamAccessPolicies
 	}
 
+	if payload.Quay != nil {
+		registry.Quay = *payload.Quay
+	}
+
 	err = handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist registry changes inside the database", err}

api/http/handler/resourcecontrols/resourcecontrol_create.go

@@ -78,6 +78,8 @@ func (handler *Handler) resourceControlCreate(w http.ResponseWriter, r *http.Req
 	switch payload.Type {
 	case "container":
 		resourceControlType = portainer.ContainerResourceControl
+	case "container-group":
+		resourceControlType = portainer.ContainerGroupResourceControl
 	case "service":
 		resourceControlType = portainer.ServiceResourceControl
 	case "volume":

api/http/handler/settings/settings_public.go

@@ -18,6 +18,8 @@ type publicSettingsResponse struct {
 	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures" example:"true"`
 	// The URL used for oauth login
 	OAuthLoginURI string `json:"OAuthLoginURI" example:"https://gitlab.com/oauth"`
+	// The URL used for oauth logout
+	OAuthLogoutURI string `json:"OAuthLogoutURI" example:"https://gitlab.com/oauth/logout"`
 	// Whether telemetry is enabled
 	EnableTelemetry bool `json:"EnableTelemetry" example:"true"`
 }
@@ -34,20 +36,32 @@ type publicSettingsResponse struct {
 func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve the settings from the database", err}
-	}
-
-	publicSettings := &publicSettingsResponse{
-		LogoURL:                   settings.LogoURL,
-		AuthenticationMethod:      settings.AuthenticationMethod,
-		EnableEdgeComputeFeatures: settings.EnableEdgeComputeFeatures,
-		EnableTelemetry:           settings.EnableTelemetry,
-		OAuthLoginURI: fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s&prompt=login",
-			settings.OAuthSettings.AuthorizationURI,
-			settings.OAuthSettings.ClientID,
-			settings.OAuthSettings.RedirectURI,
-			settings.OAuthSettings.Scopes),
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve the settings from the database", Err: err}
 	}
 
+	publicSettings := generatePublicSettings(settings)
 	return response.JSON(w, publicSettings)
 }
+
+func generatePublicSettings(appSettings *portainer.Settings) *publicSettingsResponse {
+	publicSettings := &publicSettingsResponse{
+		LogoURL:                   appSettings.LogoURL,
+		AuthenticationMethod:      appSettings.AuthenticationMethod,
+		EnableEdgeComputeFeatures: appSettings.EnableEdgeComputeFeatures,
+		EnableTelemetry:           appSettings.EnableTelemetry,
+	}
+	//if OAuth authentication is on, compose the related fields from application settings
+	if publicSettings.AuthenticationMethod == portainer.AuthenticationOAuth {
+		publicSettings.OAuthLogoutURI = appSettings.OAuthSettings.LogoutURI
+		publicSettings.OAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
+			appSettings.OAuthSettings.AuthorizationURI,
+			appSettings.OAuthSettings.ClientID,
+			appSettings.OAuthSettings.RedirectURI,
+			appSettings.OAuthSettings.Scopes)
+		//control prompt=login param according to the SSO setting
+		if !appSettings.OAuthSettings.SSO {
+			publicSettings.OAuthLoginURI += "&prompt=login"
+		}
+	}
+	return publicSettings
+}

api/http/handler/settings/settings_public_test.go

@@ -0,0 +1,70 @@
+package settings
+
+import (
+	"fmt"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+const (
+	dummyOAuthClientID          = "1a2b3c4d"
+	dummyOAuthScopes            = "scopes"
+	dummyOAuthAuthenticationURI = "example.com/auth"
+	dummyOAuthRedirectURI       = "example.com/redirect"
+	dummyOAuthLogoutURI         = "example.com/logout"
+)
+
+var (
+	dummyOAuthLoginURI string
+	mockAppSettings    *portainer.Settings
+)
+
+func setup() {
+	dummyOAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
+		dummyOAuthAuthenticationURI,
+		dummyOAuthClientID,
+		dummyOAuthRedirectURI,
+		dummyOAuthScopes)
+	mockAppSettings = &portainer.Settings{
+		AuthenticationMethod: portainer.AuthenticationOAuth,
+		OAuthSettings: portainer.OAuthSettings{
+			AuthorizationURI: dummyOAuthAuthenticationURI,
+			ClientID:         dummyOAuthClientID,
+			Scopes:           dummyOAuthScopes,
+			RedirectURI:      dummyOAuthRedirectURI,
+			LogoutURI:        dummyOAuthLogoutURI,
+		},
+	}
+}
+
+func TestGeneratePublicSettingsWithSSO(t *testing.T) {
+	setup()
+	mockAppSettings.OAuthSettings.SSO = true
+	publicSettings := generatePublicSettings(mockAppSettings)
+	if publicSettings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		t.Errorf("wrong AuthenticationMethod, want: %d, got: %d", portainer.AuthenticationOAuth, publicSettings.AuthenticationMethod)
+	}
+	if publicSettings.OAuthLoginURI != dummyOAuthLoginURI {
+		t.Errorf("wrong OAuthLoginURI when SSO is switched on, want: %s, got: %s", dummyOAuthLoginURI, publicSettings.OAuthLoginURI)
+	}
+	if publicSettings.OAuthLogoutURI != dummyOAuthLogoutURI {
+		t.Errorf("wrong OAuthLogoutURI, want: %s, got: %s", dummyOAuthLogoutURI, publicSettings.OAuthLogoutURI)
+	}
+}
+
+func TestGeneratePublicSettingsWithoutSSO(t *testing.T) {
+	setup()
+	mockAppSettings.OAuthSettings.SSO = false
+	publicSettings := generatePublicSettings(mockAppSettings)
+	if publicSettings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		t.Errorf("wrong AuthenticationMethod, want: %d, got: %d", portainer.AuthenticationOAuth, publicSettings.AuthenticationMethod)
+	}
+	expectedOAuthLoginURI := dummyOAuthLoginURI + "&prompt=login"
+	if publicSettings.OAuthLoginURI != expectedOAuthLoginURI {
+		t.Errorf("wrong OAuthLoginURI when SSO is switched off, want: %s, got: %s", expectedOAuthLoginURI, publicSettings.OAuthLoginURI)
+	}
+	if publicSettings.OAuthLogoutURI != dummyOAuthLogoutURI {
+		t.Errorf("wrong OAuthLogoutURI, want: %s, got: %s", dummyOAuthLogoutURI, publicSettings.OAuthLogoutURI)
+	}
+}

api/http/handler/stacks/create_compose_stack.go

@@ -5,9 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"path"
-	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/asaskevich/govalidator"
@@ -18,13 +16,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 )
 
-// this is coming from libcompose
-// https://github.com/portainer/libcompose/blob/master/project/context.go#L117-L120
-func normalizeStackName(name string) string {
-	r := regexp.MustCompile("[^a-z0-9]+")
-	return r.ReplaceAllString(strings.ToLower(name), "")
-}
-
 type composeStackFromFileContentPayload struct {
 	// Name of the stack
 	Name string `example:"myStack" validate:"required"`
@@ -38,7 +29,7 @@ func (payload *composeStackFromFileContentPayload) Validate(r *http.Request) err
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid stack name")
 	}
-	payload.Name = normalizeStackName(payload.Name)
+
 	if govalidator.IsNull(payload.StackFileContent) {
 		return errors.New("Invalid stack file content")
 	}
@@ -49,9 +40,11 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 	var payload composeStackFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
+	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
+
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
@@ -76,7 +69,7 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 	stackFolder := strconv.Itoa(int(stack.ID))
 	projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist Compose file on disk", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist Compose file on disk", Err: err}
 	}
 	stack.ProjectPath = projectPath
 
@@ -90,14 +83,14 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 
 	err = handler.deployComposeStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
 	doCleanUp = false
@@ -129,16 +122,14 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid stack name")
 	}
-	payload.Name = normalizeStackName(payload.Name)
+
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
 	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
 		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
 	}
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
-	}
+
 	return nil
 }
 
@@ -146,7 +137,12 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 	var payload composeStackFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
+	if payload.ComposeFilePathInRepository == "" {
+		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
 	}
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
@@ -154,7 +150,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
+		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
 		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
 	}
 
@@ -187,7 +183,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 
 	err = handler.cloneGitRepository(gitCloneParams)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
 	config, configErr := handler.createComposeDeployConfig(r, stack, endpoint)
@@ -197,14 +193,14 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 
 	err = handler.deployComposeStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
 	doCleanUp = false
@@ -217,41 +213,43 @@ type composeStackFromFileUploadPayload struct {
 	Env              []portainer.Pair
 }
 
-func (payload *composeStackFromFileUploadPayload) Validate(r *http.Request) error {
+func decodeRequestForm(r *http.Request) (*composeStackFromFileUploadPayload, error) {
+	payload := &composeStackFromFileUploadPayload{}
 	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
 	if err != nil {
-		return errors.New("Invalid stack name")
+		return nil, errors.New("Invalid stack name")
 	}
-	payload.Name = normalizeStackName(name)
+	payload.Name = name
 
 	composeFileContent, _, err := request.RetrieveMultiPartFormFile(r, "file")
 	if err != nil {
-		return errors.New("Invalid Compose file. Ensure that the Compose file is uploaded correctly")
+		return nil, errors.New("Invalid Compose file. Ensure that the Compose file is uploaded correctly")
 	}
 	payload.StackFileContent = composeFileContent
 
 	var env []portainer.Pair
 	err = request.RetrieveMultiPartFormJSONValue(r, "Env", &env, true)
 	if err != nil {
-		return errors.New("Invalid Env parameter")
+		return nil, errors.New("Invalid Env parameter")
 	}
 	payload.Env = env
-	return nil
+	return payload, nil
 }
 
 func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {
-	payload := &composeStackFromFileUploadPayload{}
-	err := payload.Validate(r)
+	payload, err := decodeRequestForm(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
+	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
+
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
+		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
 		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
 	}
 
@@ -270,7 +268,7 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 	stackFolder := strconv.Itoa(int(stack.ID))
 	projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, payload.StackFileContent)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist Compose file on disk", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist Compose file on disk", Err: err}
 	}
 	stack.ProjectPath = projectPath
 
@@ -284,14 +282,14 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 
 	err = handler.deployComposeStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
 	doCleanUp = false
@@ -310,23 +308,23 @@ type composeStackDeploymentConfig struct {
 func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) (*composeStackDeploymentConfig, *httperror.HandlerError) {
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
 	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve DockerHub details from the database", Err: err}
 	}
 
 	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
+		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve registries from the database", Err: err}
 	}
 	filteredRegistries := security.FilterRegistries(registries, securityContext)
 
 	user, err := handler.DataStore.User().User(securityContext.UserID)
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to load user information from the database", err}
+		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
 
 	config := &composeStackDeploymentConfig{
@@ -358,6 +356,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 		!securitySettings.AllowPrivilegedModeForRegularUsers ||
 		!securitySettings.AllowHostNamespaceForRegularUsers ||
 		!securitySettings.AllowDeviceMappingForRegularUsers ||
+		!securitySettings.AllowSysctlSettingForRegularUsers ||
 		!securitySettings.AllowContainerCapabilitiesForRegularUsers) &&
 		!isAdminOrEndpointAdmin {
 

api/http/handler/stacks/create_kubernetes_stack.go

@@ -10,6 +10,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
 )
 
 type kubernetesStackPayload struct {
@@ -33,6 +34,10 @@ type createKubernetesStackResponse struct {
 }
 
 func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	if !endpointutils.IsKubernetesEndpoint(endpoint) {
+		return &httperror.HandlerError{http.StatusBadRequest, "Endpoint type does not match", errors.New("Endpoint type does not match")}
+	}
+
 	var payload kubernetesStackPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
@@ -45,15 +50,24 @@ func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Req
 	}
 
 	resp := &createKubernetesStackResponse{
-		Output: string(output),
+		Output: output,
 	}
 
 	return response.JSON(w, resp)
 }
 
-func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	return handler.KubernetesDeployer.Deploy(endpoint, data, composeFormat, namespace)
+	if composeFormat {
+		convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(stackConfig)
+		if err != nil {
+			return "", err
+		}
+		stackConfig = string(convertedConfig)
+	}
+
+	return handler.KubernetesDeployer.Deploy(endpoint, stackConfig, namespace)
+
 }

api/http/handler/stacks/handler.go

@@ -52,6 +52,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackInspect))).Methods(http.MethodGet)
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackDelete))).Methods(http.MethodDelete)
+	h.Handle("/stacks/{id}/associate",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.stackAssociate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/file",

api/http/handler/stacks/stack_associate.go

@@ -0,0 +1,91 @@
+package stacks
+
+import (
+	"fmt"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/stackutils"
+	"net/http"
+	"time"
+)
+
+// PUT request on /api/stacks/:id/associate?endpointId=<endpointId>&swarmId=<swarmId>&orphanedRunning=<orphanedRunning>
+func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+	}
+
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+	}
+
+	swarmId, err := request.RetrieveQueryParameter(r, "swarmId", true)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: swarmId", err}
+	}
+
+	orphanedRunning, err := request.RetrieveBooleanQueryParameter(r, "orphanedRunning", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: orphanedRunning", err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	user, err := handler.DataStore.User().User(securityContext.UserID)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to load user information from the database", err}
+	}
+
+	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+	}
+
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+	}
+
+	if resourceControl != nil {
+		resourceControl.ResourceID = fmt.Sprintf("%d_%s", endpointID, stack.Name)
+
+		err = handler.DataStore.ResourceControl().UpdateResourceControl(resourceControl.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist resource control changes inside the database", err}
+		}
+	}
+
+	stack.EndpointID = portainer.EndpointID(endpointID)
+	stack.SwarmID = swarmId
+
+	if orphanedRunning {
+		stack.Status = portainer.StackStatusActive
+	} else {
+		stack.Status = portainer.StackStatusInactive
+	}
+
+	stack.CreationDate = time.Now().Unix()
+	stack.CreatedBy = user.Username
+	stack.UpdateDate = 0
+	stack.UpdatedBy = ""
+
+	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+	}
+
+	stack.ResourceControl = resourceControl
+
+	return response.JSON(w, stack)
+}

api/http/handler/stacks/stack_create.go

@@ -192,6 +192,10 @@ func (handler *Handler) isValidStackFile(stackFileContent []byte, securitySettin
 			return errors.New("device mapping disabled for non administrator users")
 		}
 
+		if !securitySettings.AllowSysctlSettingForRegularUsers && service.Sysctls != nil && len(service.Sysctls) > 0 {
+			return errors.New("sysctl setting disabled for non administrator users")
+		}
+
 		if !securitySettings.AllowContainerCapabilitiesForRegularUsers && (len(service.CapAdd) > 0 || len(service.CapDrop) > 0) {
 			return errors.New("container capabilities disabled for non administrator users")
 		}

api/http/handler/stacks/stack_delete.go

@@ -58,42 +58,42 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
-	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
-	// The EndpointID property is not available for these stacks, this API endpoint
-	// can use the optional EndpointID query parameter to set a valid endpoint identifier to be
-	// used in the context of this request.
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
 	}
-	endpointIdentifier := stack.EndpointID
-	if endpointID != 0 {
-		endpointIdentifier = portainer.EndpointID(endpointID)
+
+	isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID
+
+	if isOrphaned && !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack")}
 	}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointIdentifier)
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
-
 	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
 	}
 
-	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-	}
-	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+	if !isOrphaned {
+		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		}
+
+		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		}
+		if !access {
+			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+		}
 	}
 
 	err = handler.deleteStack(stack, endpoint)

api/http/handler/stacks/stack_file.go

@@ -46,34 +46,38 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
-	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
-	}
-
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
-
-	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-	}
-
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
 	}
 
-	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
+	if err == bolterrors.ErrObjectNotFound {
+		if !securityContext.IsAdmin {
+			return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+		}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
-	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
+
+	if endpoint != nil {
+		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		}
+
+		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		}
+
+		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		}
+		if !access {
+			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
+		}
 	}
 
 	stackFileContent, err := handler.FileService.GetFileContent(path.Join(stack.ProjectPath, stack.EntryPoint))