feat(app): introduce help and support view

feat(app): help and support secondary view layout proposal

feat(app): redirect to help page on first login only

refactor(app): make UI layout more in line with mockups

feat(app/help): redirect to the appropriate view on help exit

.gitignore

@@ -11,3 +11,5 @@ api/cmd/portainer/portainer*
 __debug_bin
 
 api/docs
+.idea
+.env

.vscode.example/settings.json

@@ -0,0 +1,4 @@
+{
+  "go.lintTool": "golangci-lint",
+  "go.lintFlags": ["--fast", "-E", "exportloopref"]
+}

api/archive/zip.go

@@ -3,10 +3,13 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
+	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -52,3 +55,60 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 
 	return outFile.Close()
 }
+
+// UnzipFile will decompress a zip archive, moving all files and folders
+// within the zip file (parameter 1) to an output directory (parameter 2).
+func UnzipFile(src string, dest string) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		p := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+		if !strings.HasPrefix(p, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: illegal file path", p)
+		}
+
+		if f.FileInfo().IsDir() {
+			// Make Folder
+			os.MkdirAll(p, os.ModePerm)
+			continue
+		}
+
+		err = unzipFile(f, p)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func unzipFile(f *zip.File, p string) error {
+	// Make File
+	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
+		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
+	}
+	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
+	}
+	defer outFile.Close()
+	rc, err := f.Open()
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
+	}
+	defer rc.Close()
+
+	_, err = io.Copy(outFile, rc)
+
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
+	}
+
+	return nil
+}

api/archive/zip_test.go

@@ -0,0 +1,32 @@
+package archive
+
+import (
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestUnzipFile(t *testing.T) {
+	dir, err := ioutil.TempDir("", "unzip-test-")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+	/*
+		Archive structure.
+		├── 0
+		│	├── 1
+		│	│	└── 2.txt
+		│	└── 1.txt
+		└── 0.txt
+	*/
+
+	err = UnzipFile("./testdata/sample_archive.zip", dir)
+
+	assert.NoError(t, err)
+	archiveDir := dir + "/sample_archive"
+	assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
+
+}

api/bolt/bolttest/datastore.go

@@ -0,0 +1,73 @@
+package bolttest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+var errTempDir = errors.New("can't create a temp dir")
+
+func MustNewTestStore(init bool) (*bolt.Store, func()) {
+	store, teardown, err := NewTestStore(init)
+	if err != nil {
+		if !errors.Is(err, errTempDir) {
+			teardown()
+		}
+		log.Fatal(err)
+	}
+
+	return store, teardown
+}
+
+func NewTestStore(init bool) (*bolt.Store, func(), error) {
+	// Creates unique temp directory in a concurrency friendly manner.
+	dataStorePath, err := ioutil.TempDir("", "boltdb")
+	if err != nil {
+		return nil, nil, errors.Wrap(errTempDir, err.Error())
+	}
+
+	fileService, err := filesystem.NewService(dataStorePath, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	store, err := bolt.NewStore(dataStorePath, fileService)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = store.Open()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if init {
+		err = store.Init()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	teardown := func() {
+		teardown(store, dataStorePath)
+	}
+
+	return store, teardown, nil
+}
+
+func teardown(store *bolt.Store, dataStorePath string) {
+	err := store.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	err = os.RemoveAll(dataStorePath)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

api/bolt/datastore.go

@@ -169,6 +169,7 @@ func (store *Store) MigrateData(force bool) error {
 			UserService:             store.UserService,
 			VersionService:          store.VersionService,
 			FileService:             store.fileService,
+			DockerhubService:        store.DockerHubService,
 			AuthorizationService:    authorization.NewService(store),
 		}
 		migrator := migrator.NewMigrator(migratorParams)

api/bolt/errors/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 var (
 	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documention to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 )

api/bolt/init.go

@@ -55,22 +55,6 @@ func (store *Store) Init() error {
 		return err
 	}
 
-	_, err = store.DockerHubService.DockerHub()
-	if err == errors.ErrObjectNotFound {
-		defaultDockerHub := &portainer.DockerHub{
-			Authentication: false,
-			Username:       "",
-			Password:       "",
-		}
-
-		err := store.DockerHubService.UpdateDockerHub(defaultDockerHub)
-		if err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
-	}
-
 	groups, err := store.EndpointGroupService.EndpointGroups()
 	if err != nil {
 		return err

api/bolt/log/log.go

@@ -0,0 +1,41 @@
+package log
+
+import (
+	"fmt"
+	"log"
+)
+
+const (
+	INFO  = "INFO"
+	ERROR = "ERROR"
+	DEBUG = "DEBUG"
+	FATAL = "FATAL"
+)
+
+type ScopedLog struct {
+	scope string
+}
+
+func NewScopedLog(scope string) *ScopedLog {
+	return &ScopedLog{scope: scope}
+}
+
+func (slog *ScopedLog) print(kind string, message string) {
+	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
+}
+
+func (slog *ScopedLog) Debug(message string) {
+	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Info(message string) {
+	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Error(message string, err error) {
+	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
+}
+
+func (slog *ScopedLog) NotImplemented(method string) {
+	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
+}

api/bolt/log/log.test.go

@@ -0,0 +1 @@
+package log

api/bolt/migrator/migrate_dbversion29.go

@@ -1,6 +1,14 @@
 package migrator
 
-func (m *Migrator) updateSettingsToDB31() error {
+func (m *Migrator) migrateDBVersionToDB30() error {
+	if err := m.migrateSettingsToDB30(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) migrateSettingsToDB30() error {
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion29_test.go

@@ -2,9 +2,12 @@ package migrator
 
 import (
 	"os"
+	"path"
 	"testing"
+	"time"
 
 	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/internal"
 	"github.com/portainer/portainer/api/bolt/settings"
 )
 
@@ -16,6 +19,34 @@ var (
 	settingsService    *settings.Service
 )
 
+// initTestingDBConn creates a raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
+	databasePath := path.Join(storePath, fileName)
+	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+	return dbConn, nil
+}
+
+// initTestingDBConn creates a settings service with raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
+	internalDBConn := &internal.DbConnection{
+		DB: dbConn,
+	}
+	settingsService, err := settings.NewService(internalDBConn)
+	if err != nil {
+		return nil, err
+	}
+	//insert a obj
+	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
+		return nil, err
+	}
+	return settingsService, nil
+}
+
 func setup() error {
 	testingDBStorePath, _ = os.Getwd()
 	testingDBFileName = "portainer-ee-mig-30.db"
@@ -35,7 +66,7 @@ func setup() error {
 	return nil
 }
 
-func TestUpdateSettingsToDB31(t *testing.T) {
+func TestMigrateSettings(t *testing.T) {
 	if err := setup(); err != nil {
 		t.Errorf("failed to complete testing setups, err: %v", err)
 	}
@@ -45,7 +76,7 @@ func TestUpdateSettingsToDB31(t *testing.T) {
 		db:              dbConn,
 		settingsService: settingsService,
 	}
-	if err := m.updateSettingsToDB31(); err != nil {
+	if err := m.migrateSettingsToDB30(); err != nil {
 		t.Errorf("failed to update settings: %v", err)
 	}
 	updatedSettings, err := m.settingsService.Settings()

api/bolt/migrator/migrate_dbversion31.go

@@ -0,0 +1,213 @@
+package migrator
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
+	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
+)
+
+func (m *Migrator) migrateDBVersionToDB32() error {
+	err := m.updateRegistriesToDB32()
+	if err != nil {
+		return err
+	}
+
+	err = m.updateDockerhubToDB32()
+	if err != nil {
+		return err
+	}
+
+	if err := m.updateVolumeResourceControlToDB32(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateRegistriesToDB32() error {
+	registries, err := m.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range registries {
+
+		registry.RegistryAccesses = portainer.RegistryAccesses{}
+
+		for _, endpoint := range endpoints {
+
+			filteredUserAccessPolicies := portainer.UserAccessPolicies{}
+			for userId, registryPolicy := range registry.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					filteredUserAccessPolicies[userId] = registryPolicy
+				}
+			}
+
+			filteredTeamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId, registryPolicy := range registry.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					filteredTeamAccessPolicies[teamId] = registryPolicy
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: filteredUserAccessPolicies,
+				TeamAccessPolicies: filteredTeamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+		m.registryService.UpdateRegistry(registry.ID, &registry)
+	}
+	return nil
+}
+
+func (m *Migrator) updateDockerhubToDB32() error {
+	dockerhub, err := m.dockerhubService.DockerHub()
+	if err == errors.ErrObjectNotFound {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if !dockerhub.Authentication {
+		return nil
+	}
+
+	registry := &portainer.Registry{
+		Type:             portainer.DockerHubRegistry,
+		Name:             "Dockerhub (authenticated - migrated)",
+		URL:              "docker.io",
+		Authentication:   true,
+		Username:         dockerhub.Username,
+		Password:         dockerhub.Password,
+		RegistryAccesses: portainer.RegistryAccesses{},
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+
+		if endpoint.Type != portainer.KubernetesLocalEnvironment &&
+			endpoint.Type != portainer.AgentOnKubernetesEnvironment &&
+			endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
+
+			userAccessPolicies := portainer.UserAccessPolicies{}
+			for userId := range endpoint.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					userAccessPolicies[userId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			teamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId := range endpoint.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					teamAccessPolicies[teamId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: userAccessPolicies,
+				TeamAccessPolicies: teamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+	}
+
+	return m.registryService.CreateRegistry(registry)
+}
+
+func (m *Migrator) updateVolumeResourceControlToDB32() error {
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return fmt.Errorf("failed fetching endpoints: %w", err)
+	}
+
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return fmt.Errorf("failed fetching resource controls: %w", err)
+	}
+
+	toUpdate := map[portainer.ResourceControlID]string{}
+	volumeResourceControls := map[string]*portainer.ResourceControl{}
+
+	for i := range resourceControls {
+		resourceControl := resourceControls[i]
+		if resourceControl.Type == portainer.VolumeResourceControl {
+			volumeResourceControls[resourceControl.ResourceID] = &resourceControl
+		}
+	}
+
+	for _, endpoint := range endpoints {
+		if !endpointutils.IsDockerEndpoint(&endpoint) {
+			continue
+		}
+
+		totalSnapshots := len(endpoint.Snapshots)
+		if totalSnapshots == 0 {
+			continue
+		}
+
+		snapshot := endpoint.Snapshots[totalSnapshots-1]
+
+		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
+		if err != nil {
+			return fmt.Errorf("failed fetching endpoint docker id: %w", err)
+		}
+
+		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
+			if volumesData["Volumes"] == nil {
+				continue
+			}
+
+			findResourcesToUpdateForDB32(endpointDockerID, volumesData, toUpdate, volumeResourceControls)
+		}
+	}
+
+	for _, resourceControl := range volumeResourceControls {
+		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
+			resourceControl.ResourceID = newResourceID
+			err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
+			if err != nil {
+				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
+			}
+
+		} else {
+			err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
+			}
+
+		}
+	}
+
+	return nil
+}
+
+func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interface{}, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
+	volumes := volumesData["Volumes"].([]interface{})
+	for _, volumeMeta := range volumes {
+		volume := volumeMeta.(map[string]interface{})
+		volumeName := volume["Name"].(string)
+		oldResourceID := fmt.Sprintf("%s%s", volumeName, volume["CreatedAt"].(string))
+		resourceControl, ok := volumeResourceControls[oldResourceID]
+
+		if ok {
+			toUpdate[resourceControl.ID] = fmt.Sprintf("%s_%s", volumeName, dockerID)
+		}
+	}
+}

api/bolt/migrator/migrate_test_helper.go

@@ -1,38 +0,0 @@
-package migrator
-
-import (
-	"path"
-	"time"
-
-	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/settings"
-)
-
-// initTestingDBConn creates a raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
-	databasePath := path.Join(storePath, fileName)
-	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return nil, err
-	}
-	return dbConn, nil
-}
-
-// initTestingDBConn creates a settings service with raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
-	internalDBConn := &internal.DbConnection{
-		DB: dbConn,
-	}
-	settingsService, err := settings.NewService(internalDBConn)
-	if err != nil {
-		return nil, err
-	}
-	//insert a obj
-	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
-		return nil, err
-	}
-	return settingsService, nil
-}

api/bolt/migrator/migrator.go

@@ -3,10 +3,12 @@ package migrator
 import (
 	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
+	plog "github.com/portainer/portainer/api/bolt/log"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -20,6 +22,8 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
+var migrateLog = plog.NewScopedLog("bolt, migrate")
+
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
@@ -41,6 +45,7 @@ type (
 		versionService          *version.Service
 		fileService             portainer.FileService
 		authorizationService    *authorization.Service
+		dockerhubService        *dockerhub.Service
 	}
 
 	// Parameters represents the required parameters to create a new Migrator instance.
@@ -63,6 +68,7 @@ type (
 		VersionService          *version.Service
 		FileService             portainer.FileService
 		AuthorizationService    *authorization.Service
+		DockerhubService        *dockerhub.Service
 	}
 )
 
@@ -87,6 +93,7 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		versionService:          parameters.VersionService,
 		fileService:             parameters.FileService,
 		authorizationService:    parameters.AuthorizationService,
+		dockerhubService:        parameters.DockerhubService,
 	}
 }
 
@@ -358,9 +365,17 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
-	// Portainer 2.5.0
-	if m.currentDBVersion < 31 {
-		err := m.updateSettingsToDB31()
+	// Portainer 2.6.0
+	if m.currentDBVersion < 30 {
+		err := m.migrateDBVersionToDB30()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.9.0
+	if m.currentDBVersion < 32 {
+		err := m.migrateDBVersionToDB32()
 		if err != nil {
 			return err
 		}

api/bolt/services.go

@@ -167,11 +167,6 @@ func (store *Store) CustomTemplate() portainer.CustomTemplateService {
 	return store.CustomTemplateService
 }
 
-// DockerHub gives access to the DockerHub data management layer
-func (store *Store) DockerHub() portainer.DockerHubService {
-	return store.DockerHubService
-}
-
 // EdgeGroup gives access to the EdgeGroup data management layer
 func (store *Store) EdgeGroup() portainer.EdgeGroupService {
 	return store.EdgeGroupService

api/bolt/user/user.go

@@ -139,6 +139,7 @@ func (service *Service) CreateUser(user *portainer.User) error {
 		id, _ := bucket.NextSequence()
 		user.ID = portainer.UserID(id)
 		user.Username = strings.ToLower(user.Username)
+		user.HasAuthenticatedOnce = false
 
 		data, err := internal.MarshalObject(user)
 		if err != nil {

api/cmd/portainer/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 
+	wrapper "github.com/portainer/docker-compose-wrapper"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/chisel"
@@ -20,6 +21,7 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/jwt"
@@ -76,20 +78,25 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 }
 
 func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper := exec.NewComposeWrapper(assetsPath, proxyManager)
-	if composeWrapper != nil {
-		return composeWrapper
+	composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
+	if err != nil {
+		if err == wrapper.ErrBinaryNotFound {
+			log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]")
+			return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
+		}
+
+		log.Fatalf("failed initalizing compose stack manager; err=%s", err)
 	}
 
-	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
+	return composeWrapper
 }
 
 func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
 	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(assetsPath)
+func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath)
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -133,8 +140,8 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService,
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore portainer.DataStore) *kubecli.ClientFactory {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
 }
 
 func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
@@ -165,6 +172,7 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 	settings.SnapshotInterval = *flags.SnapshotInterval
 	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
 	settings.EnableTelemetry = true
+	settings.OAuthSettings.SSO = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -240,6 +248,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -301,6 +310,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
+			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -378,7 +388,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
+	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
@@ -386,6 +396,9 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 	snapshotService.Start()
 
+	authorizationService := authorization.NewService(dataStore)
+	authorizationService.K8sClientFactory = kubernetesClientFactory
+
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
 	if err != nil {
 		log.Fatalf("failed initializing swarm stack manager: %v", err)
@@ -395,7 +408,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
 
-	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
@@ -458,6 +471,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	return &http.Server{
+		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
 		Status:                      applicationStatus,
 		BindAddress:                 *flags.Addr,

api/exec/compose_stack.go

@@ -0,0 +1,122 @@
+package exec
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+
+	wrapper "github.com/portainer/docker-compose-wrapper"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/proxy/factory"
+)
+
+// ComposeStackManager is a wrapper for docker-compose binary
+type ComposeStackManager struct {
+	wrapper      *wrapper.ComposeWrapper
+	configPath   string
+	proxyManager *proxy.Manager
+}
+
+// NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
+func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
+	wrap, err := wrapper.NewComposeWrapper(binaryPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ComposeStackManager{
+		wrapper:      wrap,
+		proxyManager: proxyManager,
+		configPath:   configPath,
+	}, nil
+}
+
+// NormalizeStackName returns a new stack name with unsupported characters replaced
+func (w *ComposeStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
+}
+
+// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
+func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
+	return portainer.ComposeSyntaxMaxVersion
+}
+
+// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
+func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	url, proxy, err := w.fetchEndpointProxy(endpoint)
+	if err != nil {
+		return err
+	}
+
+	if proxy != nil {
+		defer proxy.Close()
+	}
+
+	envFilePath, err := createEnvFile(stack)
+	if err != nil {
+		return err
+	}
+
+	filePath := stackFilePath(stack)
+
+	_, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath)
+	return err
+}
+
+// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	url, proxy, err := w.fetchEndpointProxy(endpoint)
+	if err != nil {
+		return err
+	}
+	if proxy != nil {
+		defer proxy.Close()
+	}
+
+	filePath := stackFilePath(stack)
+
+	_, err = w.wrapper.Down([]string{filePath}, url, stack.Name)
+	return err
+}
+
+func stackFilePath(stack *portainer.Stack) string {
+	return path.Join(stack.ProjectPath, stack.EntryPoint)
+}
+
+func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
+	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
+		return "", nil, nil
+	}
+
+	proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return fmt.Sprintf("http://127.0.0.1:%d", proxy.Port), proxy, nil
+}
+
+func createEnvFile(stack *portainer.Stack) (string, error) {
+	if stack.Env == nil || len(stack.Env) == 0 {
+		return "", nil
+	}
+
+	envFilePath := path.Join(stack.ProjectPath, "stack.env")
+
+	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return "", err
+	}
+
+	for _, v := range stack.Env {
+		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
+	}
+	envfile.Close()
+
+	return envFilePath, nil
+}

api/exec/compose_stack_integration_test.go

@@ -33,7 +33,9 @@ func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
 		Name:        "project-name",
 	}
 
-	endpoint := &portainer.Endpoint{}
+	endpoint := &portainer.Endpoint{
+		URL: "unix://",
+	}
 
 	return stack, endpoint
 }
@@ -42,14 +44,17 @@ func Test_UpAndDown(t *testing.T) {
 
 	stack, endpoint := setup(t)
 
-	w := NewComposeWrapper("", nil)
+	w, err := NewComposeStackManager("", "", nil)
+	if err != nil {
+		t.Fatalf("Failed creating manager: %s", err)
+	}
 
-	err := w.Up(stack, endpoint)
+	err = w.Up(stack, endpoint)
 	if err != nil {
 		t.Fatalf("Error calling docker-compose up: %s", err)
 	}
 
-	if containerExists(composedContainerName) == false {
+	if !containerExists(composedContainerName) {
 		t.Fatal("container should exist")
 	}
 
@@ -63,13 +68,13 @@ func Test_UpAndDown(t *testing.T) {
 	}
 }
 
-func containerExists(contaierName string) bool {
-	cmd := exec.Command(osProgram("docker"), "ps", "-a", "-f", fmt.Sprintf("name=%s", contaierName))
+func containerExists(containerName string) bool {
+	cmd := exec.Command("docker", "ps", "-a", "-f", fmt.Sprintf("name=%s", containerName))
 
 	out, err := cmd.Output()
 	if err != nil {
 		log.Fatalf("failed to list containers: %s", err)
 	}
 
-	return strings.Contains(string(out), contaierName)
+	return strings.Contains(string(out), containerName)
 }

api/exec/compose_stack_test.go

@@ -0,0 +1,112 @@
+package exec
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_stackFilePath(t *testing.T) {
+	tests := []struct {
+		name     string
+		stack    *portainer.Stack
+		expected string
+	}{
+		// {
+		// 	name:     "should return empty result if stack is missing",
+		// 	stack:    nil,
+		// 	expected: "",
+		// },
+		// {
+		// 	name:     "should return empty result if stack don't have entrypoint",
+		// 	stack:    &portainer.Stack{},
+		// 	expected: "",
+		// },
+		{
+			name: "should allow file name and dir",
+			stack: &portainer.Stack{
+				ProjectPath: "dir",
+				EntryPoint:  "file",
+			},
+			expected: path.Join("dir", "file"),
+		},
+		{
+			name: "should allow file name only",
+			stack: &portainer.Stack{
+				EntryPoint: "file",
+			},
+			expected: "file",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := stackFilePath(tt.stack)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}
+
+func Test_createEnvFile(t *testing.T) {
+	dir := t.TempDir()
+
+	tests := []struct {
+		name         string
+		stack        *portainer.Stack
+		expected     string
+		expectedFile bool
+	}{
+		// {
+		// 	name:     "should not add env file option if stack is missing",
+		// 	stack:    nil,
+		// 	expected: "",
+		// },
+		{
+			name: "should not add env file option if stack doesn't have env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+			},
+			expected: "",
+		},
+		{
+			name: "should not add env file option if stack's env variables are empty",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env:         []portainer.Pair{},
+			},
+			expected: "",
+		},
+		{
+			name: "should add env file option if stack has env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env: []portainer.Pair{
+					{Name: "var1", Value: "value1"},
+					{Name: "var2", Value: "value2"},
+				},
+			},
+			expected: "var1=value1\nvar2=value2\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, _ := createEnvFile(tt.stack)
+
+			if tt.expected != "" {
+				assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result)
+
+				f, _ := os.Open(path.Join(dir, "stack.env"))
+				content, _ := ioutil.ReadAll(f)
+
+				assert.Equal(t, tt.expected, string(content))
+			} else {
+				assert.Equal(t, "", result)
+			}
+		})
+	}
+}

api/exec/compose_wrapper.go

@@ -1,137 +0,0 @@
-package exec
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy"
-)
-
-// ComposeWrapper is a wrapper for docker-compose binary
-type ComposeWrapper struct {
-	binaryPath   string
-	proxyManager *proxy.Manager
-}
-
-// NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeWrapper(binaryPath string, proxyManager *proxy.Manager) *ComposeWrapper {
-	if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
-		return nil
-	}
-
-	return &ComposeWrapper{
-		binaryPath:   binaryPath,
-		proxyManager: proxyManager,
-	}
-}
-
-// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
-func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
-	return portainer.ComposeSyntaxMaxVersion
-}
-
-// NormalizeStackName returns a new stack name with unsupported characters replaced
-func (w *ComposeWrapper) NormalizeStackName(name string) string {
-	return name
-}
-
-// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
-func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	_, err := w.command([]string{"up", "-d"}, stack, endpoint)
-	return err
-}
-
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
-func (w *ComposeWrapper) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	_, err := w.command([]string{"down", "--remove-orphans"}, stack, endpoint)
-	return err
-}
-
-func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpoint *portainer.Endpoint) ([]byte, error) {
-	if endpoint == nil {
-		return nil, errors.New("cannot call a compose command on an empty endpoint")
-	}
-
-	program := programPath(w.binaryPath, "docker-compose")
-
-	options := setComposeFile(stack)
-
-	options = addProjectNameOption(options, stack)
-	options, err := addEnvFileOption(options, stack)
-	if err != nil {
-		return nil, err
-	}
-
-	if !(endpoint.URL == "" || strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://")) {
-
-		proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
-		if err != nil {
-			return nil, err
-		}
-
-		defer proxy.Close()
-
-		options = append(options, "-H", fmt.Sprintf("http://127.0.0.1:%d", proxy.Port))
-	}
-
-	args := append(options, command...)
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(program, args...)
-	cmd.Stderr = &stderr
-
-	out, err := cmd.Output()
-	if err != nil {
-		return out, errors.New(stderr.String())
-	}
-
-	return out, nil
-}
-
-func setComposeFile(stack *portainer.Stack) []string {
-	options := make([]string, 0)
-
-	if stack == nil || stack.EntryPoint == "" {
-		return options
-	}
-
-	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
-	options = append(options, "-f", composeFilePath)
-	return options
-}
-
-func addProjectNameOption(options []string, stack *portainer.Stack) []string {
-	if stack == nil || stack.Name == "" {
-		return options
-	}
-
-	options = append(options, "-p", stack.Name)
-	return options
-}
-
-func addEnvFileOption(options []string, stack *portainer.Stack) ([]string, error) {
-	if stack == nil || stack.Env == nil || len(stack.Env) == 0 {
-		return options, nil
-	}
-
-	envFilePath := path.Join(stack.ProjectPath, "stack.env")
-
-	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return options, err
-	}
-
-	for _, v := range stack.Env {
-		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
-	}
-	envfile.Close()
-
-	options = append(options, "--env-file", envFilePath)
-	return options, nil
-}

api/exec/compose_wrapper_test.go

@@ -1,143 +0,0 @@
-package exec
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_setComposeFile(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected []string
-	}{
-		{
-			name:     "should return empty result if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should return empty result if stack don't have entrypoint",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should allow file name and dir",
-			stack: &portainer.Stack{
-				ProjectPath: "dir",
-				EntryPoint:  "file",
-			},
-			expected: []string{"-f", path.Join("dir", "file")},
-		},
-		{
-			name: "should allow file name only",
-			stack: &portainer.Stack{
-				EntryPoint: "file",
-			},
-			expected: []string{"-f", "file"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := setComposeFile(tt.stack)
-			assert.ElementsMatch(t, tt.expected, result)
-		})
-	}
-}
-
-func Test_addProjectNameOption(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected []string
-	}{
-		{
-			name:     "should not add project option if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should not add project option if stack doesn't have name",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should add project name option if stack has a name",
-			stack: &portainer.Stack{
-				Name: "project-name",
-			},
-			expected: []string{"-p", "project-name"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			options := []string{"-a", "b"}
-			result := addProjectNameOption(options, tt.stack)
-			assert.ElementsMatch(t, append(options, tt.expected...), result)
-		})
-	}
-}
-
-func Test_addEnvFileOption(t *testing.T) {
-	dir := t.TempDir()
-
-	tests := []struct {
-		name            string
-		stack           *portainer.Stack
-		expected        []string
-		expectedContent string
-	}{
-		{
-			name:     "should not add env file option if stack is missing",
-			stack:    nil,
-			expected: []string{},
-		},
-		{
-			name:     "should not add env file option if stack doesn't have env variables",
-			stack:    &portainer.Stack{},
-			expected: []string{},
-		},
-		{
-			name: "should not add env file option if stack's env variables are empty",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env:         []portainer.Pair{},
-			},
-			expected: []string{},
-		},
-		{
-			name: "should add env file option if stack has env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env: []portainer.Pair{
-					{Name: "var1", Value: "value1"},
-					{Name: "var2", Value: "value2"},
-				},
-			},
-			expected:        []string{"--env-file", path.Join(dir, "stack.env")},
-			expectedContent: "var1=value1\nvar2=value2\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			options := []string{"-a", "b"}
-			result, _ := addEnvFileOption(options, tt.stack)
-			assert.ElementsMatch(t, append(options, tt.expected...), result)
-
-			if tt.expectedContent != "" {
-				f, _ := os.Open(path.Join(dir, "stack.env"))
-				content, _ := ioutil.ReadAll(f)
-
-				assert.Equal(t, tt.expectedContent, string(content))
-			}
-		})
-	}
-}

api/exec/kubernetes_deploy.go

@@ -2,71 +2,188 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath string
+	binaryPath           string
+	dataStore            portainer.DataStore
+	reverseTunnelService portainer.ReverseTunnelService
+	signatureService     portainer.DigitalSignatureService
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath: binaryPath,
+		binaryPath:           binaryPath,
+		dataStore:            datastore,
+		reverseTunnelService: reverseTunnelService,
+		signatureService:     signatureService,
 	}
 }
 
 // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
-// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
 // Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
-	if composeFormat {
-		convertedData, err := deployer.convertComposeData(data)
+func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
+	if endpoint.Type == portainer.KubernetesLocalEnvironment {
+		token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		data = string(convertedData)
+
+		command := path.Join(deployer.binaryPath, "kubectl")
+		if runtime.GOOS == "windows" {
+			command = path.Join(deployer.binaryPath, "kubectl.exe")
+		}
+
+		args := make([]string, 0)
+		args = append(args, "--server", endpoint.URL)
+		args = append(args, "--insecure-skip-tls-verify")
+		args = append(args, "--token", string(token))
+		args = append(args, "--namespace", namespace)
+		args = append(args, "apply", "-f", "-")
+
+		var stderr bytes.Buffer
+		cmd := exec.Command(command, args...)
+		cmd.Stderr = &stderr
+		cmd.Stdin = strings.NewReader(stackConfig)
+
+		output, err := cmd.Output()
+		if err != nil {
+			return "", errors.New(stderr.String())
+		}
+
+		return string(output), nil
 	}
 
-	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+	// agent
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		if tunnel.Status == portainer.EdgeAgentIdle {
+
+			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+			if err != nil {
+				return "", err
+			}
+
+			settings, err := deployer.dataStore.Settings().Settings()
+			if err != nil {
+				return "", err
+			}
+
+			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
+			time.Sleep(waitForAgentToConnect * 2)
+		}
+
+		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+	}
+
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return "", err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	httpCli := &http.Client{
+		Transport: transport,
+	}
+
+	if !strings.HasPrefix(endpointURL, "http") {
+		endpointURL = fmt.Sprintf("https://%s", endpointURL)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	command := path.Join(deployer.binaryPath, "kubectl")
-	if runtime.GOOS == "windows" {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
-	}
-
-	args := make([]string, 0)
-	args = append(args, "--server", endpoint.URL)
-	args = append(args, "--insecure-skip-tls-verify")
-	args = append(args, "--token", string(token))
-	args = append(args, "--namespace", namespace)
-	args = append(args, "apply", "-f", "-")
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = &stderr
-	cmd.Stdin = strings.NewReader(data)
-
-	output, err := cmd.Output()
+	reqPayload, err := json.Marshal(
+		struct {
+			StackConfig string
+			Namespace   string
+		}{
+			StackConfig: stackConfig,
+			Namespace:   namespace,
+		})
 	if err != nil {
-		return nil, errors.New(stderr.String())
+		return "", err
 	}
 
-	return output, nil
+	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
+	if err != nil {
+		return "", err
+	}
+
+	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
+	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var errorResponseData struct {
+			Message string
+			Details string
+		}
+		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
+		if err != nil {
+			output, parseStringErr := ioutil.ReadAll(resp.Body)
+			if parseStringErr != nil {
+				return "", parseStringErr
+			}
+
+			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
+
+		}
+
+		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
+	}
+
+	var responseData struct{ Output string }
+	err = json.NewDecoder(resp.Body).Decode(&responseData)
+	if err != nil {
+		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
+		if parseStringErr != nil {
+			return "", parseStringErr
+		}
+
+		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
+	}
+
+	return responseData.Output, nil
+
 }
 
-func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
+// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
+func (deployer *KubernetesDeployer) ConvertCompose(data string) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")

api/exec/swarm_stack.go

@@ -8,9 +8,11 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"runtime"
+	"strings"
 
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -42,7 +44,7 @@ func NewSwarmStackManager(binaryPath, dataPath string, signatureService portaine
 }
 
 // Login executes the docker login command against a list of registries (including DockerHub).
-func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
+func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoint *portainer.Endpoint) {
 	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	for _, registry := range registries {
 		if registry.Authentication {
@@ -50,11 +52,6 @@ func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registri
 			runCommandAndCaptureStdErr(command, registryArgs, nil, "")
 		}
 	}
-
-	if dockerhub.Authentication {
-		dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password)
-		runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "")
-	}
 }
 
 // Logout executes the docker logout command.
@@ -189,3 +186,8 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 
 	return config, nil
 }
+
+func (manager *SwarmStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
+}

api/exec/utils.go

@@ -1,24 +0,0 @@
-package exec
-
-import (
-	"os/exec"
-	"path/filepath"
-	"runtime"
-)
-
-func osProgram(program string) string {
-	if runtime.GOOS == "windows" {
-		program += ".exe"
-	}
-	return program
-}
-
-func programPath(rootPath, program string) string {
-	return filepath.Join(rootPath, osProgram(program))
-}
-
-// IsBinaryPresent returns true if corresponding program exists on PATH
-func IsBinaryPresent(program string) bool {
-	_, err := exec.LookPath(program)
-	return err == nil
-}

api/exec/utils_test.go

@@ -1,16 +0,0 @@
-package exec
-
-import (
-	"testing"
-)
-
-func Test_isBinaryPresent(t *testing.T) {
-
-	if !IsBinaryPresent("docker") {
-		t.Error("expect docker binary to exist on the path")
-	}
-
-	if IsBinaryPresent("executable-with-this-name-should-not-exist") {
-		t.Error("expect binary with a random name to be missing on the path")
-	}
-}

api/filesystem/filesystem.go

@@ -31,6 +31,8 @@ const (
 	ComposeStorePath = "compose"
 	// ComposeFileDefaultName represents the default name of a compose file.
 	ComposeFileDefaultName = "docker-compose.yml"
+	// ManifestFileDefaultName represents the default name of a k8s manifest file.
+	ManifestFileDefaultName = "k8s-deployment.yml"
 	// EdgeStackStorePath represents the subfolder where edge stack files are stored in the file store folder.
 	EdgeStackStorePath = "edge_stacks"
 	// PrivateKeyFile represents the name on disk of the file containing the private key.
@@ -279,13 +281,7 @@ func (service *Service) WriteJSONToFile(path string, content interface{}) error
 
 // FileExists checks for the existence of the specified file.
 func (service *Service) FileExists(filePath string) (bool, error) {
-	if _, err := os.Stat(filePath); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
+	return FileExists(filePath)
 }
 
 // KeyPairFilesExist checks for the existence of the key files.
@@ -510,3 +506,31 @@ func (service *Service) GetTemporaryPath() (string, error) {
 func (service *Service) GetDatastorePath() string {
 	return service.dataStorePath
 }
+
+// FileExists checks for the existence of the specified file.
+func FileExists(filePath string) (bool, error) {
+	if _, err := os.Stat(filePath); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func MoveDirectory(originalPath, newPath string) error {
+	if _, err := os.Stat(originalPath); err != nil {
+		return err
+	}
+
+	alreadyExists, err := FileExists(newPath)
+	if err != nil {
+		return err
+	}
+
+	if alreadyExists {
+		return errors.New("Target path already exists")
+	}
+
+	return os.Rename(originalPath, newPath)
+}

api/filesystem/filesystem_fileexists_test.go

@@ -0,0 +1,55 @@
+package filesystem
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_fileSystemService_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
+	service := createService(t)
+	testHelperFileExists_fileExists(t, service.FileExists)
+}
+
+func Test_fileSystemService_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
+	service := createService(t)
+	testHelperFileExists_fileNotExists(t, service.FileExists)
+}
+
+func Test_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
+	testHelperFileExists_fileExists(t, FileExists)
+}
+
+func Test_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
+	testHelperFileExists_fileNotExists(t, FileExists)
+}
+
+func testHelperFileExists_fileExists(t *testing.T, checker func(path string) (bool, error)) {
+	file, err := os.CreateTemp("", t.Name())
+	assert.NoError(t, err, "CreateTemp should not fail")
+
+	t.Cleanup(func() {
+		os.RemoveAll(file.Name())
+	})
+
+	exists, err := checker(file.Name())
+	assert.NoError(t, err, "FileExists should not fail")
+
+	assert.True(t, exists)
+}
+
+func testHelperFileExists_fileNotExists(t *testing.T, checker func(path string) (bool, error)) {
+	filePath := path.Join(os.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int()))
+
+	err := os.RemoveAll(filePath)
+	assert.NoError(t, err, "RemoveAll should not fail")
+
+	exists, err := checker(filePath)
+	assert.NoError(t, err, "FileExists should not fail")
+
+	assert.False(t, exists)
+}

api/filesystem/filesystem_move_test.go

@@ -0,0 +1,49 @@
+package filesystem
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// temporary function until upgrade to 1.16
+func tempDir(t *testing.T) string {
+	tmpDir, err := os.MkdirTemp("", "dir")
+	assert.NoError(t, err, "MkdirTemp should not fail")
+
+	return tmpDir
+}
+
+func Test_movePath_shouldFailIfOriginalPathDoesntExist(t *testing.T) {
+	tmpDir := tempDir(t)
+	missingPath := path.Join(tmpDir, "missing")
+	targetPath := path.Join(tmpDir, "target")
+
+	defer os.RemoveAll(tmpDir)
+
+	err := MoveDirectory(missingPath, targetPath)
+	assert.Error(t, err, "move directory should fail when target path exists")
+}
+
+func Test_movePath_shouldFailIfTargetPathDoesExist(t *testing.T) {
+	originalPath := tempDir(t)
+	missingPath := tempDir(t)
+
+	defer os.RemoveAll(originalPath)
+	defer os.RemoveAll(missingPath)
+
+	err := MoveDirectory(originalPath, missingPath)
+	assert.Error(t, err, "move directory should fail when target path exists")
+}
+
+func Test_movePath_success(t *testing.T) {
+	originalPath := tempDir(t)
+
+	defer os.RemoveAll(originalPath)
+
+	err := MoveDirectory(originalPath, fmt.Sprintf("%s-old", originalPath))
+	assert.NoError(t, err)
+}

api/filesystem/filesystem_test.go

@@ -0,0 +1,22 @@
+package filesystem
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createService(t *testing.T) *Service {
+	dataStorePath := path.Join(os.TempDir(), t.Name())
+
+	service, err := NewService(dataStorePath, "")
+	assert.NoError(t, err, "NewService should not fail")
+
+	t.Cleanup(func() {
+		os.RemoveAll(dataStorePath)
+	})
+
+	return service
+}

api/git/azure.go

@@ -0,0 +1,219 @@
+package git
+
+import (
+	"context"
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+const (
+	azureDevOpsHost        = "dev.azure.com"
+	visualStudioHostSuffix = ".visualstudio.com"
+)
+
+func isAzureUrl(s string) bool {
+	return strings.Contains(s, azureDevOpsHost) ||
+		strings.Contains(s, visualStudioHostSuffix)
+}
+
+type azureOptions struct {
+	organisation, project, repository string
+	// a user may pass credentials in a repository URL,
+	// for example https://<username>:<password>@<domain>/<path>
+	username, password string
+}
+
+type azureDownloader struct {
+	client  *http.Client
+	baseUrl string
+}
+
+func NewAzureDownloader(client *http.Client) *azureDownloader {
+	return &azureDownloader{
+		client: client,
+		baseUrl: "https://dev.azure.com",
+	}
+}
+
+func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
+	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
+	if err != nil {
+		return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
+	}
+	defer os.Remove(zipFilepath)
+
+	err = archive.UnzipFile(zipFilepath, destination)
+	if err != nil {
+		return errors.Wrap(err, "failed to unzip file")
+	}
+
+	return nil
+}
+
+func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
+	config, err := parseUrl(options.repositoryUrl)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to parse url")
+	}
+	downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to build download url")
+	}
+	zipFile, err := ioutil.TempFile("", "azure-git-repo-*.zip")
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create temp file")
+	}
+	defer zipFile.Close()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
+	if options.username != "" || options.password != "" {
+		req.SetBasicAuth(options.username, options.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	res, err := a.client.Do(req)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to download zip with a status \"%v\"", res.Status)
+	}
+
+	_, err = io.Copy(zipFile, res.Body)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to save HTTP response to a file")
+	}
+	return zipFile.Name(), nil
+}
+
+func parseUrl(rawUrl string) (*azureOptions, error) {
+	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
+		return parseHttpUrl(rawUrl)
+	}
+	if strings.HasPrefix(rawUrl, "git@ssh") {
+		return parseSshUrl(rawUrl)
+	}
+	if strings.HasPrefix(rawUrl, "ssh://") {
+		r := []rune(rawUrl)
+		return parseSshUrl(string(r[6:])) // remove the prefix
+	}
+
+	return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl)
+}
+
+var expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
+
+func parseSshUrl(rawUrl string) (*azureOptions, error) {
+	path := strings.Split(rawUrl, "/")
+
+	unexpectedUrlErr := errors.Errorf("want url %s, got %s", expectedSshUrl, rawUrl)
+	if len(path) != 4 {
+		return nil, unexpectedUrlErr
+	}
+	return &azureOptions{
+		organisation: path[1],
+		project:      path[2],
+		repository:   path[3],
+	}, nil
+}
+
+const expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository"
+const expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository"
+
+func parseHttpUrl(rawUrl string) (*azureOptions, error) {
+	u, err := url.Parse(rawUrl)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse HTTP url")
+	}
+
+	opt := azureOptions{}
+	switch {
+	case u.Host == azureDevOpsHost:
+		path := strings.Split(u.Path, "/")
+		if len(path) != 5 {
+			return nil, errors.Errorf("want url %s, got %s", expectedAzureDevOpsHttpUrl, u)
+		}
+		opt.organisation = path[1]
+		opt.project = path[2]
+		opt.repository = path[4]
+	case strings.HasSuffix(u.Host, visualStudioHostSuffix):
+		path := strings.Split(u.Path, "/")
+		if len(path) != 4 {
+			return nil, errors.Errorf("want url %s, got %s", expectedVisualStudioHttpUrl, u)
+		}
+		opt.organisation = strings.TrimSuffix(u.Host, visualStudioHostSuffix)
+		opt.project = path[1]
+		opt.repository = path[3]
+	default:
+		return nil, errors.Errorf("unknown azure host in url \"%s\"", rawUrl)
+	}
+
+	opt.username = u.User.Username()
+	opt.password, _ = u.User.Password()
+
+	return &opt, nil
+}
+
+func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl)
+	}
+	q := u.Query()
+	// scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0
+	q.Set("scopePath", "/")
+	q.Set("download", "true")
+	q.Set("versionDescriptor.versionType", getVersionType(referenceName))
+	q.Set("versionDescriptor.version", formatReferenceName(referenceName))
+	q.Set("$format", "zip")
+	q.Set("recursionLevel", "full")
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
+const (
+	branchPrefix = "refs/heads/"
+	tagPrefix    = "refs/tags/"
+)
+
+func formatReferenceName(name string) string {
+	if strings.HasPrefix(name, branchPrefix) {
+		return strings.TrimPrefix(name, branchPrefix)
+	}
+	if strings.HasPrefix(name, tagPrefix) {
+		return strings.TrimPrefix(name, tagPrefix)
+	}
+	return name
+}
+
+func getVersionType(name string) string {
+	if strings.HasPrefix(name, branchPrefix) {
+		return "branch"
+	}
+	if strings.HasPrefix(name, tagPrefix) {
+		return "tag"
+	}
+	return "commit"
+}

api/git/azure_integration_test.go

@@ -0,0 +1,93 @@
+package git
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestService_ClonePublicRepository_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	type args struct {
+		repositoryURLFormat string
+		referenceName       string
+		username            string
+		password            string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Clone Azure DevOps repo branch",
+			args: args{
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				referenceName:       "refs/heads/main",
+				username:            "",
+				password:            pat,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Clone Azure DevOps repo tag",
+			args: args{
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				referenceName:       "refs/tags/v1.1",
+				username:            "",
+				password:            pat,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dst, err := ioutils.TempDir("", "clone")
+			assert.NoError(t, err)
+			defer os.RemoveAll(dst)
+			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
+			err = service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "")
+			assert.NoError(t, err)
+			assert.FileExists(t, filepath.Join(dst, "README.md"))
+		})
+	}
+}
+
+func TestService_ClonePrivateRepository_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	dst, err := ioutils.TempDir("", "clone")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dst)
+
+	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat)
+	assert.NoError(t, err)
+	assert.FileExists(t, filepath.Join(dst, "README.md"))
+}
+
+func getRequiredValue(t *testing.T, name string) string {
+	value, ok := os.LookupEnv(name)
+	if !ok {
+		t.Fatalf("can't find required env var \"%s\"", name)
+	}
+	return value
+}
+
+func ensureIntegrationTest(t *testing.T) {
+	if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
+		t.Skip("skip an integration test")
+	}
+}

api/git/azure_test.go

@@ -0,0 +1,250 @@
+package git
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func Test_buildDownloadUrl(t *testing.T) {
+	a := NewAzureDownloader(nil)
+	u, err := a.buildDownloadUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "refs/heads/main")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0&versionDescriptor.versionType=branch")
+	actualUrl, _ := url.Parse(u)
+	if assert.NoError(t, err) {
+		assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+		assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+		assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+		assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+	}
+}
+
+func Test_parseAzureUrl(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *azureOptions
+		wantErr bool
+	}{
+		{
+			name: "Expected SSH URL format starting with ssh://",
+			args: args{
+				url: "ssh://git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Expected SSH URL format starting with git@ssh",
+			args: args{
+				url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Unexpected SSH URL format",
+			args: args{
+				url: "git@ssh.dev.azure.com:v3/Organisation/Repository",
+			},
+			wantErr: true,
+		},
+		{
+			name: "Expected HTTPS URL format",
+			args: args{
+				url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				username:     "Organisation",
+			},
+			wantErr: false,
+		},
+		{
+			name: "HTTPS URL with credentials",
+			args: args{
+				url: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				username:     "username",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "HTTPS URL with password",
+			args: args{
+				url: "https://:password@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: &azureOptions{
+				organisation: "Organisation",
+				project:      "Project",
+				repository:   "Repository",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Visual Studio HTTPS URL with credentials",
+			args: args{
+				url: "https://username:password@organisation.visualstudio.com/project/_git/repository",
+			},
+			want: &azureOptions{
+				organisation: "organisation",
+				project:      "project",
+				repository:   "repository",
+				username:     "username",
+				password:     "password",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Unexpected HTTPS URL format",
+			args: args{
+				url: "https://Organisation@dev.azure.com/Project/_git/Repository",
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUrl(tt.args.url)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseUrl() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func Test_isAzureUrl(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Is Azure url",
+			args: args{
+				s: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			},
+			want: true,
+		},
+		{
+			name: "Is Azure url",
+			args: args{
+				s: "https://portainer.visualstudio.com/project/_git/repository",
+			},
+			want: true,
+		},
+		{
+			name: "Is NOT Azure url",
+			args: args{
+				s: "https://github.com/Organisation/Repository",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, isAzureUrl(tt.args.s))
+		})
+	}
+}
+
+func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
+	type args struct {
+		options cloneOptions
+	}
+	type basicAuth struct {
+		username, password string
+	}
+	tests := []struct {
+		name string
+		args args
+		want *basicAuth
+	}{
+		{
+			name: "username, password embedded",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+				},
+			},
+			want: &basicAuth{
+				username: "username",
+				password: "password",
+			},
+		},
+		{
+			name: "username, password embedded, clone options take precedence",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
+					username:      "u",
+					password:      "p",
+				},
+			},
+			want: &basicAuth{
+				username: "u",
+				password: "p",
+			},
+		},
+		{
+			name: "no credentials",
+			args: args{
+				options: cloneOptions{
+					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var zipRequestAuth *basicAuth
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if username, password, ok := r.BasicAuth(); ok {
+					zipRequestAuth = &basicAuth{username, password}
+				}
+				w.WriteHeader(http.StatusNotFound) // this makes function under test to return an error
+			}))
+			defer server.Close()
+
+			a := &azureDownloader{
+				client:  server.Client(),
+				baseUrl: server.URL,
+			}
+			_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
+			assert.Error(t, err)
+			assert.Equal(t, tt.want, zipRequestAuth)
+		})
+	}
+}

api/git/git.go

@@ -1,21 +1,72 @@
 package git
 
 import (
+	"context"
 	"crypto/tls"
 	"net/http"
-	"net/url"
-	"strings"
+	"os"
+	"path/filepath"
 	"time"
 
-	"gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
-	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+	"github.com/pkg/errors"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
+type cloneOptions struct {
+	repositoryUrl string
+	username      string
+	password      string
+	referenceName string
+	depth         int
+}
+
+type downloader interface {
+	download(ctx context.Context, dst string, opt cloneOptions) error
+}
+
+type gitClient struct {
+	preserveGitDirectory bool
+}
+
+func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
+	gitOptions := git.CloneOptions{
+		URL:   opt.repositoryUrl,
+		Depth: opt.depth,
+	}
+
+	if opt.password != "" || opt.username != "" {
+		gitOptions.Auth = &githttp.BasicAuth{
+			Username: opt.username,
+			Password: opt.password,
+		}
+	}
+
+	if opt.referenceName != "" {
+		gitOptions.ReferenceName = plumbing.ReferenceName(opt.referenceName)
+	}
+
+	_, err := git.PlainCloneContext(ctx, dst, false, &gitOptions)
+
+	if err != nil {
+		return errors.Wrap(err, "failed to clone git repository")
+	}
+
+	if !c.preserveGitDirectory {
+		os.RemoveAll(filepath.Join(dst, ".git"))
+	}
+
+	return nil
+}
+
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
+	azure    downloader
+	git      downloader
 }
 
 // NewService initializes a new service.
@@ -31,32 +82,29 @@ func NewService() *Service {
 
 	return &Service{
 		httpsCli: httpsCli,
+		azure:    NewAzureDownloader(httpsCli),
+		git:      gitClient{},
 	}
 }
 
-// ClonePublicRepository clones a public git repository using the specified URL in the specified
+// CloneRepository clones a git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
-	return cloneRepository(repositoryURL, referenceName, destination)
-}
-
-// ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
-// destination folder. It will use the specified username and password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
-	credentials := username + ":" + url.PathEscape(password)
-	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
-	return cloneRepository(repositoryURL, referenceName, destination)
-}
-
-func cloneRepository(repositoryURL, referenceName, destination string) error {
-	options := &git.CloneOptions{
-		URL: repositoryURL,
+func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	options := cloneOptions{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+		referenceName: referenceName,
+		depth:         1,
 	}
 
-	if referenceName != "" {
-		options.ReferenceName = plumbing.ReferenceName(referenceName)
+	return service.cloneRepository(destination, options)
+}
+
+func (service *Service) cloneRepository(destination string, options cloneOptions) error {
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.download(context.TODO(), destination, options)
 	}
 
-	_, err := git.PlainClone(destination, false, options)
-	return err
+	return service.git.download(context.TODO(), destination, options)
 }

api/git/git_integration_test.go

@@ -0,0 +1,27 @@
+package git
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService()
+
+	dst, err := ioutils.TempDir("", "clone")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dst)
+
+	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat)
+	assert.NoError(t, err)
+	assert.FileExists(t, filepath.Join(dst, "README.md"))
+}

api/git/git_test.go

@@ -0,0 +1,176 @@
+package git
+
+import (
+	"context"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/stretchr/testify/assert"
+)
+
+var bareRepoDir string
+
+func TestMain(m *testing.M) {
+	if err := testMain(m); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// testMain does extra setup/teardown before/after testing.
+// The function is separated from TestMain due to necessity to call os.Exit/log.Fatal in the latter.
+func testMain(m *testing.M) error {
+	dir, err := ioutil.TempDir("", "git-repo-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+
+	bareRepoDir = filepath.Join(dir, "test-clone.git")
+
+	file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0755)
+	if err != nil {
+		return errors.Wrap(err, "failed to open an archive")
+	}
+	err = archive.ExtractTarGz(file, dir)
+	if err != nil {
+		return errors.Wrapf(err, "failed to extract file from the archive to a folder %s\n", dir)
+	}
+
+	m.Run()
+
+	return nil
+}
+
+func Test_ClonePublicRepository_Shallow(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+	t.Logf("Cloning into %s", dir)
+	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+
+	defer os.RemoveAll(dir)
+
+	t.Logf("Cloning into %s", dir)
+	err = service.CloneRepository(dir, repositoryURL, referenceName, "", "")
+	assert.NoError(t, err)
+	assert.NoDirExists(t, filepath.Join(dir, ".git"))
+}
+
+func Test_cloneRepository(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+	destination := "shallow"
+
+	dir, err := ioutil.TempDir("", destination)
+	if err != nil {
+		t.Fatalf("failed to create a temp dir")
+	}
+	defer os.RemoveAll(dir)
+	t.Logf("Cloning into %s", dir)
+
+	err = service.cloneRepository(dir, cloneOptions{
+		repositoryUrl: repositoryURL,
+		referenceName: referenceName,
+		depth:         10,
+	})
+
+	assert.NoError(t, err)
+	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func getCommitHistoryLength(t *testing.T, err error, dir string) int {
+	repo, err := git.PlainOpen(dir)
+	if err != nil {
+		t.Fatalf("can't open a git repo at %s with error %v", dir, err)
+	}
+	iter, err := repo.Log(&git.LogOptions{All: true})
+	if err != nil {
+		t.Fatalf("can't get a commit history iterator with error %v", err)
+	}
+	count := 0
+	err = iter.ForEach(func(_ *object.Commit) error {
+		count++
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("can't iterate over the commit history with error %v", err)
+	}
+	return count
+}
+
+type testDownloader struct {
+	called bool
+}
+
+func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
+	t.called = true
+	return nil
+}
+
+func Test_cloneRepository_azure(t *testing.T) {
+	tests := []struct {
+		name   string
+		url    string
+		called bool
+	}{
+		{
+			name:   "Azure HTTP URL",
+			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			called: true,
+		},
+		{
+			name:   "Azure SSH URL",
+			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			called: true,
+		},
+		{
+			name:   "Something else",
+			url:    "https://example.com",
+			called: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azure := &testDownloader{}
+			git := &testDownloader{}
+
+			s := &Service{azure: azure, git: git}
+			s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
+
+			// if azure API is called, git isn't and vice versa
+			assert.Equal(t, tt.called, azure.called)
+			assert.Equal(t, tt.called, !git.called)
+		})
+	}
+}

api/git/types/types.go

@@ -0,0 +1,10 @@
+package gittypes
+
+type RepoConfig struct {
+	// The repo url
+	URL string `example:"https://github.com/portainer/portainer-ee.git"`
+	// The reference name
+	ReferenceName string `example:"refs/heads/branch_name"`
+	// Path to where the config file is in this url/refName
+	ConfigFilePath string `example:"docker-compose.yml"`
+}

api/go.mod

@@ -1,9 +1,9 @@
 module github.com/portainer/portainer/api
 
-go 1.13
+go 1.16
 
 require (
-	github.com/Microsoft/go-winio v0.4.14
+	github.com/Microsoft/go-winio v0.4.16
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/boltdb/bolt v1.3.1
 	github.com/containerd/containerd v1.3.1 // indirect
@@ -13,12 +13,13 @@ require (
 	github.com/docker/cli v0.0.0-20191126203649-54d085b857e9
 	github.com/docker/docker v0.0.0-00010101000000-000000000000
 	github.com/g07cha/defender v0.0.0-20180505193036-5665c627c814
+	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-ldap/ldap/v3 v3.1.8
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/websocket v1.4.1
-	github.com/imdario/mergo v0.3.8 // indirect
+	github.com/joho/godotenv v1.3.0
 	github.com/jpillora/chisel v0.0.0-20190724232113-f3a8df20e389
 	github.com/json-iterator/go v1.1.8
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
@@ -26,18 +27,20 @@ require (
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
+	github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92
 	github.com/portainer/libcompose v0.5.3
 	github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
-	github.com/stretchr/testify v1.6.1
-	golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1
-	golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/src-d/go-git.v4 v4.13.1
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	k8s.io/api v0.17.2
 	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v0.17.2
 )
 
 replace github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
+
+replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456

api/go.sum

@@ -11,10 +11,10 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Microsoft/go-winio v0.3.8 h1:dvxbxtpTIjdAbx2OtL26p4eq0iEvys/U5yrsTJb3NZI=
 github.com/Microsoft/go-winio v0.3.8/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
-github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -47,7 +47,7 @@ github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVl
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -80,6 +80,7 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -92,6 +93,15 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck=
 github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.1.0 h1:4pl5BV4o7ZG/lterP4S6WzJ6xr49Ba5ET9ygheTYahk=
+github.com/go-git/go-billy/v5 v5.1.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
+github.com/go-git/go-git/v5 v5.3.0 h1:8WKMtJR2j8RntEXR/uvTKagfEt4GYlwQ7mntE4+0GWc=
+github.com/go-git/go-git/v5 v5.3.0/go.mod h1:xdX4bWJ48aOrdhnl2XqHYstHbbp6+LFS4r4X+lNVprw=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
 github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
@@ -105,7 +115,6 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -145,13 +154,16 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669 h1:l5rH/CnVVu+HPxjtxjM90nHrm4nov3j3RF9/62UjgLs=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669/go.mod h1:kOeLNvjNBGSV3uYtFjvb72+fnZCMFJF1XDvRIjdom0g=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
@@ -168,8 +180,8 @@ github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46O
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
-github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c h1:N7A4JCA2G+j5fuFxCsJqjFU/sZe0mj8H0sSoSwbaikw=
@@ -177,13 +189,14 @@ github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c/go.mod h1:Nn
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v0.0.0-20150511174710-5cf931ef8f76/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
@@ -205,10 +218,13 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420 h1:Yu3681ykYHDfLoI6XVjL4JWmkE+3TX9yfIWwRCh1kFM=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -218,16 +234,16 @@ github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c h1:iOMba/KmaXg
 github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
-github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM=
 github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8=
 github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8=
 github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yHr4rtnirg0W0Cjvv6/DzxBIZk5sV59208=
@@ -248,9 +264,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@@ -258,24 +273,19 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
-github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
 github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
-github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
-github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
+github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@@ -288,10 +298,9 @@ golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 h1:anGSYQpPhQwXlwsu5wmfq0nWkCNaMEMUwAv13Y92hd8=
-golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -308,12 +317,10 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk=
-golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -324,27 +331,16 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181019160139-8e24a49d80f8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
-golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -356,13 +352,11 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
@@ -373,25 +367,24 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
-gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
-gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
-gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
-gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
-gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
@@ -410,6 +403,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=

api/http/handler/auth/authenticate.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -130,19 +131,38 @@ func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, use
 }
 
 func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
-	tokenData := &portainer.TokenData{
-		ID:       user.ID,
-		Username: user.Username,
-		Role:     user.Role,
+
+	tokenData := composeTokenData(user)
+
+	user.HasAuthenticatedOnce = true
+	err := handler.DataStore.User().UpdateUser(user.ID, user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
 	}
 
 	return handler.persistAndWriteToken(w, tokenData)
 }
 
+func (handler *Handler) writeTokenForOAuth(w http.ResponseWriter, user *portainer.User, expiryTime *time.Time) *httperror.HandlerError {
+	token, err := handler.JWTService.GenerateTokenForOAuth(composeTokenData(user), expiryTime)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
+	}
+
+	user.HasAuthenticatedOnce = true
+
+	err = handler.DataStore.User().UpdateUser(user.ID, user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+	}
+
+	return response.JSON(w, &authenticateResponse{JWT: token})
+}
+
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
 	}
 
 	return response.JSON(w, &authenticateResponse{JWT: token})
@@ -204,3 +224,12 @@ func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamM
 	}
 	return false
 }
+
+func composeTokenData(user *portainer.User) *portainer.TokenData {
+	return &portainer.TokenData{
+		ID:                   user.ID,
+		Username:             user.Username,
+		Role:                 user.Role,
+		HasAuthenticatedOnce: user.HasAuthenticatedOnce,
+	}
+}

api/http/handler/auth/authenticate_oauth.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"net/http"
+	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -25,7 +26,24 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id AuthenticateOauth
+func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, *time.Time, error) {
+	if code == "" {
+		return "", nil, errors.New("Invalid OAuth authorization code")
+	}
+
+	if settings == nil {
+		return "", nil, errors.New("Invalid OAuth configuration")
+	}
+
+	username, expiryTime, err := handler.OAuthService.Authenticate(code, settings)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return username, expiryTime, nil
+}
+
+// @id ValidateOAuth
 // @summary Authenticate with OAuth
 // @tags auth
 // @accept json
@@ -36,52 +54,35 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 // @failure 422 "Invalid Credentials"
 // @failure 500 "Server error"
 // @router /auth/oauth/validate [post]
-func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, error) {
-	if code == "" {
-		return "", errors.New("Invalid OAuth authorization code")
-	}
-
-	if settings == nil {
-		return "", errors.New("Invalid OAuth configuration")
-	}
-
-	username, err := handler.OAuthService.Authenticate(code, settings)
-	if err != nil {
-		return "", err
-	}
-
-	return username, nil
-}
-
 func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	var payload oauthPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve settings from the database", Err: err}
 	}
 
-	if settings.AuthenticationMethod != 3 {
-		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", errors.New("OAuth authentication is not enabled")}
+	if settings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "OAuth authentication is not enabled", Err: errors.New("OAuth authentication is not enabled")}
 	}
 
-	username, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
+	username, expiryTime, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
 	if err != nil {
 		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to authenticate through OAuth", Err: httperrors.ErrUnauthorized}
 	}
 
 	user, err := handler.DataStore.User().UserByUsername(username)
 	if err != nil && err != bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a user with the specified username from the database", Err: err}
 	}
 
 	if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
-		return &httperror.HandlerError{http.StatusForbidden, "Account not created beforehand in Portainer and automatic user provisioning not enabled", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Account not created beforehand in Portainer and automatic user provisioning not enabled", Err: httperrors.ErrUnauthorized}
 	}
 
 	if user == nil {
@@ -92,7 +93,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 		err = handler.DataStore.User().CreateUser(user)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist user inside the database", Err: err}
 		}
 
 		if settings.OAuthSettings.DefaultTeamID != 0 {
@@ -104,11 +105,11 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 			err = handler.DataStore.TeamMembership().CreateTeamMembership(membership)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
+				return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist team membership inside the database", Err: err}
 			}
 		}
 
 	}
 
-	return handler.writeToken(w, user)
+	return handler.writeTokenForOAuth(w, user, expiryTime)
 }

api/http/handler/customtemplates/customtemplate_create.go

@@ -236,16 +236,14 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 	projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
 	customTemplate.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
 		return nil, err
 	}

api/http/handler/customtemplates/git.go

@@ -1,17 +0,0 @@
-package customtemplates
-
-type cloneRepositoryParameters struct {
-	url            string
-	referenceName  string
-	path           string
-	authentication bool
-	username       string
-	password       string
-}
-
-func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error {
-	if parameters.authentication {
-		return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password)
-	}
-	return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path)
-}

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -1,28 +0,0 @@
-package dockerhub
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-)
-
-// @id DockerHubInspect
-// @summary Retrieve DockerHub information
-// @description Use this endpoint to retrieve the information used to connect to the DockerHub
-// @description **Access policy**: authenticated
-// @tags dockerhub
-// @security jwt
-// @produce json
-// @success 200 {object} portainer.DockerHub
-// @failure 500 "Server error"
-// @router /dockerhub [get]
-func (handler *Handler) dockerhubInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
-	}
-
-	hideFields(dockerhub)
-	return response.JSON(w, dockerhub)
-}

api/http/handler/dockerhub/dockerhub_update.go

@@ -1,68 +0,0 @@
-package dockerhub
-
-import (
-	"errors"
-	"net/http"
-
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-)
-
-type dockerhubUpdatePayload struct {
-	// Enable authentication against DockerHub
-	Authentication bool `validate:"required" example:"false"`
-	// Username used to authenticate against the DockerHub
-	Username string `validate:"required" example:"hub_user"`
-	// Password used to authenticate against the DockerHub
-	Password string `validate:"required" example:"hub_password"`
-}
-
-func (payload *dockerhubUpdatePayload) Validate(r *http.Request) error {
-	if payload.Authentication && (govalidator.IsNull(payload.Username) || govalidator.IsNull(payload.Password)) {
-		return errors.New("Invalid credentials. Username and password must be specified when authentication is enabled")
-	}
-	return nil
-}
-
-// @id DockerHubUpdate
-// @summary Update DockerHub information
-// @description Use this endpoint to update the information used to connect to the DockerHub
-// @description **Access policy**: administrator
-// @tags dockerhub
-// @security jwt
-// @accept json
-// @produce json
-// @param body body dockerhubUpdatePayload true "DockerHub information"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @router /dockerhub [put]
-func (handler *Handler) dockerhubUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload dockerhubUpdatePayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
-	}
-
-	dockerhub := &portainer.DockerHub{
-		Authentication: false,
-		Username:       "",
-		Password:       "",
-	}
-
-	if payload.Authentication {
-		dockerhub.Authentication = true
-		dockerhub.Username = payload.Username
-		dockerhub.Password = payload.Password
-	}
-
-	err = handler.DataStore.DockerHub().UpdateDockerHub(dockerhub)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the Dockerhub changes inside the database", err}
-	}
-
-	return response.Empty(w)
-}

api/http/handler/dockerhub/handler.go

@@ -1,33 +0,0 @@
-package dockerhub
-
-import (
-	"net/http"
-
-	"github.com/gorilla/mux"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-func hideFields(dockerHub *portainer.DockerHub) {
-	dockerHub.Password = ""
-}
-
-// Handler is the HTTP handler used to handle DockerHub operations.
-type Handler struct {
-	*mux.Router
-	DataStore portainer.DataStore
-}
-
-// NewHandler creates a handler to manage Dockerhub operations.
-func NewHandler(bouncer *security.RequestBouncer) *Handler {
-	h := &Handler{
-		Router: mux.NewRouter(),
-	}
-	h.Handle("/dockerhub",
-		bouncer.RestrictedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
-	h.Handle("/dockerhub",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
-
-	return h
-}

api/http/handler/edgestacks/edgestack_create.go

@@ -212,16 +212,14 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 	projectPath := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
 		return nil, err
 	}

api/http/handler/edgestacks/git.go

@@ -1,17 +0,0 @@
-package edgestacks
-
-type cloneRepositoryParameters struct {
-	url            string
-	referenceName  string
-	path           string
-	authentication bool
-	username       string
-	password       string
-}
-
-func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error {
-	if parameters.authentication {
-		return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password)
-	}
-	return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path)
-}

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -109,12 +109,33 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	updateAuthorizations := false
 	if payload.UserAccessPolicies != nil && !reflect.DeepEqual(payload.UserAccessPolicies, endpointGroup.UserAccessPolicies) {
 		endpointGroup.UserAccessPolicies = payload.UserAccessPolicies
+		updateAuthorizations = true
 	}
 
 	if payload.TeamAccessPolicies != nil && !reflect.DeepEqual(payload.TeamAccessPolicies, endpointGroup.TeamAccessPolicies) {
 		endpointGroup.TeamAccessPolicies = payload.TeamAccessPolicies
+		updateAuthorizations = true
+	}
+
+	if updateAuthorizations {
+		endpoints, err := handler.DataStore.Endpoint().Endpoints()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+		}
+
+		for _, endpoint := range endpoints {
+			if endpoint.GroupID == endpointGroup.ID {
+				if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+					err = handler.AuthorizationService.CleanNAPWithOverridePolicies(&endpoint, endpointGroup)
+					if err != nil {
+						return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update user authorizations", err}
+					}
+				}
+			}
+		}
 	}
 
 	err = handler.DataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, endpointGroup)

api/http/handler/endpointgroups/handler.go

@@ -1,6 +1,7 @@
 package endpointgroups
 
 import (
+	"github.com/portainer/portainer/api/internal/authorization"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -12,6 +13,7 @@ import (
 // Handler is the HTTP handler used to handle endpoint group operations.
 type Handler struct {
 	*mux.Router
+	AuthorizationService *authorization.Service
 	DataStore portainer.DataStore
 }
 

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -77,5 +77,5 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 }
 
 func isKubernetesRequest(requestURL string) bool {
-	return strings.HasPrefix(requestURL, "/api")
+	return strings.HasPrefix(requestURL, "/api") || strings.HasPrefix(requestURL, "/healthz")
 }

api/http/handler/endpoints/endpoint_create.go

@@ -156,7 +156,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @accept multipart/form-data
 // @produce json
 // @param Name formData string true "Name that will be used to identify this endpoint (example: my-endpoint)"
-// @param EndpointType formData integer true "Environment type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5)
+// @param EndpointCreationType formData integer true "Environment type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5)
 // @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine)"
 // @param PublicURL formData string false "URL or IP address where exposed containers will be reachable. Defaults to URL if not specified (example: docker.mydomain.tld:2375)"
 // @param GroupID formData int false "Endpoint group identifier. If not specified will default to 1 (unassigned)."
@@ -322,8 +322,8 @@ func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload)
 		TLSConfig: portainer.TLSConfiguration{
 			TLS: false,
 		},
-		AuthorizedUsers:     []portainer.UserID{},
-		AuthorizedTeams:     []portainer.TeamID{},
+		UserAccessPolicies:  portainer.UserAccessPolicies{},
+		TeamAccessPolicies:  portainer.TeamAccessPolicies{},
 		Extensions:          []portainer.EndpointExtension{},
 		TagIDs:              payload.TagIDs,
 		Status:              portainer.EndpointStatusUp,
@@ -471,6 +471,7 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.
 		AllowVolumeBrowserForRegularUsers: false,
 		EnableHostManagementFeatures:      false,
 
+		AllowSysctlSettingForRegularUsers:         true,
 		AllowBindMountsForRegularUsers:            true,
 		AllowPrivilegedModeForRegularUsers:        true,
 		AllowHostNamespaceForRegularUsers:         true,

api/http/handler/endpoints/endpoint_delete.go

@@ -103,6 +103,22 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
+	registries, err := handler.DataStore.Registry().Registries()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
+	}
+
+	for idx := range registries {
+		registry := &registries[idx]
+		if _, ok := registry.RegistryAccesses[endpoint.ID]; ok {
+			delete(registry.RegistryAccesses, endpoint.ID)
+			err = handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
+			if err != nil {
+				return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update registry accesses", Err: err}
+			}
+		}
+	}
+
 	return response.Empty(w)
 }
 

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -22,7 +22,7 @@ type dockerhubStatusResponse struct {
 	Limit     int `json:"limit"`
 }
 
-// GET request on /api/endpoints/{id}/dockerhub/status
+// GET request on /api/endpoints/{id}/dockerhub/{registryId}
 func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
@@ -40,13 +40,30 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")}
 	}
 
-	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
+	registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
+	}
+
+	var registry *portainer.Registry
+
+	if registryID == 0 {
+		registry = &portainer.Registry{}
+	} else {
+		registry, err = handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
+		if err == bolterrors.ErrObjectNotFound {
+			return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
+		} else if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
+		}
+
+		if registry.Type != portainer.DockerHubRegistry {
+			return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry type", errors.New("Invalid registry type")}
+		}
 	}
 
 	httpClient := client.NewHTTPClient()
-	token, err := getDockerHubToken(httpClient, dockerhub)
+	token, err := getDockerHubToken(httpClient, registry)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err}
 	}
@@ -59,7 +76,7 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
 	return response.JSON(w, resp)
 }
 
-func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) {
+func getDockerHubToken(httpClient *client.HTTPClient, registry *portainer.Registry) (string, error) {
 	type dockerhubTokenResponse struct {
 		Token string `json:"token"`
 	}
@@ -71,8 +88,8 @@ func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.Docke
 		return "", err
 	}
 
-	if dockerhub.Authentication {
-		req.SetBasicAuth(dockerhub.Username, dockerhub.Password)
+	if registry.Authentication {
+		req.SetBasicAuth(registry.Username, registry.Password)
 	}
 
 	resp, err := httpClient.Do(req)
@@ -115,8 +132,18 @@ func getDockerHubLimits(httpClient *client.HTTPClient, token string) (*dockerhub
 		return nil, errors.New("failed fetching dockerhub limits")
 	}
 
+	// An error with rateLimit-Limit or RateLimit-Remaining is likely for dockerhub pro accounts where there is no rate limit.
+	// In that specific case the headers will not be present.  Don't bubble up the error as its normal
+	// See: https://docs.docker.com/docker-hub/download-rate-limit/
 	rateLimit, err := parseRateLimitHeader(resp.Header, "RateLimit-Limit")
+	if err != nil {
+		return nil, nil
+	}
+
 	rateLimitRemaining, err := parseRateLimitHeader(resp.Header, "RateLimit-Remaining")
+	if err != nil {
+		return nil, nil
+	}
 
 	return &dockerhubStatusResponse{
 		Limit:     rateLimit,

api/http/handler/endpoints/endpoint_list.go

@@ -26,7 +26,7 @@ import (
 // @param search query string false "Search query"
 // @param groupId query int false "List endpoints of this group"
 // @param limit query int false "Limit results to this value"
-// @param type query int false "List endpoints of this type"
+// @param types query []int false "List endpoints of this type"
 // @param tagIds query []int false "search endpoints with these tags (depends on tagsPartialMatch)"
 // @param tagsPartialMatch query bool false "If true, will return endpoint which has one of tagIds, if false (or missing) will return only endpoints that has all the tags"
 // @param endpointIds query []int false "will return only these endpoints"
@@ -46,7 +46,9 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	groupID, _ := request.RetrieveNumericQueryParameter(r, "groupId", true)
 	limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true)
-	endpointType, _ := request.RetrieveNumericQueryParameter(r, "type", true)
+
+	var endpointTypes []int
+	request.RetrieveJSONQueryParameter(r, "types", &endpointTypes, true)
 
 	var tagIDs []portainer.TagID
 	request.RetrieveJSONQueryParameter(r, "tagIds", &tagIDs, true)
@@ -98,8 +100,8 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 		filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, endpointGroups, tagsMap, search)
 	}
 
-	if endpointType != 0 {
-		filteredEndpoints = filterEndpointsByType(filteredEndpoints, portainer.EndpointType(endpointType))
+	if endpointTypes != nil {
+		filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, endpointTypes)
 	}
 
 	if tagIDs != nil {
@@ -212,11 +214,16 @@ func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGrou
 	return false
 }
 
-func filterEndpointsByType(endpoints []portainer.Endpoint, endpointType portainer.EndpointType) []portainer.Endpoint {
+func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []int) []portainer.Endpoint {
 	filteredEndpoints := make([]portainer.Endpoint, 0)
 
+	typeSet := map[portainer.EndpointType]bool{}
+	for _, endpointType := range endpointTypes {
+		typeSet[portainer.EndpointType(endpointType)] = true
+	}
+
 	for _, endpoint := range endpoints {
-		if endpoint.Type == endpointType {
+		if typeSet[endpoint.Type] {
 			filteredEndpoints = append(filteredEndpoints, endpoint)
 		}
 	}

api/http/handler/endpoints/endpoint_registries_inspect.go

@@ -0,0 +1,50 @@
+package endpoints
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// GET request on /endpoints/{id}/registries/{registryId}
+func (handler *Handler) endpointRegistryInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
+	}
+
+	registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid registry identifier route variable", Err: err}
+	}
+
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a registry with the specified identifier inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a registry with the specified identifier inside the database", Err: err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
+	}
+
+	user, err := handler.DataStore.User().User(securityContext.UserID)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve user from the database", Err: err}
+	}
+
+	if !security.AuthorizedRegistryAccess(registry, user, securityContext.UserMemberships, portainer.EndpointID(endpointID)) {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
+	}
+
+	hideRegistryFields(registry, !securityContext.IsAdmin)
+	return response.JSON(w, registry)
+}

api/http/handler/endpoints/endpoint_registries_list.go

@@ -0,0 +1,130 @@
+package endpoints
+
+import (
+	"net/http"
+
+	"github.com/pkg/errors"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
+)
+
+// GET request on /endpoints/{id}/registries?namespace
+func (handler *Handler) endpointRegistriesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	user, err := handler.DataStore.User().User(securityContext.UserID)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve user from the database", err}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	isAdmin := securityContext.IsAdmin
+
+	registries, err := handler.DataStore.Registry().Registries()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
+	}
+
+	if endpointutils.IsKubernetesEndpoint(endpoint) {
+		namespace, _ := request.RetrieveQueryParameter(r, "namespace", true)
+
+		if namespace == "" && !isAdmin {
+			return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Missing namespace query parameter", Err: errors.New("missing namespace query parameter")}
+		}
+
+		if namespace != "" {
+
+			authorized, err := handler.isNamespaceAuthorized(endpoint, namespace, user.ID, securityContext.UserMemberships, isAdmin)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusNotFound, "Unable to check for namespace authorization", err}
+			}
+
+			if !authorized {
+				return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "User is not authorized to use namespace", Err: errors.New("user is not authorized to use namespace")}
+			}
+
+			registries = filterRegistriesByNamespace(registries, endpoint.ID, namespace)
+		}
+
+	} else if !isAdmin {
+		registries = security.FilterRegistries(registries, user, securityContext.UserMemberships, endpoint.ID)
+	}
+
+	for idx := range registries {
+		hideRegistryFields(&registries[idx], !isAdmin)
+	}
+
+	return response.JSON(w, registries)
+}
+
+func (handler *Handler) isNamespaceAuthorized(endpoint *portainer.Endpoint, namespace string, userId portainer.UserID, memberships []portainer.TeamMembership, isAdmin bool) (bool, error) {
+	if isAdmin || namespace == "" {
+		return true, nil
+	}
+
+	if namespace == "default" {
+		return true, nil
+	}
+
+	kcl, err := handler.K8sClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return false, errors.Wrap(err, "unable to retrieve kubernetes client")
+	}
+
+	accessPolicies, err := kcl.GetNamespaceAccessPolicies()
+	if err != nil {
+		return false, errors.Wrap(err, "unable to retrieve endpoint's namespaces policies")
+	}
+
+	namespacePolicy, ok := accessPolicies[namespace]
+	if !ok {
+		return false, nil
+	}
+
+	return security.AuthorizedAccess(userId, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies), nil
+}
+
+func filterRegistriesByNamespace(registries []portainer.Registry, endpointId portainer.EndpointID, namespace string) []portainer.Registry {
+	if namespace == "" {
+		return registries
+	}
+
+	filteredRegistries := []portainer.Registry{}
+
+	for _, registry := range registries {
+		for _, authorizedNamespace := range registry.RegistryAccesses[endpointId].Namespaces {
+			if authorizedNamespace == namespace {
+				filteredRegistries = append(filteredRegistries, registry)
+			}
+		}
+	}
+
+	return filteredRegistries
+}
+
+func hideRegistryFields(registry *portainer.Registry, hideAccesses bool) {
+	registry.Password = ""
+	registry.ManagementConfiguration = nil
+	if hideAccesses {
+		registry.RegistryAccesses = nil
+	}
+}

api/http/handler/endpoints/endpoint_registry_access.go

@@ -0,0 +1,149 @@
+package endpoints
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+type registryAccessPayload struct {
+	UserAccessPolicies portainer.UserAccessPolicies
+	TeamAccessPolicies portainer.TeamAccessPolicies
+	Namespaces         []string
+}
+
+func (payload *registryAccessPayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /endpoints/{id}/registries/{registryId}
+func (handler *Handler) endpointRegistryAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
+	}
+
+	registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid registry identifier route variable", Err: err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "User is not authorized", Err: err}
+	}
+
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
+	}
+
+	var payload registryAccessPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	if registry.RegistryAccesses == nil {
+		registry.RegistryAccesses = portainer.RegistryAccesses{}
+	}
+
+	if _, ok := registry.RegistryAccesses[endpoint.ID]; !ok {
+		registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{}
+	}
+
+	registryAccess := registry.RegistryAccesses[endpoint.ID]
+
+	if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		err := handler.updateKubeAccess(endpoint, registry, registryAccess.Namespaces, payload.Namespaces)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update kube access policies", Err: err}
+		}
+
+		registryAccess.Namespaces = payload.Namespaces
+	} else {
+		registryAccess.UserAccessPolicies = payload.UserAccessPolicies
+		registryAccess.TeamAccessPolicies = payload.TeamAccessPolicies
+	}
+
+	registry.RegistryAccesses[portainer.EndpointID(endpointID)] = registryAccess
+
+	handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
+
+	return response.Empty(w)
+}
+
+func (handler *Handler) updateKubeAccess(endpoint *portainer.Endpoint, registry *portainer.Registry, oldNamespaces, newNamespaces []string) error {
+	oldNamespacesSet := toSet(oldNamespaces)
+	newNamespacesSet := toSet(newNamespaces)
+
+	namespacesToRemove := setDifference(oldNamespacesSet, newNamespacesSet)
+	namespacesToAdd := setDifference(newNamespacesSet, oldNamespacesSet)
+
+	cli, err := handler.K8sClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return err
+	}
+
+	for namespace := range namespacesToRemove {
+		err := cli.DeleteRegistrySecret(registry, namespace)
+		if err != nil {
+			return err
+		}
+	}
+
+	for namespace := range namespacesToAdd {
+		err := cli.CreateRegistrySecret(registry, namespace)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type stringSet map[string]bool
+
+func toSet(list []string) stringSet {
+	set := stringSet{}
+	for _, el := range list {
+		set[el] = true
+	}
+	return set
+}
+
+// setDifference returns the set difference tagsA - tagsB
+func setDifference(setA stringSet, setB stringSet) stringSet {
+	set := stringSet{}
+
+	for el := range setA {
+		if !setB[el] {
+			set[el] = true
+		}
+	}
+
+	return set
+}

api/http/handler/endpoints/endpoint_update.go

@@ -151,15 +151,24 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
+	updateAuthorizations := false
+
 	if payload.Kubernetes != nil {
+		if payload.Kubernetes.Configuration.RestrictDefaultNamespace !=
+			endpoint.Kubernetes.Configuration.RestrictDefaultNamespace {
+			updateAuthorizations = true
+		}
+
 		endpoint.Kubernetes = *payload.Kubernetes
 	}
 
 	if payload.UserAccessPolicies != nil && !reflect.DeepEqual(payload.UserAccessPolicies, endpoint.UserAccessPolicies) {
+		updateAuthorizations = true
 		endpoint.UserAccessPolicies = payload.UserAccessPolicies
 	}
 
 	if payload.TeamAccessPolicies != nil && !reflect.DeepEqual(payload.TeamAccessPolicies, endpoint.TeamAccessPolicies) {
+		updateAuthorizations = true
 		endpoint.TeamAccessPolicies = payload.TeamAccessPolicies
 	}
 
@@ -252,6 +261,15 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
+	if updateAuthorizations {
+		if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+			err = handler.AuthorizationService.CleanNAPWithOverridePolicies(endpoint, nil)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update user authorizations", err}
+			}
+		}
+	}
+
 	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}

api/http/handler/endpoints/handler.go

@@ -5,6 +5,8 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/kubernetes/cli"
 
 	"net/http"
 
@@ -27,7 +29,9 @@ type Handler struct {
 	ProxyManager         *proxy.Manager
 	ReverseTunnelService portainer.ReverseTunnelService
 	SnapshotService      portainer.SnapshotService
+	K8sClientFactory     *cli.ClientFactory
 	ComposeStackManager  portainer.ComposeStackManager
+	AuthorizationService *authorization.Service
 }
 
 // NewHandler creates a handler to manage endpoint operations.
@@ -51,7 +55,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoints/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
-	h.Handle("/endpoints/{id}/dockerhub",
+	h.Handle("/endpoints/{id}/dockerhub/{registryId}",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
 	h.Handle("/endpoints/{id}/extensions",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)
@@ -61,5 +65,11 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointSnapshot))).Methods(http.MethodPost)
 	h.Handle("/endpoints/{id}/status",
 		bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)
+	h.Handle("/endpoints/{id}/registries",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistriesList))).Methods(http.MethodGet)
+	h.Handle("/endpoints/{id}/registries/{registryId}",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryInspect))).Methods(http.MethodGet)
+	h.Handle("/endpoints/{id}/registries/{registryId}",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryAccess))).Methods(http.MethodPut)
 	return h
 }

api/http/handler/handler.go

@@ -7,7 +7,6 @@ import (
 	"github.com/portainer/portainer/api/http/handler/auth"
 	"github.com/portainer/portainer/api/http/handler/backup"
 	"github.com/portainer/portainer/api/http/handler/customtemplates"
-	"github.com/portainer/portainer/api/http/handler/dockerhub"
 	"github.com/portainer/portainer/api/http/handler/edgegroups"
 	"github.com/portainer/portainer/api/http/handler/edgejobs"
 	"github.com/portainer/portainer/api/http/handler/edgestacks"
@@ -39,7 +38,6 @@ type Handler struct {
 	AuthHandler            *auth.Handler
 	BackupHandler          *backup.Handler
 	CustomTemplatesHandler *customtemplates.Handler
-	DockerHubHandler       *dockerhub.Handler
 	EdgeGroupsHandler      *edgegroups.Handler
 	EdgeJobsHandler        *edgejobs.Handler
 	EdgeStacksHandler      *edgestacks.Handler
@@ -88,8 +86,6 @@ type Handler struct {
 // @tag.description Authenticate against Portainer HTTP API
 // @tag.name custom_templates
 // @tag.description Manage Custom Templates
-// @tag.name dockerhub
-// @tag.description Manage how Portainer connects to the DockerHub
 // @tag.name edge_groups
 // @tag.description Manage Edge Groups
 // @tag.name edge_jobs
@@ -146,8 +142,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/restore"):
 		http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
-	case strings.HasPrefix(r.URL.Path, "/api/dockerhub"):
-		http.StripPrefix("/api", h.DockerHubHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/custom_templates"):
 		http.StripPrefix("/api", h.CustomTemplatesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/edge_stacks"):

api/http/handler/registries/handler.go

@@ -5,32 +5,46 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/kubernetes/cli"
 )
 
-func hideFields(registry *portainer.Registry) {
+func hideFields(registry *portainer.Registry, hideAccesses bool) {
 	registry.Password = ""
 	registry.ManagementConfiguration = nil
+	if hideAccesses {
+		registry.RegistryAccesses = nil
+	}
 }
 
 // Handler is the HTTP handler used to handle registry operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer *security.RequestBouncer
-	DataStore      portainer.DataStore
-	FileService    portainer.FileService
-	ProxyManager   *proxy.Manager
+	requestBouncer   *security.RequestBouncer
+	DataStore        portainer.DataStore
+	FileService      portainer.FileService
+	ProxyManager     *proxy.Manager
+	K8sClientFactory *cli.ClientFactory
 }
 
 // NewHandler creates a handler to manage registry operations.
 func NewHandler(bouncer *security.RequestBouncer) *Handler {
-	h := &Handler{
+	h := newHandler(bouncer)
+	h.initRouter(bouncer)
+
+	return h
+}
+
+func newHandler(bouncer *security.RequestBouncer) *Handler {
+	return &Handler{
 		Router:         mux.NewRouter(),
 		requestBouncer: bouncer,
 	}
+}
 
+func (h *Handler) initRouter(bouncer accessGuard) {
 	h.Handle("/registries",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.registryCreate))).Methods(http.MethodPost)
 	h.Handle("/registries",
@@ -45,5 +59,21 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.registryDelete))).Methods(http.MethodDelete)
 	h.PathPrefix("/registries/proxies/gitlab").Handler(
 		bouncer.AdminAccess(httperror.LoggerHandler(h.proxyRequestsToGitlabAPIWithoutRegistry)))
-	return h
+}
+
+type accessGuard interface {
+	AdminAccess(h http.Handler) http.Handler
+	RestrictedAccess(h http.Handler) http.Handler
+	AuthenticatedAccess(h http.Handler) http.Handler
+}
+
+func (handler *Handler) registriesHaveSameURLAndCredentials(r1, r2 *portainer.Registry) bool {
+	hasSameUrl := r1.URL == r2.URL
+	hasSameCredentials := r1.Authentication == r2.Authentication && (!r1.Authentication || (r1.Authentication && r1.Username == r2.Username))
+
+	if r1.Type != portainer.GitlabRegistry || r2.Type != portainer.GitlabRegistry {
+		return hasSameUrl && hasSameCredentials
+	}
+
+	return hasSameUrl && hasSameCredentials && r1.Gitlab.ProjectPath == r2.Gitlab.ProjectPath
 }

api/http/handler/registries/registry_configure.go

@@ -10,6 +10,8 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 type registryConfigurePayload struct {
@@ -93,9 +95,12 @@ func (payload *registryConfigurePayload) Validate(r *http.Request) error {
 // @failure 500 "Server error"
 // @router /registries/{id}/configure [post]
 func (handler *Handler) registryConfigure(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to configure registry", httperrors.ErrResourceAccessDenied}
 	}
 
 	payload := &registryConfigurePayload{}
@@ -104,6 +109,11 @@ func (handler *Handler) registryConfigure(w http.ResponseWriter, r *http.Request
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
+	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
+	}
+
 	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}

api/http/handler/registries/registry_create.go

@@ -2,6 +2,7 @@ package registries
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
@@ -9,15 +10,19 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 type registryCreatePayload struct {
 	// Name that will be used to identify this registry
 	Name string `example:"my-registry" validate:"required"`
-	// Registry Type. Valid values are: 1 (Quay.io), 2 (Azure container registry), 3 (custom registry) or 4 (Gitlab registry)
-	Type portainer.RegistryType `example:"1" validate:"required" enums:"1,2,3,4"`
+	// Registry Type. Valid values are: 1 (Quay.io), 2 (Azure container registry), 3 (custom registry), 4 (Gitlab registry), 5 (ProGet registry) or 6 (DockerHub)
+	Type portainer.RegistryType `example:"1" validate:"required" enums:"1,2,3,4,5,6"`
 	// URL or IP address of the Docker registry
-	URL string `example:"registry.mydomain.tld:2375" validate:"required"`
+	URL string `example:"registry.mydomain.tld:2375/feed" validate:"required"`
+	// BaseURL required for ProGet registry
+	BaseURL string `example:"registry.mydomain.tld:2375"`
 	// Is authentication against this registry enabled
 	Authentication bool `example:"false" validate:"required"`
 	// Username used to authenticate against this registry. Required when Authentication is true
@@ -30,7 +35,7 @@ type registryCreatePayload struct {
 	Quay portainer.QuayRegistryData
 }
 
-func (payload *registryCreatePayload) Validate(r *http.Request) error {
+func (payload *registryCreatePayload) Validate(_ *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid registry name")
 	}
@@ -40,9 +45,17 @@ func (payload *registryCreatePayload) Validate(r *http.Request) error {
 	if payload.Authentication && (govalidator.IsNull(payload.Username) || govalidator.IsNull(payload.Password)) {
 		return errors.New("Invalid credentials. Username and password must be specified when authentication is enabled")
 	}
-	if payload.Type != portainer.QuayRegistry && payload.Type != portainer.AzureRegistry && payload.Type != portainer.CustomRegistry && payload.Type != portainer.GitlabRegistry {
-		return errors.New("Invalid registry type. Valid values are: 1 (Quay.io), 2 (Azure container registry), 3 (custom registry) or 4 (Gitlab registry)")
+
+	switch payload.Type {
+	case portainer.QuayRegistry, portainer.AzureRegistry, portainer.CustomRegistry, portainer.GitlabRegistry, portainer.ProGetRegistry, portainer.DockerHubRegistry:
+	default:
+		return errors.New("invalid registry type. Valid values are: 1 (Quay.io), 2 (Azure container registry), 3 (custom registry), 4 (Gitlab registry), 5 (ProGet registry) or 6 (DockerHub)")
 	}
+
+	if payload.Type == portainer.ProGetRegistry && payload.BaseURL == "" {
+		return fmt.Errorf("BaseURL is required for registry type %d (ProGet)", portainer.ProGetRegistry)
+	}
+
 	return nil
 }
 
@@ -60,23 +73,41 @@ func (payload *registryCreatePayload) Validate(r *http.Request) error {
 // @failure 500 "Server error"
 // @router /registries [post]
 func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to create registry", httperrors.ErrResourceAccessDenied}
+	}
+
 	var payload registryCreatePayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
 	registry := &portainer.Registry{
-		Type:               portainer.RegistryType(payload.Type),
-		Name:               payload.Name,
-		URL:                payload.URL,
-		Authentication:     payload.Authentication,
-		Username:           payload.Username,
-		Password:           payload.Password,
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		Gitlab:             payload.Gitlab,
-		Quay:               payload.Quay,
+		Type:             portainer.RegistryType(payload.Type),
+		Name:             payload.Name,
+		URL:              payload.URL,
+		BaseURL:          payload.BaseURL,
+		Authentication:   payload.Authentication,
+		Username:         payload.Username,
+		Password:         payload.Password,
+		RegistryAccesses: portainer.RegistryAccesses{},
+		Gitlab:           payload.Gitlab,
+		Quay:             payload.Quay,
+	}
+
+	registries, err := handler.DataStore.Registry().Registries()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
+	}
+	for _, r := range registries {
+		if handler.registriesHaveSameURLAndCredentials(&r, registry) {
+			return &httperror.HandlerError{http.StatusConflict, "Another registry with the same URL and credentials already exists", errors.New("A registry is already defined for this URL and credentials")}
+		}
 	}
 
 	err = handler.DataStore.Registry().CreateRegistry(registry)
@@ -84,6 +115,6 @@ func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the registry inside the database", err}
 	}
 
-	hideFields(registry)
+	hideFields(registry, true)
 	return response.JSON(w, registry)
 }

api/http/handler/registries/registry_create_test.go

@@ -0,0 +1,113 @@
+package registries
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_registryCreatePayload_Validate(t *testing.T) {
+	basePayload := registryCreatePayload{Name: "Test registry", URL: "http://example.com"}
+	t.Run("Can't create a ProGet registry if BaseURL is empty", func(t *testing.T) {
+		payload := basePayload
+		payload.Type = portainer.ProGetRegistry
+		err := payload.Validate(nil)
+		assert.Error(t, err)
+	})
+	t.Run("Can create a GitLab registry if BaseURL is empty", func(t *testing.T) {
+		payload := basePayload
+		payload.Type = portainer.GitlabRegistry
+		err := payload.Validate(nil)
+		assert.NoError(t, err)
+	})
+	t.Run("Can create a ProGet registry if BaseURL is not empty", func(t *testing.T) {
+		payload := basePayload
+		payload.Type = portainer.ProGetRegistry
+		payload.BaseURL = "http://example.com"
+		err := payload.Validate(nil)
+		assert.NoError(t, err)
+	})
+}
+
+type testRegistryService struct {
+	portainer.RegistryService
+	createRegistry func(r *portainer.Registry) error
+	updateRegistry func(ID portainer.RegistryID, r *portainer.Registry) error
+	getRegistry    func(ID portainer.RegistryID) (*portainer.Registry, error)
+}
+
+type testDataStore struct {
+	portainer.DataStore
+	registry *testRegistryService
+}
+
+func (t testDataStore) Registry() portainer.RegistryService {
+	return t.registry
+}
+
+func (t testRegistryService) CreateRegistry(r *portainer.Registry) error {
+	return t.createRegistry(r)
+}
+
+func (t testRegistryService) UpdateRegistry(ID portainer.RegistryID, r *portainer.Registry) error {
+	return t.updateRegistry(ID, r)
+}
+
+func (t testRegistryService) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
+	return t.getRegistry(ID)
+}
+
+func (t testRegistryService) Registries() ([]portainer.Registry, error) {
+	return nil, nil
+}
+
+func TestHandler_registryCreate(t *testing.T) {
+	payload := registryCreatePayload{
+		Name:           "Test registry",
+		Type:           portainer.ProGetRegistry,
+		URL:            "http://example.com",
+		BaseURL:        "http://example.com",
+		Authentication: false,
+		Username:       "username",
+		Password:       "password",
+		Gitlab:         portainer.GitlabRegistryData{},
+	}
+	payloadBytes, err := json.Marshal(payload)
+	assert.NoError(t, err)
+	r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(payloadBytes))
+	w := httptest.NewRecorder()
+
+	restrictedContext := &security.RestrictedRequestContext{
+		IsAdmin: true,
+		UserID:  portainer.UserID(1),
+	}
+
+	ctx := security.StoreRestrictedRequestContext(r, restrictedContext)
+	r = r.WithContext(ctx)
+
+	registry := portainer.Registry{}
+	handler := Handler{}
+	handler.DataStore = testDataStore{
+		registry: &testRegistryService{
+			createRegistry: func(r *portainer.Registry) error {
+				registry = *r
+				return nil
+			},
+		},
+	}
+	handlerError := handler.registryCreate(w, r)
+	assert.Nil(t, handlerError)
+	assert.Equal(t, payload.Name, registry.Name)
+	assert.Equal(t, payload.Type, registry.Type)
+	assert.Equal(t, payload.URL, registry.URL)
+	assert.Equal(t, payload.BaseURL, registry.BaseURL)
+	assert.Equal(t, payload.Authentication, registry.Authentication)
+	assert.Equal(t, payload.Username, registry.Username)
+	assert.Equal(t, payload.Password, registry.Password)
+}

api/http/handler/registries/registry_delete.go

@@ -8,6 +8,8 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // @id RegistryDelete
@@ -23,6 +25,14 @@ import (
 // @failure 500 "Server error"
 // @router /registries/{id} [delete]
 func (handler *Handler) registryDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to delete registry", httperrors.ErrResourceAccessDenied}
+	}
+
 	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}

api/http/handler/registries/registry_inspect.go

@@ -5,7 +5,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/http/errors"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
@@ -27,6 +26,7 @@ import (
 // @failure 500 "Server error"
 // @router /registries/{id} [get]
 func (handler *Handler) registryInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+
 	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
@@ -39,11 +39,6 @@ func (handler *Handler) registryInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
 	}
 
-	err = handler.requestBouncer.RegistryAccess(r, registry)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access registry", errors.ErrEndpointAccessDenied}
-	}
-
-	hideFields(registry)
+	hideFields(registry, false)
 	return response.JSON(w, registry)
 }

api/http/handler/registries/registry_list.go

@@ -5,6 +5,7 @@ import (
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
+	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -21,21 +22,18 @@ import (
 // @failure 500 "Server error"
 // @router /registries [get]
 func (handler *Handler) registryList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to list registries, use /endpoints/:endpointId/registries route instead", httperrors.ErrResourceAccessDenied}
+	}
+
 	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
 	}
 
-	securityContext, err := security.RetrieveRestrictedRequestContext(r)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
-	}
-
-	filteredRegistries := security.FilterRegistries(registries, securityContext)
-
-	for idx := range filteredRegistries {
-		hideFields(&filteredRegistries[idx])
-	}
-
-	return response.JSON(w, filteredRegistries)
+	return response.JSON(w, registries)
 }

api/http/handler/registries/registry_update.go

@@ -9,6 +9,8 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 type registryUpdatePayload struct {
@@ -16,15 +18,16 @@ type registryUpdatePayload struct {
 	Name *string `validate:"required" example:"my-registry"`
 	// URL or IP address of the Docker registry
 	URL *string `validate:"required" example:"registry.mydomain.tld:2375"`
+	// BaseURL is used for quay registry
+	BaseURL *string `json:",omitempty" example:"registry.mydomain.tld:2375"`
 	// Is authentication against this registry enabled
 	Authentication *bool `example:"false" validate:"required"`
 	// Username used to authenticate against this registry. Required when Authentication is true
 	Username *string `example:"registry_user"`
 	// Password used to authenticate against this registry. required when Authentication is true
-	Password           *string `example:"registry_password"`
-	UserAccessPolicies portainer.UserAccessPolicies
-	TeamAccessPolicies portainer.TeamAccessPolicies
-	Quay               *portainer.QuayRegistryData
+	Password         *string `example:"registry_password"`
+	RegistryAccesses *portainer.RegistryAccesses
+	Quay             *portainer.QuayRegistryData
 }
 
 func (payload *registryUpdatePayload) Validate(r *http.Request) error {
@@ -48,17 +51,19 @@ func (payload *registryUpdatePayload) Validate(r *http.Request) error {
 // @failure 500 "Server error"
 // @router /registries/{id} [put]
 func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+	if !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to update registry", httperrors.ErrResourceAccessDenied}
+	}
+
 	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
 	}
 
-	var payload registryUpdatePayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
-	}
-
 	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
@@ -66,27 +71,26 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
 	}
 
+	var payload registryUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
 	if payload.Name != nil {
 		registry.Name = *payload.Name
 	}
 
-	if payload.URL != nil {
-		registries, err := handler.DataStore.Registry().Registries()
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
-		}
-		for _, r := range registries {
-			if r.ID != registry.ID && hasSameURL(&r, registry) {
-				return &httperror.HandlerError{http.StatusConflict, "Another registry with the same URL already exists", errors.New("A registry is already defined for this URL")}
-			}
-		}
+	shouldUpdateSecrets := false
 
-		registry.URL = *payload.URL
+	if registry.Type == portainer.ProGetRegistry && payload.BaseURL != nil {
+		registry.BaseURL = *payload.BaseURL
 	}
 
 	if payload.Authentication != nil {
 		if *payload.Authentication {
 			registry.Authentication = true
+			shouldUpdateSecrets = shouldUpdateSecrets || (payload.Username != nil && *payload.Username != registry.Username) || (payload.Password != nil && *payload.Password != registry.Password)
 
 			if payload.Username != nil {
 				registry.Username = *payload.Username
@@ -103,12 +107,35 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	if payload.UserAccessPolicies != nil {
-		registry.UserAccessPolicies = payload.UserAccessPolicies
+	if payload.URL != nil {
+		shouldUpdateSecrets = shouldUpdateSecrets || (*payload.URL != registry.URL)
+
+		registry.URL = *payload.URL
+		registries, err := handler.DataStore.Registry().Registries()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
+		}
+		for _, r := range registries {
+			if r.ID != registry.ID && handler.registriesHaveSameURLAndCredentials(&r, registry) {
+				return &httperror.HandlerError{http.StatusConflict, "Another registry with the same URL and credentials already exists", errors.New("A registry is already defined for this URL and credentials")}
+			}
+		}
 	}
 
-	if payload.TeamAccessPolicies != nil {
-		registry.TeamAccessPolicies = payload.TeamAccessPolicies
+	if shouldUpdateSecrets {
+		for endpointID, endpointAccess := range registry.RegistryAccesses {
+			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update access to registry", err}
+			}
+
+			if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+				err = handler.updateEndpointRegistryAccess(endpoint, registry, endpointAccess)
+				if err != nil {
+					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update access to registry", err}
+				}
+			}
+		}
 	}
 
 	if payload.Quay != nil {
@@ -123,10 +150,24 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 	return response.JSON(w, registry)
 }
 
-func hasSameURL(r1, r2 *portainer.Registry) bool {
-	if r1.Type != portainer.GitlabRegistry || r2.Type != portainer.GitlabRegistry {
-		return r1.URL == r2.URL
+func (handler *Handler) updateEndpointRegistryAccess(endpoint *portainer.Endpoint, registry *portainer.Registry, endpointAccess portainer.RegistryAccessPolicies) error {
+
+	cli, err := handler.K8sClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return err
 	}
 
-	return r1.URL == r2.URL && r1.Gitlab.ProjectPath == r2.Gitlab.ProjectPath
+	for _, namespace := range endpointAccess.Namespaces {
+		err := cli.DeleteRegistrySecret(registry, namespace)
+		if err != nil {
+			return err
+		}
+
+		err = cli.CreateRegistrySecret(registry, namespace)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

api/http/handler/registries/registry_update_test.go

@@ -0,0 +1,88 @@
+package registries
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/stretchr/testify/assert"
+)
+
+func ps(s string) *string {
+	return &s
+}
+
+func pb(b bool) *bool {
+	return &b
+}
+
+type TestBouncer struct{}
+
+func (t TestBouncer) AdminAccess(h http.Handler) http.Handler {
+	return h
+}
+
+func (t TestBouncer) RestrictedAccess(h http.Handler) http.Handler {
+	return h
+}
+
+func (t TestBouncer) AuthenticatedAccess(h http.Handler) http.Handler {
+	return h
+}
+
+func TestHandler_registryUpdate(t *testing.T) {
+	payload := registryUpdatePayload{
+		Name:           ps("Updated test registry"),
+		URL:            ps("http://example.org/feed"),
+		BaseURL:        ps("http://example.org"),
+		Authentication: pb(true),
+		Username:       ps("username"),
+		Password:       ps("password"),
+	}
+	payloadBytes, err := json.Marshal(payload)
+	assert.NoError(t, err)
+	registry := portainer.Registry{Type: portainer.ProGetRegistry, ID: 5}
+	r := httptest.NewRequest(http.MethodPut, "/registries/5", bytes.NewReader(payloadBytes))
+	w := httptest.NewRecorder()
+
+	restrictedContext := &security.RestrictedRequestContext{
+		IsAdmin: true,
+		UserID:  portainer.UserID(1),
+	}
+
+	ctx := security.StoreRestrictedRequestContext(r, restrictedContext)
+	r = r.WithContext(ctx)
+
+	updatedRegistry := portainer.Registry{}
+	handler := newHandler(nil)
+	handler.initRouter(TestBouncer{})
+	handler.DataStore = testDataStore{
+		registry: &testRegistryService{
+			getRegistry: func(_ portainer.RegistryID) (*portainer.Registry, error) {
+				return &registry, nil
+			},
+			updateRegistry: func(ID portainer.RegistryID, r *portainer.Registry) error {
+				assert.Equal(t, ID, r.ID)
+				updatedRegistry = *r
+				return nil
+			},
+		},
+	}
+
+	handler.Router.ServeHTTP(w, r)
+	assert.Equal(t, http.StatusOK, w.Code)
+	// Registry type should remain intact
+	assert.Equal(t, registry.Type, updatedRegistry.Type)
+
+	assert.Equal(t, *payload.Name, updatedRegistry.Name)
+	assert.Equal(t, *payload.URL, updatedRegistry.URL)
+	assert.Equal(t, *payload.BaseURL, updatedRegistry.BaseURL)
+	assert.Equal(t, *payload.Authentication, updatedRegistry.Authentication)
+	assert.Equal(t, *payload.Username, updatedRegistry.Username)
+	assert.Equal(t, *payload.Password, updatedRegistry.Password)
+
+}

api/http/handler/settings/settings_public.go

@@ -18,6 +18,8 @@ type publicSettingsResponse struct {
 	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures" example:"true"`
 	// The URL used for oauth login
 	OAuthLoginURI string `json:"OAuthLoginURI" example:"https://gitlab.com/oauth"`
+	// The URL used for oauth logout
+	OAuthLogoutURI string `json:"OAuthLogoutURI" example:"https://gitlab.com/oauth/logout"`
 	// Whether telemetry is enabled
 	EnableTelemetry bool `json:"EnableTelemetry" example:"true"`
 }
@@ -34,20 +36,32 @@ type publicSettingsResponse struct {
 func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve the settings from the database", err}
-	}
-
-	publicSettings := &publicSettingsResponse{
-		LogoURL:                   settings.LogoURL,
-		AuthenticationMethod:      settings.AuthenticationMethod,
-		EnableEdgeComputeFeatures: settings.EnableEdgeComputeFeatures,
-		EnableTelemetry:           settings.EnableTelemetry,
-		OAuthLoginURI: fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s&prompt=login",
-			settings.OAuthSettings.AuthorizationURI,
-			settings.OAuthSettings.ClientID,
-			settings.OAuthSettings.RedirectURI,
-			settings.OAuthSettings.Scopes),
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve the settings from the database", Err: err}
 	}
 
+	publicSettings := generatePublicSettings(settings)
 	return response.JSON(w, publicSettings)
 }
+
+func generatePublicSettings(appSettings *portainer.Settings) *publicSettingsResponse {
+	publicSettings := &publicSettingsResponse{
+		LogoURL:                   appSettings.LogoURL,
+		AuthenticationMethod:      appSettings.AuthenticationMethod,
+		EnableEdgeComputeFeatures: appSettings.EnableEdgeComputeFeatures,
+		EnableTelemetry:           appSettings.EnableTelemetry,
+	}
+	//if OAuth authentication is on, compose the related fields from application settings
+	if publicSettings.AuthenticationMethod == portainer.AuthenticationOAuth {
+		publicSettings.OAuthLogoutURI = appSettings.OAuthSettings.LogoutURI
+		publicSettings.OAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
+			appSettings.OAuthSettings.AuthorizationURI,
+			appSettings.OAuthSettings.ClientID,
+			appSettings.OAuthSettings.RedirectURI,
+			appSettings.OAuthSettings.Scopes)
+		//control prompt=login param according to the SSO setting
+		if !appSettings.OAuthSettings.SSO {
+			publicSettings.OAuthLoginURI += "&prompt=login"
+		}
+	}
+	return publicSettings
+}

api/http/handler/settings/settings_public_test.go

@@ -0,0 +1,70 @@
+package settings
+
+import (
+	"fmt"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+const (
+	dummyOAuthClientID          = "1a2b3c4d"
+	dummyOAuthScopes            = "scopes"
+	dummyOAuthAuthenticationURI = "example.com/auth"
+	dummyOAuthRedirectURI       = "example.com/redirect"
+	dummyOAuthLogoutURI         = "example.com/logout"
+)
+
+var (
+	dummyOAuthLoginURI string
+	mockAppSettings    *portainer.Settings
+)
+
+func setup() {
+	dummyOAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
+		dummyOAuthAuthenticationURI,
+		dummyOAuthClientID,
+		dummyOAuthRedirectURI,
+		dummyOAuthScopes)
+	mockAppSettings = &portainer.Settings{
+		AuthenticationMethod: portainer.AuthenticationOAuth,
+		OAuthSettings: portainer.OAuthSettings{
+			AuthorizationURI: dummyOAuthAuthenticationURI,
+			ClientID:         dummyOAuthClientID,
+			Scopes:           dummyOAuthScopes,
+			RedirectURI:      dummyOAuthRedirectURI,
+			LogoutURI:        dummyOAuthLogoutURI,
+		},
+	}
+}
+
+func TestGeneratePublicSettingsWithSSO(t *testing.T) {
+	setup()
+	mockAppSettings.OAuthSettings.SSO = true
+	publicSettings := generatePublicSettings(mockAppSettings)
+	if publicSettings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		t.Errorf("wrong AuthenticationMethod, want: %d, got: %d", portainer.AuthenticationOAuth, publicSettings.AuthenticationMethod)
+	}
+	if publicSettings.OAuthLoginURI != dummyOAuthLoginURI {
+		t.Errorf("wrong OAuthLoginURI when SSO is switched on, want: %s, got: %s", dummyOAuthLoginURI, publicSettings.OAuthLoginURI)
+	}
+	if publicSettings.OAuthLogoutURI != dummyOAuthLogoutURI {
+		t.Errorf("wrong OAuthLogoutURI, want: %s, got: %s", dummyOAuthLogoutURI, publicSettings.OAuthLogoutURI)
+	}
+}
+
+func TestGeneratePublicSettingsWithoutSSO(t *testing.T) {
+	setup()
+	mockAppSettings.OAuthSettings.SSO = false
+	publicSettings := generatePublicSettings(mockAppSettings)
+	if publicSettings.AuthenticationMethod != portainer.AuthenticationOAuth {
+		t.Errorf("wrong AuthenticationMethod, want: %d, got: %d", portainer.AuthenticationOAuth, publicSettings.AuthenticationMethod)
+	}
+	expectedOAuthLoginURI := dummyOAuthLoginURI + "&prompt=login"
+	if publicSettings.OAuthLoginURI != expectedOAuthLoginURI {
+		t.Errorf("wrong OAuthLoginURI when SSO is switched off, want: %s, got: %s", expectedOAuthLoginURI, publicSettings.OAuthLoginURI)
+	}
+	if publicSettings.OAuthLogoutURI != dummyOAuthLogoutURI {
+		t.Errorf("wrong OAuthLogoutURI, want: %s, got: %s", dummyOAuthLogoutURI, publicSettings.OAuthLogoutURI)
+	}
+}

api/http/handler/stacks/create_compose_stack.go

@@ -169,19 +169,10 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
-	}
-
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
@@ -246,11 +237,11 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
 		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
@@ -299,7 +290,6 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 type composeStackDeploymentConfig struct {
 	stack      *portainer.Stack
 	endpoint   *portainer.Endpoint
-	dockerhub  *portainer.DockerHub
 	registries []portainer.Registry
 	isAdmin    bool
 	user       *portainer.User
@@ -311,26 +301,20 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai
 		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
-	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
+	user, err := handler.DataStore.User().User(securityContext.UserID)
 	if err != nil {
-		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve DockerHub details from the database", Err: err}
+		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
 
 	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
 		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve registries from the database", Err: err}
 	}
-	filteredRegistries := security.FilterRegistries(registries, securityContext)
-
-	user, err := handler.DataStore.User().User(securityContext.UserID)
-	if err != nil {
-		return nil, &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
-	}
+	filteredRegistries := security.FilterRegistries(registries, user, securityContext.UserMemberships, endpoint.ID)
 
 	config := &composeStackDeploymentConfig{
 		stack:      stack,
 		endpoint:   endpoint,
-		dockerhub:  dockerhub,
 		registries: filteredRegistries,
 		isAdmin:    securityContext.IsAdmin,
 		user:       user,
@@ -375,7 +359,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	handler.SwarmStackManager.Login(config.dockerhub, config.registries, config.endpoint)
+	handler.SwarmStackManager.Login(config.registries, config.endpoint)
 
 	err = handler.ComposeStackManager.Up(config.stack, config.endpoint)
 	if err != nil {

api/http/handler/stacks/create_kubernetes_stack.go

@@ -2,7 +2,11 @@ package stacks
 
 import (
 	"errors"
+	"io/ioutil"
 	"net/http"
+	"path/filepath"
+	"strconv"
+	"time"
 
 	"github.com/asaskevich/govalidator"
 
@@ -10,15 +14,29 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
 )
 
-type kubernetesStackPayload struct {
+const defaultReferenceName = "refs/heads/master"
+
+type kubernetesStringDeploymentPayload struct {
 	ComposeFormat    bool
 	Namespace        string
 	StackFileContent string
 }
 
-func (payload *kubernetesStackPayload) Validate(r *http.Request) error {
+type kubernetesGitDeploymentPayload struct {
+	ComposeFormat            bool
+	Namespace                string
+	RepositoryURL            string
+	RepositoryReferenceName  string
+	RepositoryAuthentication bool
+	RepositoryUsername       string
+	RepositoryPassword       string
+	FilePathInRepository     string
+}
+
+func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.StackFileContent) {
 		return errors.New("Invalid stack file content")
 	}
@@ -28,32 +46,146 @@ func (payload *kubernetesStackPayload) Validate(r *http.Request) error {
 	return nil
 }
 
+func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Namespace) {
+		return errors.New("Invalid namespace")
+	}
+	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
+		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
+	}
+	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
+		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+	if govalidator.IsNull(payload.FilePathInRepository) {
+		return errors.New("Invalid file path in repository")
+	}
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultReferenceName
+	}
+	return nil
+}
+
 type createKubernetesStackResponse struct {
 	Output string `json:"Output"`
 }
 
-func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
-	var payload kubernetesStackPayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	var payload kubernetesStringDeploymentPayload
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
+	stackID := handler.DataStore.Stack().GetNextIdentifier()
+	stack := &portainer.Stack{
+		ID:           portainer.StackID(stackID),
+		Type:         portainer.KubernetesStack,
+		EndpointID:   endpoint.ID,
+		EntryPoint:   filesystem.ManifestFileDefaultName,
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+	}
+
+	stackFolder := strconv.Itoa(int(stack.ID))
+	projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist Kubernetes manifest file on disk", Err: err}
+	}
+	stack.ProjectPath = projectPath
+
+	doCleanUp := true
+	defer handler.cleanUp(stack, &doCleanUp)
+
 	output, err := handler.deployKubernetesStack(endpoint, payload.StackFileContent, payload.ComposeFormat, payload.Namespace)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to deploy Kubernetes stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
+	}
+
+	err = handler.DataStore.Stack().CreateStack(stack)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err}
 	}
 
 	resp := &createKubernetesStackResponse{
-		Output: string(output),
+		Output: output,
 	}
 
 	return response.JSON(w, resp)
 }
 
-func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	var payload kubernetesGitDeploymentPayload
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	stackID := handler.DataStore.Stack().GetNextIdentifier()
+	stack := &portainer.Stack{
+		ID:           portainer.StackID(stackID),
+		Type:         portainer.KubernetesStack,
+		EndpointID:   endpoint.ID,
+		EntryPoint:   payload.FilePathInRepository,
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+	}
+
+	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
+	stack.ProjectPath = projectPath
+
+	doCleanUp := true
+	defer handler.cleanUp(stack, &doCleanUp)
+
+	stackFileContent, err := handler.cloneManifestContentFromGitRepo(&payload, stack.ProjectPath)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to process manifest from Git repository", Err: err}
+	}
+
+	output, err := handler.deployKubernetesStack(endpoint, stackFileContent, payload.ComposeFormat, payload.Namespace)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
+	}
+
+	err = handler.DataStore.Stack().CreateStack(stack)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
+	}
+
+	resp := &createKubernetesStackResponse{
+		Output: output,
+	}
+	return response.JSON(w, resp)
+}
+
+func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	return handler.KubernetesDeployer.Deploy(endpoint, data, composeFormat, namespace)
+	if composeFormat {
+		convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(stackConfig)
+		if err != nil {
+			return "", err
+		}
+		stackConfig = string(convertedConfig)
+	}
+
+	return handler.KubernetesDeployer.Deploy(endpoint, stackConfig, namespace)
+
+}
+
+func (handler *Handler) cloneManifestContentFromGitRepo(gitInfo *kubernetesGitDeploymentPayload, projectPath string) (string, error) {
+	repositoryUsername := gitInfo.RepositoryUsername
+	repositoryPassword := gitInfo.RepositoryPassword
+	if !gitInfo.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
+	}
+
+	err := handler.GitService.CloneRepository(projectPath, gitInfo.RepositoryURL, gitInfo.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		return "", err
+	}
+	content, err := ioutil.ReadFile(filepath.Join(projectPath, gitInfo.FilePathInRepository))
+	if err != nil {
+		return "", err
+	}
+	return string(content), nil
 }

api/http/handler/stacks/create_kubernetes_stack_test.go

@@ -0,0 +1,64 @@
+package stacks
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type git struct {
+	content string
+}
+
+func (g *git) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error {
+	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
+}
+func (g *git) ClonePublicRepository(repositoryURL string, referenceName string, destination string) error {
+	return ioutil.WriteFile(path.Join(destination, "deployment.yml"), []byte(g.content), 0755)
+}
+func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
+	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
+}
+
+func TestCloneAndConvertGitRepoFile(t *testing.T) {
+	dir, err := os.MkdirTemp("", "kube-create-stack")
+	assert.NoError(t, err, "failed to create a tmp dir")
+	defer os.RemoveAll(dir)
+
+	content := `apiVersion: apps/v1
+	kind: Deployment
+	metadata:
+		name: nginx-deployment
+		labels:
+			app: nginx
+	spec:
+		replicas: 3
+		selector:
+			matchLabels:
+				app: nginx
+		template:
+			metadata:
+				labels:
+					app: nginx
+			spec:
+				containers:
+				- name: nginx
+					image: nginx:1.14.2
+					ports:
+					- containerPort: 80`
+
+	h := &Handler{
+		GitService: &git{
+			content: content,
+		},
+	}
+	gitInfo := &kubernetesGitDeploymentPayload{
+		FilePathInRepository: "deployment.yml",
+	}
+	fileContent, err := h.cloneManifestContentFromGitRepo(gitInfo, dir)
+	assert.NoError(t, err, "failed to clone or convert the file from Git repo")
+	assert.Equal(t, content, fileContent)
+}

api/http/handler/stacks/create_swarm_stack.go

@@ -47,6 +47,8 @@ func (handler *Handler) createSwarmStackFromFileContent(w http.ResponseWriter, r
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
+	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
+
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
@@ -148,6 +150,8 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
+	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
+
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
@@ -173,21 +177,12 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:            payload.RepositoryURL,
-		referenceName:  payload.RepositoryReferenceName,
-		path:           projectPath,
-		authentication: payload.RepositoryAuthentication,
-		username:       payload.RepositoryUsername,
-		password:       payload.RepositoryPassword,
-	}
-
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
 	config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
@@ -253,6 +248,8 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
+	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
+
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
@@ -309,7 +306,6 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r
 type swarmStackDeploymentConfig struct {
 	stack      *portainer.Stack
 	endpoint   *portainer.Endpoint
-	dockerhub  *portainer.DockerHub
 	registries []portainer.Registry
 	prune      bool
 	isAdmin    bool
@@ -322,26 +318,20 @@ func (handler *Handler) createSwarmDeployConfig(r *http.Request, stack *portaine
 		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
 	}
 
-	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
+	user, err := handler.DataStore.User().User(securityContext.UserID)
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to load user information from the database", err}
 	}
 
 	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
 		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
 	}
-	filteredRegistries := security.FilterRegistries(registries, securityContext)
-
-	user, err := handler.DataStore.User().User(securityContext.UserID)
-	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to load user information from the database", err}
-	}
+	filteredRegistries := security.FilterRegistries(registries, user, securityContext.UserMemberships, endpoint.ID)
 
 	config := &swarmStackDeploymentConfig{
 		stack:      stack,
 		endpoint:   endpoint,
-		dockerhub:  dockerhub,
 		registries: filteredRegistries,
 		prune:      prune,
 		isAdmin:    securityContext.IsAdmin,
@@ -376,7 +366,7 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	handler.SwarmStackManager.Login(config.dockerhub, config.registries, config.endpoint)
+	handler.SwarmStackManager.Login(config.registries, config.endpoint)
 
 	err = handler.SwarmStackManager.Deploy(config.stack, config.prune, config.endpoint)
 	if err != nil {

api/http/handler/stacks/git.go

@@ -1,17 +0,0 @@
-package stacks
-
-type cloneRepositoryParameters struct {
-	url            string
-	referenceName  string
-	path           string
-	authentication bool
-	username       string
-	password       string
-}
-
-func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error {
-	if parameters.authentication {
-		return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password)
-	}
-	return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path)
-}

api/http/handler/stacks/handler.go

@@ -52,8 +52,12 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackInspect))).Methods(http.MethodGet)
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackDelete))).Methods(http.MethodDelete)
+	h.Handle("/stacks/{id}/associate",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.stackAssociate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut)
+	h.Handle("/stacks/{id}/git",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/file",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet)
 	h.Handle("/stacks/{id}/migrate",

api/http/handler/stacks/stack_associate.go

@@ -0,0 +1,91 @@
+package stacks
+
+import (
+	"fmt"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/stackutils"
+	"net/http"
+	"time"
+)
+
+// PUT request on /api/stacks/:id/associate?endpointId=<endpointId>&swarmId=<swarmId>&orphanedRunning=<orphanedRunning>
+func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+	}
+
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+	}
+
+	swarmId, err := request.RetrieveQueryParameter(r, "swarmId", true)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: swarmId", err}
+	}
+
+	orphanedRunning, err := request.RetrieveBooleanQueryParameter(r, "orphanedRunning", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: orphanedRunning", err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	user, err := handler.DataStore.User().User(securityContext.UserID)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to load user information from the database", err}
+	}
+
+	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+	}
+
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+	}
+
+	if resourceControl != nil {
+		resourceControl.ResourceID = fmt.Sprintf("%d_%s", endpointID, stack.Name)
+
+		err = handler.DataStore.ResourceControl().UpdateResourceControl(resourceControl.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist resource control changes inside the database", err}
+		}
+	}
+
+	stack.EndpointID = portainer.EndpointID(endpointID)
+	stack.SwarmID = swarmId
+
+	if orphanedRunning {
+		stack.Status = portainer.StackStatusActive
+	} else {
+		stack.Status = portainer.StackStatusInactive
+	}
+
+	stack.CreationDate = time.Now().Unix()
+	stack.CreatedBy = user.Username
+	stack.UpdateDate = 0
+	stack.UpdatedBy = ""
+
+	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+	}
+
+	stack.ResourceControl = resourceControl
+
+	return response.JSON(w, stack)
+}

api/http/handler/stacks/stack_create.go

@@ -2,6 +2,7 @@ package stacks
 
 import (
 	"errors"
+	"fmt"
 	"log"
 	"net/http"
 
@@ -12,9 +13,10 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	httperrors "github.com/portainer/portainer/api/http/errors"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
@@ -76,7 +78,7 @@ func (handler *Handler) stackCreate(w http.ResponseWriter, r *http.Request) *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	if !endpoint.SecuritySettings.AllowStackManagementForRegularUsers {
+	if endpointutils.IsDockerEndpoint(endpoint) && !endpoint.SecuritySettings.AllowStackManagementForRegularUsers {
 		securityContext, err := security.RetrieveRestrictedRequestContext(r)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve user info from request context", err}
@@ -110,11 +112,7 @@ func (handler *Handler) stackCreate(w http.ResponseWriter, r *http.Request) *htt
 	case portainer.DockerComposeStack:
 		return handler.createComposeStack(w, r, method, endpoint, tokenData.ID)
 	case portainer.KubernetesStack:
-		if tokenData.Role != portainer.AdministratorRole {
-			return &httperror.HandlerError{http.StatusForbidden, "Access denied", httperrors.ErrUnauthorized}
-		}
-
-		return handler.createKubernetesStack(w, r, endpoint)
+		return handler.createKubernetesStack(w, r, method, endpoint)
 	}
 
 	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: type. Value must be one of: 1 (Swarm stack) or 2 (Compose stack)", errors.New(request.ErrInvalidQueryParameter)}
@@ -147,6 +145,16 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request,
 	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)}
 }
 
+func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	switch method {
+	case "string":
+		return handler.createKubernetesStackFromFileContent(w, r, endpoint)
+	case "repository":
+		return handler.createKubernetesStackFromGitRepository(w, r, endpoint)
+	}
+	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string or repository", Err: errors.New(request.ErrInvalidQueryParameter)}
+}
+
 func (handler *Handler) isValidStackFile(stackFileContent []byte, securitySettings *portainer.EndpointSecuritySettings) error {
 	composeConfigYAML, err := loader.ParseYAML(stackFileContent)
 	if err != nil {
@@ -226,3 +234,22 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port
 	stack.ResourceControl = resourceControl
 	return response.JSON(w, stack)
 }
+
+func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error {
+	if !auth {
+		username = ""
+		password = ""
+	}
+
+	err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password)
+	if err != nil {
+		return fmt.Errorf("unable to clone git repository: %w", err)
+	}
+
+	stack.GitConfig = &gittypes.RepoConfig{
+		URL:            repositoryURL,
+		ReferenceName:  refName,
+		ConfigFilePath: configFilePath,
+	}
+	return nil
+}

api/http/handler/stacks/stack_create_test.go

@@ -0,0 +1,29 @@
+package stacks
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) {
+	handler := NewHandler(&security.RequestBouncer{})
+	handler.GitService = testhelpers.NewGitService()
+
+	url := "url"
+	refName := "ref"
+	configPath := "path"
+	stack := &portainer.Stack{}
+	err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "")
+	assert.NoError(t, err, "clone and save should not fail")
+
+	assert.Equal(t, gittypes.RepoConfig{
+		URL:            url,
+		ReferenceName:  refName,
+		ConfigFilePath: configPath,
+	}, *stack.GitConfig)
+}

api/http/handler/stacks/stack_delete.go

@@ -27,7 +27,7 @@ import (
 // @success 204 "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
-// @failure 404 " not found"
+// @failure 404 "Not found"
 // @failure 500 "Server error"
 // @router /stacks/{id} [delete]
 func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -58,42 +58,42 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
-	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
-	// The EndpointID property is not available for these stacks, this API endpoint
-	// can use the optional EndpointID query parameter to set a valid endpoint identifier to be
-	// used in the context of this request.
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
 	}
-	endpointIdentifier := stack.EndpointID
-	if endpointID != 0 {
-		endpointIdentifier = portainer.EndpointID(endpointID)
+
+	isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID
+
+	if isOrphaned && !securityContext.IsAdmin {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack")}
 	}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointIdentifier)
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
-
 	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
 	}
 
-	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-	}
-	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+	if !isOrphaned {
+		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		}
+
+		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		}
+		if !access {
+			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+		}
 	}
 
 	err = handler.deleteStack(stack, endpoint)

api/http/handler/stacks/stack_file.go

@@ -46,34 +46,38 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
-	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
-	}
-
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
-
-	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-	}
-
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
 	}
 
-	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
+	if err == bolterrors.ErrObjectNotFound {
+		if !securityContext.IsAdmin {
+			return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+		}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
-	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
+
+	if endpoint != nil {
+		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		}
+
+		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		}
+
+		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		}
+		if !access {
+			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
+		}
 	}
 
 	stackFileContent, err := handler.FileService.GetFileContent(path.Join(stack.ProjectPath, stack.EntryPoint))

api/http/handler/stacks/stack_inspect.go

@@ -1,6 +1,7 @@
 package stacks
 
 import (
+	"github.com/portainer/portainer/api/http/errors"
 	"net/http"
 
 	httperror "github.com/portainer/libhttp/error"
@@ -8,7 +9,6 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
@@ -40,38 +40,42 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+		if !securityContext.IsAdmin {
+			return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+		}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
+	if endpoint != nil {
+		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		}
 
-	securityContext, err := security.RetrieveRestrictedRequestContext(r)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve user info from request context", err}
-	}
+		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		}
 
-	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-	}
+		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		}
+		if !access {
+			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
+		}
 
-	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-	}
-	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", errors.ErrResourceAccessDenied}
-	}
-
-	if resourceControl != nil {
-		stack.ResourceControl = resourceControl
+		if resourceControl != nil {
+			stack.ResourceControl = resourceControl
+		}
 	}
 
 	return response.JSON(w, stack)

api/http/handler/stacks/stack_list.go

@@ -1,6 +1,7 @@
 package stacks
 
 import (
+	httperrors "github.com/portainer/portainer/api/http/errors"
 	"net/http"
 
 	httperror "github.com/portainer/libhttp/error"
@@ -12,8 +13,9 @@ import (
 )
 
 type stackListOperationFilters struct {
-	SwarmID    string `json:"SwarmID"`
-	EndpointID int    `json:"EndpointID"`
+	SwarmID               string `json:"SwarmID"`
+	EndpointID            int    `json:"EndpointID"`
+	IncludeOrphanedStacks bool   `json:"IncludeOrphanedStacks"`
 }
 
 // @id StackList
@@ -37,11 +39,16 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: filters", err}
 	}
 
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
+	}
+
 	stacks, err := handler.DataStore.Stack().Stacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve stacks from the database", err}
 	}
-	stacks = filterStacks(stacks, &filters)
+	stacks = filterStacks(stacks, &filters, endpoints)
 
 	resourceControls, err := handler.DataStore.ResourceControl().ResourceControls()
 	if err != nil {
@@ -56,6 +63,10 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 	stacks = authorization.DecorateStacks(stacks, resourceControls)
 
 	if !securityContext.IsAdmin {
+		if filters.IncludeOrphanedStacks {
+			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access orphaned stacks", httperrors.ErrUnauthorized}
+		}
+
 		user, err := handler.DataStore.User().User(securityContext.UserID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve user information from the database", err}
@@ -72,13 +83,20 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 	return response.JSON(w, stacks)
 }
 
-func filterStacks(stacks []portainer.Stack, filters *stackListOperationFilters) []portainer.Stack {
+func filterStacks(stacks []portainer.Stack, filters *stackListOperationFilters, endpoints []portainer.Endpoint) []portainer.Stack {
 	if filters.EndpointID == 0 && filters.SwarmID == "" {
 		return stacks
 	}
 
 	filteredStacks := make([]portainer.Stack, 0, len(stacks))
 	for _, stack := range stacks {
+		if filters.IncludeOrphanedStacks && isOrphanedStack(stack, endpoints) {
+			if (stack.Type == portainer.DockerComposeStack && filters.SwarmID == "") || (stack.Type == portainer.DockerSwarmStack && filters.SwarmID != "") {
+				filteredStacks = append(filteredStacks, stack)
+			}
+			continue
+		}
+
 		if stack.Type == portainer.DockerComposeStack && stack.EndpointID == portainer.EndpointID(filters.EndpointID) {
 			filteredStacks = append(filteredStacks, stack)
 		}
@@ -89,3 +107,13 @@ func filterStacks(stacks []portainer.Stack, filters *stackListOperationFilters)
 
 	return filteredStacks
 }
+
+func isOrphanedStack(stack portainer.Stack, endpoints []portainer.Endpoint) bool {
+	for _, endpoint := range endpoints {
+		if stack.EndpointID == endpoint.ID {
+			return false
+		}
+	}
+
+	return true
+}

api/http/handler/stacks/stack_start.go

@@ -26,7 +26,7 @@ import (
 // @success 200 {object} portainer.Stack "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
-// @failure 404 " not found"
+// @failure 404 "Not found"
 // @failure 500 "Server error"
 // @router /stacks/{id}/start [post]
 func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/stacks/stack_stop.go

@@ -24,7 +24,7 @@ import (
 // @success 200 {object} portainer.Stack "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
-// @failure 404 " not found"
+// @failure 404 "Not found"
 // @failure 500 "Server error"
 // @router /stacks/{id}/stop [post]
 func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/stacks/stack_update.go

@@ -61,7 +61,7 @@ func (payload *updateSwarmStackPayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.Stack "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
-// @failure 404 " not found"
+// @failure 404 "Not found"
 // @failure 500 "Server error"
 // @router /stacks/{id} [put]
 func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -160,6 +160,7 @@ func (handler *Handler) updateComposeStack(r *http.Request, stack *portainer.Sta
 
 	stack.UpdateDate = time.Now().Unix()
 	stack.UpdatedBy = config.user.Username
+	stack.Status = portainer.StackStatusActive
 
 	err = handler.deployComposeStack(config)
 	if err != nil {
@@ -191,6 +192,7 @@ func (handler *Handler) updateSwarmStack(r *http.Request, stack *portainer.Stack
 
 	stack.UpdateDate = time.Now().Unix()
 	stack.UpdatedBy = config.user.Username
+	stack.Status = portainer.StackStatusActive
 
 	err = handler.deploySwarmStack(config)
 	if err != nil {

api/http/handler/stacks/stack_update_git.go

@@ -0,0 +1,196 @@
+package stacks
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/filesystem"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/stackutils"
+)
+
+type updateStackGitPayload struct {
+	RepositoryReferenceName  string
+	RepositoryAuthentication bool
+	RepositoryUsername       string
+	RepositoryPassword       string
+}
+
+func (payload *updateStackGitPayload) Validate(r *http.Request) error {
+	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
+		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	}
+	return nil
+}
+
+// @id StackUpdateGit
+// @summary Redeploy a stack
+// @description Pull and redeploy a stack via Git
+// @description **Access policy**: restricted
+// @tags stacks
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "Stack identifier"
+// @param endpointId query int false "Stacks created before version 1.18.0 might not have an associated endpoint identifier. Use this optional parameter to set the endpoint identifier used by the stack."
+// @param body body updateStackGitPayload true "Git configs for pull and redeploy a stack"
+// @success 200 {object} portainer.Stack "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied"
+// @failure 404 "Not found"
+// @failure 500 "Server error"
+// @router /stacks/{id}/git [put]
+func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+	}
+
+	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+	}
+
+	if stack.GitConfig == nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err}
+	}
+
+	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
+	// The EndpointID property is not available for these stacks, this API endpoint
+	// can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack.
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+	}
+	if endpointID != int(stack.EndpointID) {
+		stack.EndpointID = portainer.EndpointID(endpointID)
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+	}
+	if !access {
+		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+	}
+
+	var payload updateStackGitPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+
+	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
+	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err}
+	}
+
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
+	}
+
+	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
+		if restoreError != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError)
+		}
+
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
+	}
+
+	defer func() {
+		err = handler.FileService.RemoveDirectory(backupProjectPath)
+		if err != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err)
+		}
+	}()
+
+	httpErr := handler.deployStack(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+	}
+
+	return response.JSON(w, stack)
+}
+
+func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	if stack.Type == portainer.DockerSwarmStack {
+		config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
+		if httpErr != nil {
+			return httpErr
+		}
+
+		err := handler.deploySwarmStack(config)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		}
+
+		stack.UpdateDate = time.Now().Unix()
+		stack.UpdatedBy = config.user.Username
+		stack.Status = portainer.StackStatusActive
+
+		return nil
+	}
+
+	config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err := handler.deployComposeStack(config)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+	}
+
+	stack.UpdateDate = time.Now().Unix()
+	stack.UpdatedBy = config.user.Username
+	stack.Status = portainer.StackStatusActive
+
+	return nil
+}

api/http/handler/teams/team_delete.go

@@ -3,11 +3,12 @@ package teams
 import (
 	"net/http"
 
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 )
 
 // @id TeamDelete
@@ -29,7 +30,7 @@ func (handler *Handler) teamDelete(w http.ResponseWriter, r *http.Request) *http
 	}
 
 	_, err = handler.DataStore.Team().Team(portainer.TeamID(teamID))
-	if err == errors.ErrObjectNotFound {
+	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a team with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a team with the specified identifier inside the database", err}
@@ -45,5 +46,27 @@ func (handler *Handler) teamDelete(w http.ResponseWriter, r *http.Request) *http
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to delete associated team memberships from the database", err}
 	}
 
+	// update default team if deleted team was default
+	err = handler.updateDefaultTeamIfDeleted(portainer.TeamID(teamID))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to reset default team", err}
+	}
+
 	return response.Empty(w)
 }
+
+// updateDefaultTeamIfDeleted resets the default team to nil if default team was the deleted team
+func (handler *Handler) updateDefaultTeamIfDeleted(teamID portainer.TeamID) error {
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return errors.Wrap(err, "failed to fetch settings")
+	}
+
+	if teamID != settings.OAuthSettings.DefaultTeamID {
+		return nil
+	}
+
+	settings.OAuthSettings.DefaultTeamID = 0
+	err = handler.DataStore.Settings().UpdateSettings(settings)
+	return errors.Wrap(err, "failed to update settings")
+}

api/http/handler/templates/git.go

@@ -1,17 +0,0 @@
-package templates
-
-type cloneRepositoryParameters struct {
-	url            string
-	referenceName  string
-	path           string
-	authentication bool
-	username       string
-	password       string
-}
-
-func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error {
-	if parameters.authentication {
-		return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password)
-	}
-	return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path)
-}

api/http/handler/templates/template_file.go

@@ -63,12 +63,7 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht
 
 	defer handler.cleanUp(projectPath)
 
-	gitCloneParams := &cloneRepositoryParameters{
-		url:  payload.RepositoryURL,
-		path: projectPath,
-	}
-
-	err = handler.cloneGitRepository(gitCloneParams)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
 	}

api/http/proxy/factory/azure/containergroup.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy/factory/responseutils"
+	"github.com/portainer/portainer/api/http/proxy/factory/utils"
 )
 
 // proxy for /subscriptions/*/resourceGroups/*/providers/Microsoft.ContainerInstance/containerGroups/*
@@ -23,12 +23,42 @@ func (transport *Transport) proxyContainerGroupRequest(request *http.Request) (*
 }
 
 func (transport *Transport) proxyContainerGroupPutRequest(request *http.Request) (*http.Response, error) {
+	//add a lock before processing existense check
+	transport.mutex.Lock()
+	defer transport.mutex.Unlock()
+
+	//generate a temp http GET request based on the current PUT request
+	validationRequest := &http.Request{
+		Method: http.MethodGet,
+		URL:    request.URL,
+		Header: http.Header{
+			"Authorization": []string{request.Header.Get("Authorization")},
+		},
+	}
+
+	//fire the request to Azure API to validate if there is an existing container instance with the same name
+	//positive - reject the request
+	//negative - continue the process
+	validationResponse, err := http.DefaultTransport.RoundTrip(validationRequest)
+	if err != nil {
+		return validationResponse, err
+	}
+
+	if validationResponse.StatusCode >= 200 && validationResponse.StatusCode < 300 {
+		resp := &http.Response{}
+		errObj := map[string]string{
+			"message": "A container instance with the same name already exists inside the selected resource group",
+		}
+		err = utils.RewriteResponse(resp, errObj, http.StatusConflict)
+		return resp, err
+	}
+
 	response, err := http.DefaultTransport.RoundTrip(request)
 	if err != nil {
 		return response, err
 	}
 
-	responseObject, err := responseutils.GetResponseAsJSONObject(response)
+	responseObject, err := utils.GetResponseAsJSONObject(response)
 	if err != nil {
 		return response, err
 	}
@@ -50,7 +80,7 @@ func (transport *Transport) proxyContainerGroupPutRequest(request *http.Request)
 
 	responseObject = decorateObject(responseObject, resourceControl)
 
-	err = responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+	err = utils.RewriteResponse(response, responseObject, http.StatusOK)
 	if err != nil {
 		return response, err
 	}
@@ -64,7 +94,7 @@ func (transport *Transport) proxyContainerGroupGetRequest(request *http.Request)
 		return response, err
 	}
 
-	responseObject, err := responseutils.GetResponseAsJSONObject(response)
+	responseObject, err := utils.GetResponseAsJSONObject(response)
 	if err != nil {
 		return nil, err
 	}
@@ -76,7 +106,7 @@ func (transport *Transport) proxyContainerGroupGetRequest(request *http.Request)
 
 	responseObject = transport.decorateContainerGroup(responseObject, context)
 
-	responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+	utils.RewriteResponse(response, responseObject, http.StatusOK)
 
 	return response, nil
 }
@@ -88,7 +118,7 @@ func (transport *Transport) proxyContainerGroupDeleteRequest(request *http.Reque
 	}
 
 	if !transport.userCanDeleteContainerGroup(request, context) {
-		return responseutils.WriteAccessDeniedResponse()
+		return utils.WriteAccessDeniedResponse()
 	}
 
 	response, err := http.DefaultTransport.RoundTrip(request)
@@ -96,14 +126,14 @@ func (transport *Transport) proxyContainerGroupDeleteRequest(request *http.Reque
 		return response, err
 	}
 
-	responseObject, err := responseutils.GetResponseAsJSONObject(response)
+	responseObject, err := utils.GetResponseAsJSONObject(response)
 	if err != nil {
 		return nil, err
 	}
 
 	transport.removeResourceControl(responseObject, context)
 
-	responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+	utils.RewriteResponse(response, responseObject, http.StatusOK)
 
 	return response, nil
 }

api/http/proxy/factory/azure/containergroups.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/portainer/portainer/api/http/proxy/factory/responseutils"
+	"github.com/portainer/portainer/api/http/proxy/factory/utils"
 )
 
 // proxy for /subscriptions/*/providers/Microsoft.ContainerInstance/containerGroups
@@ -23,7 +23,7 @@ func (transport *Transport) proxyContainerGroupsGetRequest(request *http.Request
 		return nil, err
 	}
 
-	responseObject, err := responseutils.GetResponseAsJSONObject(response)
+	responseObject, err := utils.GetResponseAsJSONObject(response)
 	if err != nil {
 		return nil, err
 	}
@@ -39,10 +39,10 @@ func (transport *Transport) proxyContainerGroupsGetRequest(request *http.Request
 		filteredValue := transport.filterContainerGroups(decoratedValue, context)
 		responseObject["value"] = filteredValue
 
-		responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+		utils.RewriteResponse(response, responseObject, http.StatusOK)
 	} else {
 		return nil, fmt.Errorf("The container groups response has no value property")
 	}
 
 	return response, nil
-}
+}

api/http/proxy/factory/docker/access_control.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/portainer/portainer/api/internal/stackutils"
 
-	"github.com/portainer/portainer/api/http/proxy/factory/responseutils"
+	"github.com/portainer/portainer/api/http/proxy/factory/utils"
 	"github.com/portainer/portainer/api/internal/authorization"
 
 	portainer "github.com/portainer/portainer/api"
@@ -162,7 +162,7 @@ func (transport *Transport) applyAccessControlOnResource(parameters *resourceOpe
 		systemResourceControl := findSystemNetworkResourceControl(responseObject)
 		if systemResourceControl != nil {
 			responseObject = decorateObject(responseObject, systemResourceControl)
-			return responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+			return utils.RewriteResponse(response, responseObject, http.StatusOK)
 		}
 	}
 
@@ -175,15 +175,15 @@ func (transport *Transport) applyAccessControlOnResource(parameters *resourceOpe
 	}
 
 	if resourceControl == nil && (executor.operationContext.isAdmin) {
-		return responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+		return utils.RewriteResponse(response, responseObject, http.StatusOK)
 	}
 
 	if executor.operationContext.isAdmin || (resourceControl != nil && authorization.UserCanAccessResource(executor.operationContext.userID, executor.operationContext.userTeamIDs, resourceControl)) {
 		responseObject = decorateObject(responseObject, resourceControl)
-		return responseutils.RewriteResponse(response, responseObject, http.StatusOK)
+		return utils.RewriteResponse(response, responseObject, http.StatusOK)
 	}
 
-	return responseutils.RewriteAccessDeniedResponse(response)
+	return utils.RewriteAccessDeniedResponse(response)
 }
 
 func (transport *Transport) applyAccessControlOnResourceList(parameters *resourceOperationParameters, resourceData []interface{}, executor *operationExecutor) ([]interface{}, error) {

api/http/proxy/factory/docker/configs.go

@@ -7,7 +7,7 @@ import (
 	"github.com/docker/docker/client"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy/factory/responseutils"
+	"github.com/portainer/portainer/api/http/proxy/factory/utils"
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
@@ -34,7 +34,7 @@ func getInheritedResourceControlFromConfigLabels(dockerClient *client.Client, en
 func (transport *Transport) configListOperation(response *http.Response, executor *operationExecutor) error {
 	// ConfigList response is a JSON array
 	// https://docs.docker.com/engine/api/v1.30/#operation/ConfigList
-	responseArray, err := responseutils.GetResponseAsJSONArray(response)
+	responseArray, err := utils.GetResponseAsJSONArray(response)
 	if err != nil {
 		return err
 	}
@@ -50,7 +50,7 @@ func (transport *Transport) configListOperation(response *http.Response, executo
 		return err
 	}
 
-	return responseutils.RewriteResponse(response, responseArray, http.StatusOK)
+	return utils.RewriteResponse(response, responseArray, http.StatusOK)
 }
 
 // configInspectOperation extracts the response as a JSON object, verify that the user
@@ -58,7 +58,7 @@ func (transport *Transport) configListOperation(response *http.Response, executo
 func (transport *Transport) configInspectOperation(response *http.Response, executor *operationExecutor) error {
 	// ConfigInspect response is a JSON object
 	// https://docs.docker.com/engine/api/v1.30/#operation/ConfigInspect
-	responseObject, err := responseutils.GetResponseAsJSONObject(response)
+	responseObject, err := utils.GetResponseAsJSONObject(response)
 	if err != nil {
 		return err
 	}
@@ -78,9 +78,9 @@ func (transport *Transport) configInspectOperation(response *http.Response, exec
 // https://docs.docker.com/engine/api/v1.37/#operation/ConfigList
 // https://docs.docker.com/engine/api/v1.37/#operation/ConfigInspect
 func selectorConfigLabels(responseObject map[string]interface{}) map[string]interface{} {
-	secretSpec := responseutils.GetJSONObject(responseObject, "Spec")
+	secretSpec := utils.GetJSONObject(responseObject, "Spec")
 	if secretSpec != nil {
-		secretLabelsObject := responseutils.GetJSONObject(secretSpec, "Labels")
+		secretLabelsObject := utils.GetJSONObject(secretSpec, "Labels")
 		return secretLabelsObject
 	}
 	return nil