fix(edgejobs): fix data race in edge jobs tasks collect EE-4766

Co-authored-by: Matt Hook <hookenz@gmail.com>

.eslintrc.yml

@@ -99,12 +99,17 @@ overrides:
       '@typescript-eslint/explicit-module-boundary-types': off
       '@typescript-eslint/no-unused-vars': 'error'
       '@typescript-eslint/no-explicit-any': 'error'
-      'jsx-a11y/label-has-associated-control': ['error', { 'assert': 'either' }]
+      'jsx-a11y/label-has-associated-control': ['error', { 'assert': 'either', controlComponents: ['Input', 'Checkbox'] }]
       'react/function-component-definition': ['error', { 'namedComponents': 'function-declaration' }]
       'react/jsx-no-bind': off
       'no-await-in-loop': 'off'
       'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
-      'regex/invalid': ['error', [{ 'regex': 'data-feather="(.*)"', 'message': 'Please use `react-feather` package instead' }]]
+      'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
+    overrides: # allow props spreading for hoc files
+      - files:
+          - app/**/with*.ts{,x}
+        rules:
+          'react/jsx-props-no-spreading': off
   - files:
       - app/**/*.test.*
     extends:

api/chisel/schedules.go

@@ -31,7 +31,6 @@ func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
 
 	for _, tunnel := range service.tunnelDetailsMap {
-		// Filter in-place
 		n := 0
 		for _, edgeJob := range tunnel.Jobs {
 			if edgeJob.ID != edgeJobID {
@@ -45,3 +44,20 @@ func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 
 	service.mu.Unlock()
 }
+
+func (service *Service) RemoveEdgeJobFromEndpoint(endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID) {
+	service.mu.Lock()
+	tunnel := service.getTunnelDetails(endpointID)
+
+	n := 0
+	for _, edgeJob := range tunnel.Jobs {
+		if edgeJob.ID != edgeJobID {
+			tunnel.Jobs[n] = edgeJob
+			n++
+		}
+	}
+
+	tunnel.Jobs = tunnel.Jobs[:n]
+
+	service.mu.Unlock()
+}

api/cli/confirm.go

@@ -2,14 +2,14 @@ package cli
 
 import (
 	"bufio"
-	"log"
+	"fmt"
 	"os"
 	"strings"
 )
 
 // Confirm starts a rollback db cli application
 func Confirm(message string) (bool, error) {
-	log.Printf("%s [y/N]", message)
+	fmt.Printf("%s [y/N]", message)
 
 	reader := bufio.NewReader(os.Stdin)
 	answer, err := reader.ReadString('\n')

api/cmd/portainer/main.go

@@ -8,9 +8,9 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"time"
 
-	"github.com/portainer/libhelm"
+	libstack "github.com/portainer/docker-compose-wrapper"
+	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/build"
@@ -19,6 +19,7 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/database"
 	"github.com/portainer/portainer/api/database/boltdb"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/demo"
@@ -33,8 +34,10 @@ import (
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/edge/edgestacks"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/ssl"
+	"github.com/portainer/portainer/api/internal/upgrade"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
@@ -42,7 +45,9 @@ import (
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
+	"github.com/portainer/portainer/pkg/libhelm"
 
+	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
 )
 
@@ -98,8 +103,6 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 
 		log.Info().Msg("exiting rollback")
 		os.Exit(0)
-
-		return nil
 	}
 
 	// Init sets some defaults - it's basically a migration
@@ -109,24 +112,27 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	}
 
 	if isNew {
-		// from MigrateData
-		store.VersionService.StoreDBVersion(portainer.DBVersion)
+		instanceId, err := uuid.NewV4()
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed generating instance id")
+		}
 
-		err := updateSettingsFromFlags(store, flags)
+		// from MigrateData
+		v := models.Version{
+			SchemaVersion: portainer.APIVersion,
+			Edition:       int(portainer.PortainerCE),
+			InstanceID:    instanceId.String(),
+		}
+		store.VersionService.UpdateVersion(&v)
+
+		err = updateSettingsFromFlags(store, flags)
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed updating settings from flags")
 		}
 	} else {
-		storedVersion, err := store.VersionService.DBVersion()
+		err = store.MigrateData()
 		if err != nil {
-			log.Fatal().Err(err).Msg("failure during creation of new database")
-		}
-
-		if storedVersion != portainer.DBVersion {
-			err = store.MigrateData()
-			if err != nil {
-				log.Fatal().Err(err).Msg("failed migration")
-			}
+			log.Fatal().Err(err).Msg("failed migration")
 		}
 	}
 
@@ -139,22 +145,13 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	go func() {
 		<-shutdownCtx.Done()
 		defer connection.Close()
-
-		exportFilename := path.Join(*flags.Data, fmt.Sprintf("export-%d.json", time.Now().Unix()))
-
-		err := store.Export(exportFilename)
-		if err != nil {
-			log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
-		} else {
-			log.Debug().Str("filename", exportFilename).Msg("exported")
-		}
 	}()
 
 	return store
 }
 
-func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
+func initComposeStackManager(composeDeployer libstack.Deployer, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+	composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating compose manager")
 	}
@@ -594,6 +591,8 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	digitalSignatureService := initDigitalSignatureService()
 
+	edgeStacksService := edgestacks.NewService(dataStore)
+
 	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
 	if err != nil {
 		log.Fatal().Err(err).Msg("")
@@ -633,7 +632,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	dockerConfigPath := fileService.GetDockerConfigPath()
 
-	composeStackManager := initComposeStackManager(*flags.Assets, dockerConfigPath, reverseTunnelService, proxyManager)
+	composeDeployer, err := compose.NewComposeDeployer(*flags.Assets, dockerConfigPath)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing compose deployer")
+	}
+
+	composeStackManager := initComposeStackManager(composeDeployer, reverseTunnelService, proxyManager)
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
@@ -719,6 +723,28 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
+	upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing upgrade service")
+	}
+
+	// FIXME: In 2.16 we changed the way ingress controller permissions are
+	// stored. Instead of being stored as annotation on an ingress rule, we keep
+	// them in our database. However, in order to run the migration we need an
+	// admin kube client to run lookup the old ingress rules and compare them
+	// with the current existing ingress classes.
+	//
+	// Unfortunately, our migrations run as part of the database initialization
+	// and our kubeclients require an initialized database. So it is not
+	// possible to do this migration as part of our normal flow. We DO have a
+	// migration which toggles a boolean in kubernetes configuration that
+	// indicated that this "post init" migration should be run. If/when this is
+	// resolved we can remove this function.
+	err = kubernetesClientFactory.PostInitMigrateIngresses()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failure during creation of new database")
+	}
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
@@ -728,6 +754,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		HTTPEnabled:                 sslDBSettings.HTTPEnabled,
 		AssetsPath:                  *flags.Assets,
 		DataStore:                   dataStore,
+		EdgeStacksService:           edgeStacksService,
 		SwarmStackManager:           swarmStackManager,
 		ComposeStackManager:         composeStackManager,
 		KubernetesDeployer:          kubernetesDeployer,
@@ -753,6 +780,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ShutdownTrigger:             shutdownTrigger,
 		StackDeployer:               stackDeployer,
 		DemoService:                 demoService,
+		UpgradeService:              upgradeService,
 	}
 }
 

api/connection.go

@@ -26,7 +26,7 @@ type Connection interface {
 	UpdateObject(bucketName string, key []byte, object interface{}) error
 	UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
 	DeleteObject(bucketName string, key []byte) error
-	DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error
+	DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error
 	GetNextIdentifier(bucketName string) int
 	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
 	CreateObjectWithId(bucketName string, id int, obj interface{}) error

api/database/boltdb/db.go

@@ -267,13 +267,12 @@ func (connection *DbConnection) DeleteObject(bucketName string, key []byte) erro
 
 // DeleteAllObjects delete all objects where matching() returns (id, ok).
 // TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
-func (connection *DbConnection) DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error {
+func (connection *DbConnection) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
 	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var obj interface{}
 			err := connection.UnmarshalObject(v, &obj)
 			if err != nil {
 				return err
@@ -422,7 +421,6 @@ func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error)
 	err := connection.View(func(tx *bolt.Tx) error {
 		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
 			bucketName := string(name)
-			bucket = tx.Bucket([]byte(bucketName))
 			seqId := bucket.Sequence()
 			buckets[bucketName] = int(seqId)
 			return nil

api/database/boltdb/export.go

@@ -14,7 +14,6 @@ func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
 	err := connection.View(func(tx *bolt.Tx) error {
 		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
 			bucketName := string(name)
-			bucket = tx.Bucket([]byte(bucketName))
 			seqId := bucket.Sequence()
 			buckets[bucketName] = int(seqId)
 			return nil

api/database/models/version.go

@@ -0,0 +1,8 @@
+package models
+
+type Version struct {
+	SchemaVersion string
+	MigratorCount int
+	Edition       int
+	InstanceID    string
+}

api/dataservices/edgegroup/edgegroup.go

@@ -8,10 +8,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "edgegroups"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "edgegroups"
 
 // Service represents a service for managing Edge group data.
 type Service struct {
@@ -68,12 +66,22 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 	return &group, nil
 }
 
-// UpdateEdgeGroup updates an Edge group.
+// Deprecated: Use UpdateEdgeGroupFunc instead.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, group)
 }
 
+// UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
+func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
+	id := service.connection.ConvertToKey(int(ID))
+	edgeGroup := &portainer.EdgeGroup{}
+
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
+		updateFunc(edgeGroup)
+	})
+}
+
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
 	identifier := service.connection.ConvertToKey(int(ID))

api/dataservices/edgejob/edgejob.go

@@ -80,12 +80,22 @@ func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJo
 	)
 }
 
-// UpdateEdgeJob updates an Edge job by ID
+// Deprecated: use UpdateEdgeJobFunc instead
 func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
 }
 
+// UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
+func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
+	id := service.connection.ConvertToKey(int(ID))
+	edgeJob := &portainer.EdgeJob{}
+
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+		updateFunc(edgeJob)
+	})
+}
+
 // DeleteEdgeJob deletes an Edge job
 func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := service.connection.ConvertToKey(int(ID))

api/dataservices/edgestack/edgestack.go

@@ -82,12 +82,22 @@ func (service *Service) Create(id portainer.EdgeStackID, edgeStack *portainer.Ed
 	)
 }
 
-// UpdateEdgeStack updates an Edge stack.
+// Deprecated: Use UpdateEdgeStackFunc instead.
 func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, edgeStack)
 }
 
+// UpdateEdgeStackFunc updates an Edge stack inside a transaction avoiding data races.
+func (service *Service) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
+	id := service.connection.ConvertToKey(int(ID))
+	edgeStack := &portainer.EdgeStack{}
+
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeStack, func() {
+		updateFunc(edgeStack)
+	})
+}
+
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	identifier := service.connection.ConvertToKey(int(ID))

api/dataservices/edgeupdateschedule/edgeupdateschedule.go

@@ -1,186 +0,0 @@
-package edgeupdateschedule
-
-import (
-	"fmt"
-	"sync"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/edgetypes"
-
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
-)
-
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "edge_update_schedule"
-)
-
-// Service represents a service for managing Edge Update Schedule data.
-type Service struct {
-	connection portainer.Connection
-
-	mu                 sync.Mutex
-	idxActiveSchedules map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
-}
-
-// NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	service := &Service{
-		connection: connection,
-	}
-
-	service.idxActiveSchedules = map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation{}
-
-	schedules, err := service.List()
-	if err != nil {
-		return nil, errors.WithMessage(err, "Unable to list schedules")
-	}
-
-	for _, schedule := range schedules {
-		service.setRelation(&schedule)
-	}
-
-	return service, nil
-}
-
-func (service *Service) ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	return service.idxActiveSchedules[environmentID]
-}
-
-func (service *Service) ActiveSchedules(environmentsIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	schedules := []edgetypes.EndpointUpdateScheduleRelation{}
-
-	for _, environmentID := range environmentsIDs {
-		if s, ok := service.idxActiveSchedules[environmentID]; ok {
-			schedules = append(schedules, *s)
-		}
-	}
-
-	return schedules
-}
-
-// List return an array containing all the items in the bucket.
-func (service *Service) List() ([]edgetypes.UpdateSchedule, error) {
-	var list = make([]edgetypes.UpdateSchedule, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&edgetypes.UpdateSchedule{},
-		func(obj interface{}) (interface{}, error) {
-			item, ok := obj.(*edgetypes.UpdateSchedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeUpdateSchedule object")
-				return nil, fmt.Errorf("Failed to convert to EdgeUpdateSchedule object: %s", obj)
-			}
-			list = append(list, *item)
-			return &edgetypes.UpdateSchedule{}, nil
-		})
-
-	return list, err
-}
-
-// Item returns a item by ID.
-func (service *Service) Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error) {
-	var item edgetypes.UpdateSchedule
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &item)
-	if err != nil {
-		return nil, err
-	}
-
-	return &item, nil
-}
-
-// Create assign an ID to a new object and saves it.
-func (service *Service) Create(item *edgetypes.UpdateSchedule) error {
-	err := service.connection.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			item.ID = edgetypes.UpdateScheduleID(id)
-			return int(item.ID), item
-		},
-	)
-
-	if err != nil {
-		return err
-	}
-
-	return service.setRelation(item)
-}
-
-// Update updates an item.
-func (service *Service) Update(id edgetypes.UpdateScheduleID, item *edgetypes.UpdateSchedule) error {
-	identifier := service.connection.ConvertToKey(int(id))
-	err := service.connection.UpdateObject(BucketName, identifier, item)
-	if err != nil {
-		return err
-	}
-
-	service.cleanRelation(id)
-
-	return service.setRelation(item)
-}
-
-// Delete deletes an item.
-func (service *Service) Delete(id edgetypes.UpdateScheduleID) error {
-
-	service.cleanRelation(id)
-
-	identifier := service.connection.ConvertToKey(int(id))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
-func (service *Service) cleanRelation(id edgetypes.UpdateScheduleID) {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	for _, schedule := range service.idxActiveSchedules {
-		if schedule != nil && schedule.ScheduleID == id {
-			delete(service.idxActiveSchedules, schedule.EnvironmentID)
-		}
-	}
-}
-
-func (service *Service) setRelation(schedule *edgetypes.UpdateSchedule) error {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	for environmentID, environmentStatus := range schedule.Status {
-		if environmentStatus.Status != edgetypes.UpdateScheduleStatusPending {
-			continue
-		}
-
-		// this should never happen
-		if service.idxActiveSchedules[environmentID] != nil && service.idxActiveSchedules[environmentID].ScheduleID != schedule.ID {
-			return errors.New("Multiple schedules are pending for the same environment")
-		}
-
-		service.idxActiveSchedules[environmentID] = &edgetypes.EndpointUpdateScheduleRelation{
-			EnvironmentID: environmentID,
-			ScheduleID:    schedule.ID,
-			TargetVersion: environmentStatus.TargetVersion,
-			Status:        environmentStatus.Status,
-			Error:         environmentStatus.Error,
-			Type:          schedule.Type,
-		}
-	}
-
-	return nil
-}

api/dataservices/errors/errors.go

@@ -4,7 +4,8 @@ import "errors"
 
 var (
 	// TODO: i'm pretty sure this needs wrapping at several levels
-	ErrObjectNotFound = errors.New("object not found inside the database")
-	ErrWrongDBEdition = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
-	ErrDBImportFailed = errors.New("importing backup failed")
+	ErrObjectNotFound     = errors.New("object not found inside the database")
+	ErrWrongDBEdition     = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrDBImportFailed     = errors.New("importing backup failed")
+	ErrDatabaseIsUpdating = errors.New("database is currently in updating state. Failed prior upgrade. Please restore from backup or delete the database and restart Portainer")
 )

api/dataservices/interface.go

@@ -6,8 +6,8 @@ import (
 	"io"
 	"time"
 
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/portainer/portainer/api/edgetypes"
 
 	portainer "github.com/portainer/portainer/api"
 )
@@ -28,7 +28,6 @@ type (
 		EdgeGroup() EdgeGroupService
 		EdgeJob() EdgeJobService
 		EdgeStack() EdgeStackService
-		EdgeUpdateSchedule() EdgeUpdateScheduleService
 		Endpoint() EndpointService
 		EndpointGroup() EndpointGroupService
 		EndpointRelation() EndpointRelationService
@@ -68,6 +67,7 @@ type (
 		EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error)
 		Create(group *portainer.EdgeGroup) error
 		UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error
+		UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(group *portainer.EdgeGroup)) error
 		DeleteEdgeGroup(ID portainer.EdgeGroupID) error
 		BucketName() string
 	}
@@ -78,28 +78,19 @@ type (
 		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
 		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
 		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
 		BucketName() string
 	}
 
-	EdgeUpdateScheduleService interface {
-		ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation
-		ActiveSchedules(environmentIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation
-		List() ([]edgetypes.UpdateSchedule, error)
-		Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error)
-		Create(edgeUpdateSchedule *edgetypes.UpdateSchedule) error
-		Update(ID edgetypes.UpdateScheduleID, edgeUpdateSchedule *edgetypes.UpdateSchedule) error
-		Delete(ID edgetypes.UpdateScheduleID) error
-		BucketName() string
-	}
-
 	// EdgeStackService represents a service to manage Edge stacks
 	EdgeStackService interface {
 		EdgeStacks() ([]portainer.EdgeStack, error)
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
 		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
+		UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
 		GetNextIdentifier() int
 		BucketName() string
@@ -279,6 +270,7 @@ type (
 		DeleteTeamMembershipByUserID(userID portainer.UserID) error
 		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
 		BucketName() string
+		DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error
 	}
 
 	// TunnelServerService represents a service for managing data associated to the tunnel server
@@ -302,12 +294,11 @@ type (
 
 	// VersionService represents a service for managing version data
 	VersionService interface {
-		DBVersion() (int, error)
 		Edition() (portainer.SoftwareEdition, error)
 		InstanceID() (string, error)
-		StoreDBVersion(version int) error
-		StoreInstanceID(ID string) error
-		BucketName() string
+		UpdateInstanceID(ID string) error
+		Version() (*models.Version, error)
+		UpdateVersion(*models.Version) error
 	}
 
 	// WebhookService represents a service for managing webhook data.

api/dataservices/tag/tag.go

@@ -91,11 +91,9 @@ func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *p
 	id := service.connection.ConvertToKey(int(ID))
 	tag := &portainer.Tag{}
 
-	service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
+	return service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
 		updateFunc(tag)
 	})
-
-	return nil
 }
 
 // DeleteTag deletes a tag.

api/dataservices/teammembership/teammembership.go

@@ -144,6 +144,7 @@ func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) erro
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
 	return service.connection.DeleteAllObjects(
 		BucketName,
+		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
@@ -164,6 +165,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
 	return service.connection.DeleteAllObjects(
 		BucketName,
+		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
@@ -179,3 +181,23 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 			return -1, false
 		})
 }
+
+func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
+	return service.connection.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.TeamID == teamID && membership.UserID == userID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
+}

api/dataservices/version/version.go

@@ -1,17 +1,18 @@
 package version
 
 import (
-	"strconv"
+	"errors"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
 const (
 	// BucketName represents the name of the bucket where this service stores data.
 	BucketName  = "version"
-	versionKey  = "DB_VERSION"
-	instanceKey = "INSTANCE_ID"
-	editionKey  = "EDITION"
+	versionKey  = "VERSION"
 	updatingKey = "DB_UPDATING"
 )
 
@@ -20,10 +21,6 @@ type Service struct {
 	connection portainer.Connection
 }
 
-func (service *Service) BucketName() string {
-	return BucketName
-}
-
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
@@ -36,56 +33,87 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-// DBVersion retrieves the stored database version.
-func (service *Service) DBVersion() (int, error) {
-	var version string
-	err := service.connection.GetObject(BucketName, []byte(versionKey), &version)
+func (service *Service) SchemaVersion() (string, error) {
+	v, err := service.Version()
 	if err != nil {
-		return 0, err
+		return "", err
 	}
-	return strconv.Atoi(version)
+
+	return v.SchemaVersion, nil
+}
+
+func (service *Service) UpdateSchemaVersion(version string) error {
+	v, err := service.Version()
+	if err != nil {
+		return err
+	}
+
+	v.SchemaVersion = version
+	return service.UpdateVersion(v)
 }
 
-// Edition retrieves the stored portainer edition.
 func (service *Service) Edition() (portainer.SoftwareEdition, error) {
-	var edition string
-	err := service.connection.GetObject(BucketName, []byte(editionKey), &edition)
+	v, err := service.Version()
 	if err != nil {
 		return 0, err
 	}
-	e, err := strconv.Atoi(edition)
-	if err != nil {
-		return 0, err
-	}
-	return portainer.SoftwareEdition(e), nil
-}
 
-// StoreDBVersion store the database version.
-func (service *Service) StoreDBVersion(version int) error {
-	return service.connection.UpdateObject(BucketName, []byte(versionKey), strconv.Itoa(version))
+	return portainer.SoftwareEdition(v.Edition), nil
 }
 
 // IsUpdating retrieves the database updating status.
 func (service *Service) IsUpdating() (bool, error) {
 	var isUpdating bool
 	err := service.connection.GetObject(BucketName, []byte(updatingKey), &isUpdating)
+	if err != nil && errors.Is(err, dserrors.ErrObjectNotFound) {
+		return false, nil
+	}
 	return isUpdating, err
 }
 
 // StoreIsUpdating store the database updating status.
 func (service *Service) StoreIsUpdating(isUpdating bool) error {
-	return service.connection.UpdateObject(BucketName, []byte(updatingKey), isUpdating)
+	return service.connection.DeleteObject(BucketName, []byte(updatingKey))
 }
 
 // InstanceID retrieves the stored instance ID.
 func (service *Service) InstanceID() (string, error) {
-	var id string
-	err := service.connection.GetObject(BucketName, []byte(instanceKey), &id)
-	return id, err
+	v, err := service.Version()
+	if err != nil {
+		return "", err
+	}
+
+	return v.InstanceID, nil
 }
 
 // StoreInstanceID store the instance ID.
-func (service *Service) StoreInstanceID(ID string) error {
-	return service.connection.UpdateObject(BucketName, []byte(instanceKey), ID)
+func (service *Service) UpdateInstanceID(id string) error {
+	v, err := service.Version()
+	if err != nil {
+		if !dataservices.IsErrObjectNotFound(err) {
+			return err
+		}
 
+		v = &models.Version{}
+	}
+
+	v.InstanceID = id
+	return service.UpdateVersion(v)
+}
+
+// Version retrieve the version object.
+func (service *Service) Version() (*models.Version, error) {
+	var v models.Version
+
+	err := service.connection.GetObject(BucketName, []byte(versionKey), &v)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v, nil
+}
+
+// UpdateVersion persists a Version object.
+func (service *Service) UpdateVersion(version *models.Version) error {
+	return service.connection.UpdateObject(BucketName, []byte(versionKey), version)
 }

api/datastore/backup.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/rs/zerolog/log"
 )
 
@@ -53,7 +54,7 @@ func (store *Store) copyDBFile(from string, to string) error {
 
 // BackupOptions provide a helper to inject backup options
 type BackupOptions struct {
-	Version        int // I can't find this used for anything other than a filename
+	Version        string
 	BackupDir      string
 	BackupFileName string
 	BackupPath     string
@@ -70,26 +71,32 @@ func getBackupRestoreOptions(backupDir string) *BackupOptions {
 }
 
 // Backup current database with default options
-func (store *Store) Backup() (string, error) {
-	return store.backupWithOptions(nil)
+func (store *Store) Backup(version *models.Version) (string, error) {
+	if version == nil {
+		return store.backupWithOptions(nil)
+	}
+
+	return store.backupWithOptions(&BackupOptions{
+		Version: version.SchemaVersion,
+	})
 }
 
 func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}
 	}
-	if options.Version == 0 {
-		version, err := store.version()
+	if options.Version == "" {
+		v, err := store.VersionService.Version()
 		if err != nil {
-			version = 0
+			options.Version = ""
 		}
-		options.Version = version
+		options.Version = v.SchemaVersion
 	}
 	if options.BackupDir == "" {
 		options.BackupDir = store.commonBackupDir()
 	}
 	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
 	}
 	if options.BackupPath == "" {
 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
@@ -168,7 +175,6 @@ func (store *Store) removeWithOptions(options *BackupOptions) error {
 
 	if os.IsNotExist(err) {
 		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
-
 		return err
 	}
 
@@ -176,7 +182,6 @@ func (store *Store) removeWithOptions(options *BackupOptions) error {
 	err = os.Remove(options.BackupPath)
 	if err != nil {
 		log.Error().Err(err).Msg("failed")
-
 		return err
 	}
 

api/datastore/backup_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 )
 
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, false, true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	connection := store.GetConnection()
@@ -45,10 +46,13 @@ func TestBackup(t *testing.T) {
 	defer teardown()
 
 	t.Run("Backup should create default db backup", func(t *testing.T) {
-		store.VersionService.StoreDBVersion(portainer.DBVersion)
+		v := models.Version{
+			SchemaVersion: portainer.APIVersion,
+		}
+		store.VersionService.UpdateVersion(&v)
 		store.backupWithOptions(nil)
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}

api/datastore/datastore.go

@@ -13,22 +13,6 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func (store *Store) version() (int, error) {
-	version, err := store.VersionService.DBVersion()
-	if store.IsErrObjectNotFound(err) {
-		version = 0
-	}
-	return version, err
-}
-
-func (store *Store) edition() portainer.SoftwareEdition {
-	edition, err := store.VersionService.Edition()
-	if store.IsErrObjectNotFound(err) {
-		edition = portainer.PortainerCE
-	}
-	return edition
-}
-
 // NewStore initializes a new Store and the associated services
 func NewStore(storePath string, fileService portainer.FileService, connection portainer.Connection) *Store {
 	return &Store{
@@ -39,8 +23,6 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
-	newStore = true
-
 	encryptionReq, err := store.connection.NeedsEncryptionMigration()
 	if err != nil {
 		return false, err
@@ -55,31 +37,24 @@ func (store *Store) Open() (newStore bool, err error) {
 
 	err = store.connection.Open()
 	if err != nil {
-		return newStore, err
+		return false, err
 	}
 
 	err = store.initServices()
 	if err != nil {
-		return newStore, err
+		return false, err
 	}
 
-	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	version, err := store.VersionService.DBVersion()
+	// If no settings object exists then assume we have a new store
+	_, err = store.SettingsService.Settings()
 	if err != nil {
 		if store.IsErrObjectNotFound(err) {
-			return newStore, nil
+			return true, nil
 		}
-
-		return newStore, err
+		return false, err
 	}
 
-	if version > 0 {
-		log.Debug().Int("version", version).Msg("opened existing store")
-
-		return false, nil
-	}
-
-	return newStore, nil
+	return false, nil
 }
 
 func (store *Store) Close() error {
@@ -94,17 +69,29 @@ func (store *Store) BackupTo(w io.Writer) error {
 
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
-	if store.edition() != portainer.PortainerCE {
+	if store.edition() != portainer.Edition {
 		return portainerErrors.ErrWrongDBEdition
 	}
 	return nil
 }
 
+func (store *Store) edition() portainer.SoftwareEdition {
+	edition, err := store.VersionService.Edition()
+	if store.IsErrObjectNotFound(err) {
+		edition = portainer.PortainerCE
+	}
+	return edition
+}
+
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
 	return e == portainerErrors.ErrObjectNotFound
 }
 
+func (store *Store) Connection() portainer.Connection {
+	return store.connection
+}
+
 func (store *Store) Rollback(force bool) error {
 	return store.connectionRollback(force)
 }

api/datastore/init.go

@@ -1,18 +1,12 @@
 package datastore
 
 import (
-	"github.com/gofrs/uuid"
 	portainer "github.com/portainer/portainer/api"
 )
 
 // Init creates the default data set.
 func (store *Store) Init() error {
-	err := store.checkOrCreateInstanceID()
-	if err != nil {
-		return err
-	}
-
-	err = store.checkOrCreateDefaultSettings()
+	err := store.checkOrCreateDefaultSettings()
 	if err != nil {
 		return err
 	}
@@ -25,21 +19,6 @@ func (store *Store) Init() error {
 	return store.checkOrCreateDefaultData()
 }
 
-func (store *Store) checkOrCreateInstanceID() error {
-	_, err := store.VersionService.InstanceID()
-	if store.IsErrObjectNotFound(err) {
-		uid, err := uuid.NewV4()
-		if err != nil {
-			return err
-		}
-
-		instanceID := uid.String()
-		return store.VersionService.StoreInstanceID(instanceID)
-	}
-
-	return err
-}
-
 func (store *Store) checkOrCreateDefaultSettings() error {
 	// TODO: these need to also be applied when importing
 	settings, err := store.SettingsService.Settings()

api/datastore/migrate_data.go

@@ -4,37 +4,69 @@ import (
 	"fmt"
 	"runtime/debug"
 
-	portainer "github.com/portainer/portainer/api"
+	portaineree "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/cli"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/database/models"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/internal/authorization"
 
-	werrors "github.com/pkg/errors"
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
 
 const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
 
 func (store *Store) MigrateData() error {
-	version, err := store.version()
+	updating, err := store.VersionService.IsUpdating()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "while checking if the store is updating")
 	}
 
-	// Backup Database
-	backupPath, err := store.Backup()
-	if err != nil {
-		return werrors.Wrap(err, "while backing up db before migration")
+	if updating {
+		return dserrors.ErrDatabaseIsUpdating
 	}
 
-	migratorParams := &migrator.MigratorParameters{
-		DatabaseVersion:         version,
+	// migrate new version bucket if required (doesn't write anything to db yet)
+	version, err := store.getOrMigrateLegacyVersion()
+	if err != nil {
+		return errors.Wrap(err, "while migrating legacy version")
+	}
+
+	migratorParams := store.newMigratorParameters(version)
+	migrator := migrator.NewMigrator(migratorParams)
+
+	if !migrator.NeedsMigration() {
+		return nil
+	}
+
+	// before we alter anything in the DB, create a backup
+	backupPath, err := store.Backup(version)
+	if err != nil {
+		return errors.Wrap(err, "while backing up database")
+	}
+
+	err = store.FailSafeMigrate(migrator, version)
+	if err != nil {
+		err = store.restoreWithOptions(&BackupOptions{BackupPath: backupPath})
+		if err != nil {
+			return errors.Wrap(err, "failed to restore database")
+		}
+
+		log.Info().Msg("database restored to previous version")
+		return errors.Wrap(err, "failed to migrate database")
+	}
+
+	return nil
+}
+
+func (store *Store) newMigratorParameters(version *models.Version) *migrator.MigratorParameters {
+	return &migrator.MigratorParameters{
+		CurrentDBVersion:        version,
 		EndpointGroupService:    store.EndpointGroupService,
 		EndpointService:         store.EndpointService,
 		EndpointRelationService: store.EndpointRelationService,
 		ExtensionService:        store.ExtensionService,
-		FDOProfilesService:      store.FDOProfilesService,
 		RegistryService:         store.RegistryService,
 		ResourceControlService:  store.ResourceControlService,
 		RoleService:             store.RoleService,
@@ -49,99 +81,42 @@ func (store *Store) MigrateData() error {
 		FileService:             store.fileService,
 		DockerhubService:        store.DockerHubService,
 		AuthorizationService:    authorization.NewService(store),
+		EdgeStackService:        store.EdgeStackService,
 	}
-
-	// restore on error
-	err = store.connectionMigrateData(migratorParams)
-	if err != nil {
-		log.Error().Err(err).Msg("while DB migration, restoring DB")
-
-		// Restore options
-		options := BackupOptions{
-			BackupPath: backupPath,
-		}
-
-		err := store.restoreWithOptions(&options)
-		if err != nil {
-			log.Fatal().
-				Str("database_file", store.databasePath()).
-				Str("backup", options.BackupPath).Err(err).
-				Msg("failed restoring the backup, Portainer database file needs to restored manually by replacing the database file with a recent backup")
-		}
-	}
-
-	return err
 }
 
 // FailSafeMigrate backup and restore DB if migration fail
-func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) (err error) {
+func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version *models.Version) (err error) {
 	defer func() {
 		if e := recover(); e != nil {
-			store.Rollback(true)
 			// return error with cause and stacktrace (recover() doesn't include a stacktrace)
 			err = fmt.Errorf("%v %s", e, string(debug.Stack()))
 		}
 	}()
 
-	// !Important: we must use a named return value in the function definition and not a local
-	// !variable referenced from the closure or else the return value will be incorrectly set
-	return migrator.Migrate()
-}
-
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-// if force is true, then migrate regardless.
-func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParameters) error {
-	migrator := migrator.NewMigrator(migratorParams)
-
-	// backup db file before upgrading DB to support rollback
-	isUpdating, err := migratorParams.VersionService.IsUpdating()
-	if err != nil && err != errors.ErrObjectNotFound {
-		return err
-	}
-
-	if !isUpdating && migrator.Version() != portainer.DBVersion {
-		err = store.backupVersion(migrator)
-		if err != nil {
-			return werrors.Wrapf(err, "failed to backup database")
-		}
-	}
-
-	if migrator.Version() < portainer.DBVersion {
-		log.Info().
-			Int("migrator_version", migrator.Version()).
-			Int("db_version", portainer.DBVersion).
-			Msg("migrating database")
-
-		err = store.FailSafeMigrate(migrator)
-		if err != nil {
-			log.Error().Err(err).Msg("an error occurred during database migration")
-
-			return err
-		}
-	}
-
-	return nil
-}
-
-// backupVersion will backup the database or panic if any errors occur
-func (store *Store) backupVersion(migrator *migrator.Migrator) error {
-	log.Info().Msg("backing up database prior to version upgrade")
-
-	options := getBackupRestoreOptions(store.commonBackupDir())
-
-	_, err := store.backupWithOptions(options)
+	err = store.VersionService.StoreIsUpdating(true)
 	if err != nil {
-		log.Error().Err(err).Msg("an error occurred during database backup")
+		return errors.Wrap(err, "while updating the store")
+	}
 
-		removalErr := store.removeWithOptions(options)
-		if removalErr != nil {
-			log.Error().Err(err).Msg("an error occurred during store removal prior to backup")
-		}
+	// now update the version to the new struct (if required)
+	err = store.finishMigrateLegacyVersion(version)
+	if err != nil {
+		return errors.Wrap(err, "while updating version")
+	}
 
+	log.Info().Msg("migrating database from version " + version.SchemaVersion + " to " + portaineree.APIVersion)
+
+	err = migrator.Migrate()
+	if err != nil {
 		return err
 	}
 
+	err = store.VersionService.StoreIsUpdating(false)
+	if err != nil {
+		return errors.Wrap(err, "failed to update the store")
+	}
+
 	return nil
 }
 

api/datastore/migrate_data_test.go

@@ -10,39 +10,41 @@ import (
 	"strings"
 	"testing"
 
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/rs/zerolog/log"
 )
 
 // testVersion is a helper which tests current store version against wanted version
-func testVersion(store *Store, versionWant int, t *testing.T) {
-	v, err := store.VersionService.DBVersion()
+func testVersion(store *Store, versionWant string, t *testing.T) {
+	v, err := store.VersionService.Version()
 	if err != nil {
-		t.Errorf("Expect store version to be %d but was %d with error: %s", versionWant, v, err)
+		t.Errorf("Expect store version to be %s but was %s with error: %s", versionWant, v.SchemaVersion, err)
 	}
-	if v != versionWant {
-		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
+	if v.SchemaVersion != versionWant {
+		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
 	}
 }
 
 func TestMigrateData(t *testing.T) {
 	snapshotTests := []struct {
-		testName string
-		srcPath  string
-		wantPath string
+		testName           string
+		srcPath            string
+		wantPath           string
+		overrideInstanceId bool
 	}{
 		{
-			testName: "migrate version 24 to latest",
-			srcPath:  "test_data/input_24.json",
-			wantPath: "test_data/output_24_to_latest.json",
+			testName:           "migrate version 24 to latest",
+			srcPath:            "test_data/input_24.json",
+			wantPath:           "test_data/output_24_to_latest.json",
+			overrideInstanceId: true,
 		},
 	}
 	for _, test := range snapshotTests {
 		t.Run(test.testName, func(t *testing.T) {
-			err := migrateDBTestHelper(t, test.srcPath, test.wantPath)
+			err := migrateDBTestHelper(t, test.srcPath, test.wantPath, test.overrideInstanceId)
 			if err != nil {
 				t.Errorf(
 					"Failed migrating mock database %v: %v",
@@ -53,111 +55,111 @@ func TestMigrateData(t *testing.T) {
 		})
 	}
 
-	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store, teardown := MustNewTestStore(t, false, true)
-		defer teardown()
+	// t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
+	// 	newStore, store, teardown := MustNewTestStore(t, true, false)
+	// 	defer teardown()
 
-		if !newStore {
-			t.Error("Expect a new DB")
-		}
+	// 	if !newStore {
+	// 		t.Error("Expect a new DB")
+	// 	}
 
-		// not called for new stores
-		//store.MigrateData()
+	// 	testVersion(store, portainer.APIVersion, t)
+	// 	store.Close()
 
-		testVersion(store, portainer.DBVersion, t)
-		store.Close()
+	// 	newStore, _ = store.Open()
+	// 	if newStore {
+	// 		t.Error("Expect store to NOT be new DB")
+	// 	}
+	// })
 
-		newStore, _ = store.Open()
-		if newStore {
-			t.Error("Expect store to NOT be new DB")
-		}
-	})
+	// tests := []struct {
+	// 	version         string
+	// 	expectedVersion string
+	// }{
+	// 	{version: "1.24.1", expectedVersion: portainer.APIVersion},
+	// 	{version: "2.0.0", expectedVersion: portainer.APIVersion},
+	// }
+	// for _, tc := range tests {
+	// 	_, store, teardown := MustNewTestStore(t, true, true)
+	// 	defer teardown()
 
-	tests := []struct {
-		version         int
-		expectedVersion int
-	}{
-		{version: 17, expectedVersion: portainer.DBVersion},
-		{version: 21, expectedVersion: portainer.DBVersion},
-	}
-	for _, tc := range tests {
-		_, store, teardown := MustNewTestStore(t, true, true)
-		defer teardown()
+	// 	// Setup data
+	// 	v := models.Version{SchemaVersion: tc.version}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		// Setup data
-		store.VersionService.StoreDBVersion(tc.version)
+	// 	// Required roles by migrations 22.2
+	// 	store.RoleService.Create(&portainer.Role{ID: 1})
+	// 	store.RoleService.Create(&portainer.Role{ID: 2})
+	// 	store.RoleService.Create(&portainer.Role{ID: 3})
+	// 	store.RoleService.Create(&portainer.Role{ID: 4})
 
-		// Required roles by migrations 22.2
-		store.RoleService.Create(&portainer.Role{ID: 1})
-		store.RoleService.Create(&portainer.Role{ID: 2})
-		store.RoleService.Create(&portainer.Role{ID: 3})
-		store.RoleService.Create(&portainer.Role{ID: 4})
+	// 	t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
+	// 		store.MigrateData()
+	// 		testVersion(store, tc.expectedVersion, t)
+	// 	})
 
-		t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
-			store.MigrateData()
-			testVersion(store, tc.expectedVersion, t)
-		})
+	// 	t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
+	// 		store.Rollback(true)
+	// 		store.Open()
+	// 		testVersion(store, tc.version, t)
+	// 	})
+	// }
 
-		t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) {
-			store.Rollback(true)
-			store.Open()
-			testVersion(store, tc.version, t)
-		})
-	}
+	// t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(t, false, true)
-		defer teardown()
+	// 	v := models.Version{SchemaVersion: "1.24.1"}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		version := 17
-		store.VersionService.StoreDBVersion(version)
+	// 	store.MigrateData()
 
-		store.MigrateData()
+	// 	testVersion(store, v.SchemaVersion, t)
+	// })
 
-		testVersion(store, version, t)
-	})
+	// t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(t, false, true)
-		defer teardown()
-		store.VersionService.StoreDBVersion(0)
+	// 	v := models.Version{SchemaVersion: "0.0.0"}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		store.MigrateData()
+	// 	store.MigrateData()
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-		if !isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
-		}
-	})
+	// 	if !isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should exist; file=%s", options.BackupPath)
+	// 	}
+	// })
 
-	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(t, false, true)
-		defer teardown()
+	// t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-		store.VersionService.StoreIsUpdating(true)
+	// 	store.VersionService.StoreIsUpdating(true)
 
-		store.MigrateData()
+	// 	store.MigrateData()
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
+	// 	if isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+	// 	}
+	// })
 
-	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(t, false, true)
-		defer teardown()
+	// t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-		store.MigrateData()
+	// 	store.MigrateData()
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
+	// 	if isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+	// 	}
+	// })
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
@@ -179,18 +181,23 @@ func Test_getBackupRestoreOptions(t *testing.T) {
 
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := 21
-		_, store, teardown := MustNewTestStore(t, false, true)
+		version := models.Version{SchemaVersion: "2.4.0"}
+		_, store, teardown := MustNewTestStore(t, true, false)
 		defer teardown()
-		store.VersionService.StoreDBVersion(version)
 
-		_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		err := store.VersionService.UpdateVersion(&version)
+		if err != nil {
+			t.Errorf("Failed updating version: %v", err)
+		}
+
+		_, err = store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
 
-		// Change the current edition
-		err = store.VersionService.StoreDBVersion(version + 10)
+		// Change the current version
+		version2 := models.Version{SchemaVersion: "2.6.0"}
+		err = store.VersionService.UpdateVersion(&version2)
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
@@ -199,12 +206,17 @@ func TestRollback(t *testing.T) {
 		if err != nil {
 			t.Logf("Rollback failed: %s", err)
 			t.Fail()
-
 			return
 		}
 
-		store.Open()
-		testVersion(store, version, t)
+		_, err = store.Open()
+		if err != nil {
+			t.Logf("Open failed: %s", err)
+			t.Fail()
+			return
+		}
+
+		testVersion(store, version.SchemaVersion, t)
 	})
 }
 
@@ -220,15 +232,20 @@ func isFileExist(path string) bool {
 // migrateDBTestHelper loads a json representation of a bolt database from srcPath,
 // parses it into a database, runs a migration on that database, and then
 // compares it with an expected output database.
-func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
+func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanceId bool) error {
 	srcJSON, err := os.ReadFile(srcPath)
 	if err != nil {
 		t.Fatalf("failed loading source JSON file %v: %v", srcPath, err)
 	}
 
 	// Parse source json to db.
-	_, store, teardown := MustNewTestStore(t, true, false)
-	defer teardown()
+	// When we create a new test store, it sets its version field automatically to latest.
+	_, store, _ := MustNewTestStore(t, true, false)
+
+	fmt.Println("store.path=", store.GetConnection().GetDatabaseFilePath())
+	store.connection.DeleteObject("version", []byte("VERSION"))
+
+	//	defer teardown()
 	err = importJSON(t, bytes.NewReader(srcJSON), store)
 	if err != nil {
 		return err
@@ -240,6 +257,21 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
 		return err
 	}
 
+	if overrideInstanceId {
+		// old versions of portainer did not have instance-id.  Because this gets generated
+		// we need to override the expected output to match the expected value to pass the test
+		v, err := store.VersionService.Version()
+		if err != nil {
+			return err
+		}
+
+		v.InstanceID = "463d5c47-0ea5-4aca-85b1-405ceefee254"
+		err = store.VersionService.UpdateVersion(v)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Assert that our database connection is using bolt so we can call
 	// exportJson rather than ExportRaw. The exportJson function allows us to
 	// strip out the metadata which we don't want for our tests.
@@ -316,42 +348,65 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
 				t.Logf("failed casting %s to map[string]interface{}", k)
 			}
 
+			// New format db
+			version, ok := versions["VERSION"]
+			if ok {
+				err := con.CreateObjectWithStringId(
+					k,
+					[]byte("VERSION"),
+					version,
+				)
+				if err != nil {
+					t.Logf("failed writing VERSION in %s: %v", k, err)
+				}
+			}
+
+			// old format db
+
 			dbVersion, ok := versions["DB_VERSION"]
-			if !ok {
-				t.Logf("failed getting DB_VERSION from %s", k)
-			}
+			if ok {
+				numDBVersion, ok := dbVersion.(json.Number)
+				if !ok {
+					t.Logf("failed parsing DB_VERSION as json number from %s", k)
+				}
 
-			numDBVersion, ok := dbVersion.(json.Number)
-			if !ok {
-				t.Logf("failed parsing DB_VERSION as json number from %s", k)
-			}
+				intDBVersion, err := numDBVersion.Int64()
+				if err != nil {
+					t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
+				}
 
-			intDBVersion, err := numDBVersion.Int64()
-			if err != nil {
-				t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
-			}
-
-			err = con.CreateObjectWithStringId(
-				k,
-				[]byte("DB_VERSION"),
-				int(intDBVersion),
-			)
-			if err != nil {
-				t.Logf("failed writing DB_VERSION in %s: %v", k, err)
+				err = con.CreateObjectWithStringId(
+					k,
+					[]byte("DB_VERSION"),
+					int(intDBVersion),
+				)
+				if err != nil {
+					t.Logf("failed writing DB_VERSION in %s: %v", k, err)
+				}
 			}
 
 			instanceID, ok := versions["INSTANCE_ID"]
-			if !ok {
-				t.Logf("failed getting INSTANCE_ID from %s", k)
+			if ok {
+				err = con.CreateObjectWithStringId(
+					k,
+					[]byte("INSTANCE_ID"),
+					instanceID,
+				)
+				if err != nil {
+					t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
+				}
 			}
 
-			err = con.CreateObjectWithStringId(
-				k,
-				[]byte("INSTANCE_ID"),
-				instanceID,
-			)
-			if err != nil {
-				t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
+			edition, ok := versions["EDITION"]
+			if ok {
+				err = con.CreateObjectWithStringId(
+					k,
+					[]byte("EDITION"),
+					edition,
+				)
+				if err != nil {
+					t.Logf("failed writing EDITION in %s: %v", k, err)
+				}
 			}
 
 		case "dockerhub":

api/datastore/migrate_dbversion29_test.go

@@ -54,7 +54,6 @@ func TestMigrateSettings(t *testing.T) {
 	}
 
 	m := migrator.NewMigrator(&migrator.MigratorParameters{
-		DatabaseVersion:         29,
 		EndpointGroupService:    store.EndpointGroupService,
 		EndpointService:         store.EndpointService,
 		EndpointRelationService: store.EndpointRelationService,

api/datastore/migrate_legacyversion.go

@@ -0,0 +1,115 @@
+package datastore
+
+import (
+	portaineree "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+const (
+	bucketName         = "version"
+	legacyDBVersionKey = "DB_VERSION"
+	legacyInstanceKey  = "INSTANCE_ID"
+	legacyEditionKey   = "EDITION"
+)
+
+var dbVerToSemVerMap = map[int]string{
+	18: "1.21",
+	19: "1.22",
+	20: "1.22.1",
+	21: "1.22.2",
+	22: "1.23",
+	23: "1.24",
+	24: "1.24.1",
+	25: "2.0",
+	26: "2.1",
+	27: "2.2",
+	28: "2.4",
+	29: "2.4",
+	30: "2.6",
+	31: "2.7",
+	32: "2.9",
+	33: "2.9.1",
+	34: "2.10",
+	35: "2.9.3",
+	36: "2.11",
+	40: "2.13",
+	50: "2.14",
+	51: "2.14.1",
+	52: "2.14.2",
+	60: "2.15",
+	61: "2.15.1",
+	70: "2.16",
+	80: "2.17",
+}
+
+func dbVersionToSemanticVersion(dbVersion int) string {
+	if dbVersion < 18 {
+		return "1.0.0"
+	}
+
+	ver, ok := dbVerToSemVerMap[dbVersion]
+	if ok {
+		return ver
+	}
+
+	// We should always return something sensible
+	switch {
+	case dbVersion < 40:
+		return "2.11"
+	case dbVersion < 50:
+		return "2.13"
+	case dbVersion < 60:
+		return "2.14.2"
+	case dbVersion < 70:
+		return "2.15.1"
+	}
+
+	return "2.16.0"
+}
+
+// getOrMigrateLegacyVersion to new Version struct
+func (store *Store) getOrMigrateLegacyVersion() (*models.Version, error) {
+	// Very old versions of portainer did not have a version bucket, lets set some defaults
+	dbVersion := 24
+	edition := int(portaineree.PortainerCE)
+	instanceId := ""
+
+	// If we already have a version key, we don't need to migrate
+	v, err := store.VersionService.Version()
+	if err == nil || !dataservices.IsErrObjectNotFound(err) {
+		return v, err
+	}
+
+	err = store.connection.GetObject(bucketName, []byte(legacyDBVersionKey), &dbVersion)
+	if err != nil && !dataservices.IsErrObjectNotFound(err) {
+		return nil, err
+	}
+
+	err = store.connection.GetObject(bucketName, []byte(legacyEditionKey), &edition)
+	if err != nil && !dataservices.IsErrObjectNotFound(err) {
+		return nil, err
+	}
+
+	err = store.connection.GetObject(bucketName, []byte(legacyInstanceKey), &instanceId)
+	if err != nil && !dataservices.IsErrObjectNotFound(err) {
+		return nil, err
+	}
+
+	return &models.Version{
+		SchemaVersion: dbVersionToSemanticVersion(dbVersion),
+		Edition:       edition,
+		InstanceID:    string(instanceId),
+	}, nil
+}
+
+// finishMigrateLegacyVersion writes the new version to the DB and removes the old version keys from the DB
+func (store *Store) finishMigrateLegacyVersion(versionToWrite *models.Version) error {
+	err := store.VersionService.UpdateVersion(versionToWrite)
+
+	// Remove legacy keys if present
+	store.connection.DeleteObject(bucketName, []byte(legacyDBVersionKey))
+	store.connection.DeleteObject(bucketName, []byte(legacyEditionKey))
+	store.connection.DeleteObject(bucketName, []byte(legacyInstanceKey))
+	return err
+}

api/datastore/migrator/migrate_ce.go

@@ -4,140 +4,122 @@ import (
 	"reflect"
 	"runtime"
 
+	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 
-	"github.com/pkg/errors"
+	"github.com/Masterminds/semver"
 	"github.com/rs/zerolog/log"
 )
 
-type migration struct {
-	dbversion int
-	migrate   func() error
-}
-
 func migrationError(err error, context string) error {
 	return errors.Wrap(err, "failed in "+context)
 }
 
-func newMigration(dbversion int, migrate func() error) migration {
-	return migration{
-		dbversion: dbversion,
-		migrate:   migrate,
-	}
-}
-
-func dbTooOldError() error {
-	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
-}
-
 func GetFunctionName(i interface{}) string {
 	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
 }
 
 // Migrate checks the database version and migrate the existing data to the most recent data model.
 func (m *Migrator) Migrate() error {
-	// set DB to updating status
-	err := m.versionService.StoreIsUpdating(true)
+	version, err := m.versionService.Version()
 	if err != nil {
-		return migrationError(err, "StoreIsUpdating")
+		return migrationError(err, "get version service")
 	}
 
-	migrations := []migration{
-		// Portainer < 1.21.0
-		newMigration(17, dbTooOldError),
-
-		// Portainer 1.21.0
-		newMigration(18, m.updateUsersToDBVersion18),
-		newMigration(18, m.updateEndpointsToDBVersion18),
-		newMigration(18, m.updateEndpointGroupsToDBVersion18),
-		newMigration(18, m.updateRegistriesToDBVersion18),
-
-		// 1.22.0
-		newMigration(19, m.updateSettingsToDBVersion19),
-
-		// 1.22.1
-		newMigration(20, m.updateUsersToDBVersion20),
-		newMigration(20, m.updateSettingsToDBVersion20),
-		newMigration(20, m.updateSchedulesToDBVersion20),
-
-		// Portainer 1.23.0
-		// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-		newMigration(22, m.updateResourceControlsToDBVersion22),
-		newMigration(22, m.updateUsersAndRolesToDBVersion22),
-
-		// Portainer 1.24.0
-		newMigration(23, m.updateTagsToDBVersion23),
-		newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
-
-		// Portainer 1.24.1
-		newMigration(24, m.updateSettingsToDB24),
-
-		// Portainer 2.0.0
-		newMigration(25, m.updateSettingsToDB25),
-		newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
-
-		// Portainer 2.1.0
-		newMigration(26, m.updateEndpointSettingsToDB25),
-
-		// Portainer 2.2.0
-		newMigration(27, m.updateStackResourceControlToDB27),
-
-		// Portainer 2.6.0
-		newMigration(30, m.migrateDBVersionToDB30),
-
-		// Portainer 2.9.0
-		newMigration(32, m.migrateDBVersionToDB32),
-
-		// Portainer 2.9.1, 2.9.2
-		newMigration(33, m.migrateDBVersionToDB33),
-
-		// Portainer 2.10
-		newMigration(34, m.migrateDBVersionToDB34),
-
-		// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
-		newMigration(35, m.migrateDBVersionToDB35),
-
-		newMigration(36, m.migrateDBVersionToDB36),
-
-		// Portainer 2.13
-		newMigration(40, m.migrateDBVersionToDB40),
-
-		// Portainer 2.14
-		newMigration(50, m.migrateDBVersionToDB50),
-
-		// Portainer 2.15
-		newMigration(60, m.migrateDBVersionToDB60),
-
-		// Portainer 2.16
-		newMigration(70, m.migrateDBVersionToDB70),
+	schemaVersion, err := semver.NewVersion(version.SchemaVersion)
+	if err != nil {
+		return migrationError(err, "invalid db schema version")
 	}
 
-	var lastDbVersion int
-	for _, migration := range migrations {
-		if m.currentDBVersion < migration.dbversion {
+	newMigratorCount := 0
+	apiVersion := semver.MustParse(portainer.APIVersion)
+	if schemaVersion.Equal(apiVersion) {
+		// detect and run migrations when the versions are the same.
+		// e.g. development builds
+		latestMigrations := m.LatestMigrations()
+		if latestMigrations.Version.Equal(schemaVersion) &&
+			version.MigratorCount != len(latestMigrations.MigrationFuncs) {
+			err := runMigrations(latestMigrations.MigrationFuncs)
+			if err != nil {
+				return err
+			}
+			newMigratorCount = len(latestMigrations.MigrationFuncs)
+		}
+	} else {
+		// regular path when major/minor/patch versions differ
+		for _, migration := range m.migrations {
+			if schemaVersion.LessThan(migration.Version) {
 
-			// Print the next line only when the version changes
-			if migration.dbversion > lastDbVersion {
-				log.Info().Int("to_version", migration.dbversion).Msg("migrating DB")
+				log.Info().Msgf("migrating data to %s", migration.Version.String())
+				err := runMigrations(migration.MigrationFuncs)
+				if err != nil {
+					return err
+				}
 			}
 
-			err := migration.migrate()
-			if err != nil {
-				return migrationError(err, GetFunctionName(migration.migrate))
+			if apiVersion.Equal(migration.Version) {
+				newMigratorCount = len(migration.MigrationFuncs)
 			}
 		}
-		lastDbVersion = migration.dbversion
 	}
 
-	log.Info().Int("version", portainer.DBVersion).Msg("setting DB version")
+	err = m.Always()
+	if err != nil {
+		return migrationError(err, "Always migrations returned error")
+	}
 
-	err = m.versionService.StoreDBVersion(portainer.DBVersion)
+	version.SchemaVersion = portainer.APIVersion
+	version.MigratorCount = newMigratorCount
+
+	err = m.versionService.UpdateVersion(version)
 	if err != nil {
 		return migrationError(err, "StoreDBVersion")
 	}
 
-	log.Info().Int("version", portainer.DBVersion).Msg("updated DB version")
+	log.Info().Msgf("db migrated to %s", portainer.APIVersion)
 
-	// reset DB updating status
-	return m.versionService.StoreIsUpdating(false)
+	return nil
+}
+
+func runMigrations(migrationFuncs []func() error) error {
+	for _, migrationFunc := range migrationFuncs {
+		err := migrationFunc()
+		if err != nil {
+			return migrationError(err, GetFunctionName(migrationFunc))
+		}
+	}
+	return nil
+}
+
+func (m *Migrator) NeedsMigration() bool {
+	// we need to migrate if anything changes with the version in the DB vs what our software version is.
+	// If the version matches, then it's all down to the number of migration funcs we have for the current version
+	// i.e. the MigratorCount
+
+	// In this particular instance we should log a fatal error
+	if m.CurrentDBEdition() != portainer.PortainerCE {
+		log.Fatal().Msg("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+		return false
+	}
+
+	if m.CurrentSemanticDBVersion().LessThan(semver.MustParse(portainer.APIVersion)) {
+		return true
+	}
+
+	// Check if we have any migrations for the current version
+	latestMigrations := m.LatestMigrations()
+	if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
+		if m.currentDBVersion.MigratorCount != len(latestMigrations.MigrationFuncs) {
+			return true
+		}
+	} else {
+		// One remaining possibility if we get here.  If our migrator count > 0 and we have no migration funcs
+		// for the current version (i.e. they were deleted during development).  Then we we need to migrate.
+		// This is to reset the migrator count back to 0
+		if m.currentDBVersion.MigratorCount > 0 {
+			return true
+		}
+	}
+
+	return false
 }

api/datastore/migrator/migrate_dbversion33.go

@@ -12,7 +12,7 @@ func (m *Migrator) migrateDBVersionToDB34() error {
 	return MigrateStackEntryPoint(m.stackService)
 }
 
-// MigrateStackEntryPoint exported for testing (blah.)
+// MigrateStackEntryPoint exported for testing
 func MigrateStackEntryPoint(stackService dataservices.StackService) error {
 	stacks, err := stackService.Stacks()
 	if err != nil {

api/datastore/migrator/migrate_dbversion34.go

@@ -1,12 +1,7 @@
 package migrator
 
-import "github.com/rs/zerolog/log"
-
 func (m *Migrator) migrateDBVersionToDB35() error {
 	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
 	// calling it again will now fix the issue as the function has been repaired.
-
-	log.Info().Msg("updating dockerhub registries")
-
 	return m.updateDockerhubToDB32()
 }

api/datastore/migrator/migrate_dbversion60.go

@@ -19,12 +19,13 @@ func (m *Migrator) addGpuInputFieldDB60() error {
 	}
 
 	for _, endpoint := range endpoints {
-		endpoint.Gpus = []portainer.Pair{}
-		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-		if err != nil {
-			return err
+		if endpoint.Gpus == nil {
+			endpoint.Gpus = []portainer.Pair{}
+			err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
 		}
-
 	}
 
 	return nil

api/datastore/migrator/migrate_dbversion70.go

@@ -7,7 +7,7 @@ import (
 )
 
 func (m *Migrator) migrateDBVersionToDB70() error {
-	log.Info().Msg("- add IngressAvailabilityPerNamespace field")
+	log.Info().Msg("add IngressAvailabilityPerNamespace field")
 	if err := m.updateIngressFieldsForEnvDB70(); err != nil {
 		return err
 	}
@@ -60,6 +60,7 @@ func (m *Migrator) updateIngressFieldsForEnvDB70() error {
 	for _, endpoint := range endpoints {
 		endpoint.Kubernetes.Configuration.IngressAvailabilityPerNamespace = true
 		endpoint.Kubernetes.Configuration.AllowNoneIngressClass = false
+		endpoint.PostInitMigrations.MigrateIngresses = true
 
 		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
 		if err != nil {

api/datastore/migrator/migrate_dbversion71.go

@@ -0,0 +1,36 @@
+package migrator
+
+import (
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDBVersionToDB71() error {
+	log.Info().Msg("removing orphaned snapshots")
+
+	snapshots, err := m.snapshotService.Snapshots()
+	if err != nil {
+		return err
+	}
+
+	for _, s := range snapshots {
+		_, err := m.endpointService.Endpoint(s.EndpointID)
+		if err == nil {
+			log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("keeping snapshot")
+			continue
+		} else if err != errors.ErrObjectNotFound {
+			log.Debug().Int("endpoint_id", int(s.EndpointID)).Err(err).Msg("database error")
+			return err
+		}
+
+		log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("removing snapshot")
+
+		err = m.snapshotService.DeleteSnapshot(s.EndpointID)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion80.go

@@ -0,0 +1,47 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDBVersionToDB80() error {
+	return m.updateEdgeStackStatusForDB80()
+}
+
+func (m *Migrator) updateEdgeStackStatusForDB80() error {
+	log.Info().Msg("transfer type field to details field for edge stack status")
+
+	edgeStacks, err := m.edgeStackService.EdgeStacks()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeStack := range edgeStacks {
+		for endpointId, status := range edgeStack.Status {
+			switch status.Type {
+			case portainer.EdgeStackStatusPending:
+				status.Details.Pending = true
+			case portainer.EdgeStackStatusOk:
+				status.Details.Ok = true
+			case portainer.EdgeStackStatusError:
+				status.Details.Error = true
+			case portainer.EdgeStackStatusAcknowledged:
+				status.Details.Acknowledged = true
+			case portainer.EdgeStackStatusRemove:
+				status.Details.Remove = true
+			case portainer.EdgeStackStatusRemoteUpdateSuccess:
+				status.Details.RemoteUpdateSuccess = true
+			}
+
+			edgeStack.Status[endpointId] = status
+		}
+
+		err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -1,7 +1,15 @@
 package migrator
 
 import (
+	"errors"
+
+	"github.com/portainer/portainer/api/dataservices/edgestack"
+
+	"github.com/Masterminds/semver"
+	"github.com/rs/zerolog/log"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices/dockerhub"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
@@ -25,7 +33,9 @@ import (
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		currentDBVersion        int
+		currentDBVersion *models.Version
+		migrations       []Migrations
+
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -45,11 +55,12 @@ type (
 		fileService             portainer.FileService
 		authorizationService    *authorization.Service
 		dockerhubService        *dockerhub.Service
+		edgeStackService        *edgestack.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
 	MigratorParameters struct {
-		DatabaseVersion         int
+		CurrentDBVersion        *models.Version
 		EndpointGroupService    *endpointgroup.Service
 		EndpointService         *endpoint.Service
 		EndpointRelationService *endpointrelation.Service
@@ -69,13 +80,14 @@ type (
 		FileService             portainer.FileService
 		AuthorizationService    *authorization.Service
 		DockerhubService        *dockerhub.Service
+		EdgeStackService        *edgestack.Service
 	}
 )
 
 // NewMigrator creates a new Migrator.
 func NewMigrator(parameters *MigratorParameters) *Migrator {
-	return &Migrator{
-		currentDBVersion:        parameters.DatabaseVersion,
+	migrator := &Migrator{
+		currentDBVersion:        parameters.CurrentDBVersion,
 		endpointGroupService:    parameters.EndpointGroupService,
 		endpointService:         parameters.EndpointService,
 		endpointRelationService: parameters.EndpointRelationService,
@@ -95,10 +107,115 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		fileService:             parameters.FileService,
 		authorizationService:    parameters.AuthorizationService,
 		dockerhubService:        parameters.DockerhubService,
+		edgeStackService:        parameters.EdgeStackService,
 	}
+
+	migrator.initMigrations()
+	return migrator
 }
 
-// Version exposes version of database
-func (migrator *Migrator) Version() int {
-	return migrator.currentDBVersion
+func (m *Migrator) CurrentDBVersion() string {
+	return m.currentDBVersion.SchemaVersion
+}
+
+func (m *Migrator) CurrentDBEdition() portainer.SoftwareEdition {
+	return portainer.SoftwareEdition(m.currentDBVersion.Edition)
+}
+
+func (m *Migrator) CurrentSemanticDBVersion() *semver.Version {
+	v, err := semver.NewVersion(m.currentDBVersion.SchemaVersion)
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("failed to parse current version")
+	}
+
+	return v
+}
+
+func (m *Migrator) addMigrations(v string, funcs ...func() error) {
+	m.migrations = append(m.migrations, Migrations{
+		Version:        semver.MustParse(v),
+		MigrationFuncs: funcs,
+	})
+}
+
+func (m *Migrator) LatestMigrations() Migrations {
+	return m.migrations[len(m.migrations)-1]
+}
+
+// !NOTE: Migration funtions should ideally be idempotent.
+// !      Which simply means the function can run over the same data many times but only transform it once.
+// !      In practice this really just means an extra check or two to ensure we're not destroying valid data.
+// !      This is not a hard rule though.  Understand the limitations.  A migration function may only run over
+// !      the data more than once if a new migration function is added and the version of your database schema is
+// !      the same.  e.g. two developers working on the same version add two different functions for different things.
+// !      This increases the migration funcs count and so they all run again.
+
+type Migrations struct {
+	Version        *semver.Version
+	MigrationFuncs MigrationFuncs
+}
+
+type MigrationFuncs []func() error
+
+func (m *Migrator) initMigrations() {
+	// !IMPORTANT: Do not be tempted to alter the order of these migrations.
+	// !           Even though one of them looks out of order. Caused by history related
+	// !           to maintaining two versions and releasing at different times
+
+	m.addMigrations("1.0.0", dbTooOldError) // default version found after migration
+
+	m.addMigrations("1.21",
+		m.updateUsersToDBVersion18,
+		m.updateEndpointsToDBVersion18,
+		m.updateEndpointGroupsToDBVersion18,
+		m.updateRegistriesToDBVersion18)
+
+	m.addMigrations("1.22", m.updateSettingsToDBVersion19)
+
+	m.addMigrations("1.22.1",
+		m.updateUsersToDBVersion20,
+		m.updateSettingsToDBVersion20,
+		m.updateSchedulesToDBVersion20)
+
+	m.addMigrations("1.23",
+		m.updateResourceControlsToDBVersion22,
+		m.updateUsersAndRolesToDBVersion22)
+
+	m.addMigrations("1.24",
+		m.updateTagsToDBVersion23,
+		m.updateEndpointsAndEndpointGroupsToDBVersion23)
+
+	m.addMigrations("1.24.1", m.updateSettingsToDB24)
+
+	m.addMigrations("2.0",
+		m.updateSettingsToDB25,
+		m.updateStacksToDB24)
+
+	m.addMigrations("2.1", m.updateEndpointSettingsToDB25)
+	m.addMigrations("2.2", m.updateStackResourceControlToDB27)
+	m.addMigrations("2.6", m.migrateDBVersionToDB30)
+	m.addMigrations("2.9", m.migrateDBVersionToDB32)
+	m.addMigrations("2.9.2", m.migrateDBVersionToDB33)
+	m.addMigrations("2.10.0", m.migrateDBVersionToDB34)
+	m.addMigrations("2.9.3", m.migrateDBVersionToDB35)
+	m.addMigrations("2.12", m.migrateDBVersionToDB36)
+	m.addMigrations("2.13", m.migrateDBVersionToDB40)
+	m.addMigrations("2.14", m.migrateDBVersionToDB50)
+	m.addMigrations("2.15", m.migrateDBVersionToDB60)
+	m.addMigrations("2.16", m.migrateDBVersionToDB70)
+	m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
+	m.addMigrations("2.17", m.migrateDBVersionToDB80)
+
+	// Add new migrations below...
+	// One function per migration, each versions migration funcs in the same file.
+}
+
+// Always is always run at the end of migrations
+func (m *Migrator) Always() error {
+	// currently nothing to be done in CE... yet
+	return nil
+}
+
+func dbTooOldError() error {
+	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support")
 }

api/datastore/services.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/dataservices/apikeyrepository"
 	"github.com/portainer/portainer/api/dataservices/customtemplate"
@@ -14,7 +14,6 @@ import (
 	"github.com/portainer/portainer/api/dataservices/edgegroup"
 	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"github.com/portainer/portainer/api/dataservices/edgestack"
-	"github.com/portainer/portainer/api/dataservices/edgeupdateschedule"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
@@ -50,7 +49,6 @@ type Store struct {
 	DockerHubService          *dockerhub.Service
 	EdgeGroupService          *edgegroup.Service
 	EdgeJobService            *edgejob.Service
-	EdgeUpdateScheduleService *edgeupdateschedule.Service
 	EdgeStackService          *edgestack.Service
 	EndpointGroupService      *endpointgroup.Service
 	EndpointService           *endpoint.Service
@@ -95,12 +93,6 @@ func (store *Store) initServices() error {
 	}
 	store.DockerHubService = dockerhubService
 
-	edgeUpdateScheduleService, err := edgeupdateschedule.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeUpdateScheduleService = edgeUpdateScheduleService
-
 	edgeStackService, err := edgestack.NewService(store.connection)
 	if err != nil {
 		return err
@@ -263,11 +255,6 @@ func (store *Store) EdgeJob() dataservices.EdgeJobService {
 	return store.EdgeJobService
 }
 
-// EdgeUpdateSchedule gives access to the EdgeUpdateSchedule data management layer
-func (store *Store) EdgeUpdateSchedule() dataservices.EdgeUpdateScheduleService {
-	return store.EdgeUpdateScheduleService
-}
-
 // EdgeStack gives access to the EdgeStack data management layer
 func (store *Store) EdgeStack() dataservices.EdgeStackService {
 	return store.EdgeStackService
@@ -395,7 +382,7 @@ type storeExport struct {
 	Team               []portainer.Team               `json:"teams,omitempty"`
 	TunnelServer       portainer.TunnelServerInfo     `json:"tunnel_server,omitempty"`
 	User               []portainer.User               `json:"users,omitempty"`
-	Version            map[string]string              `json:"version,omitempty"`
+	Version            models.Version                 `json:"version,omitempty"`
 	Webhook            []portainer.Webhook            `json:"webhooks,omitempty"`
 	Metadata           map[string]interface{}         `json:"metadata,omitempty"`
 }
@@ -588,14 +575,12 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Webhook = webhooks
 	}
 
-	v, err := store.Version().DBVersion()
-	if err != nil && !store.IsErrObjectNotFound(err) {
-		log.Error().Err(err).Msg("exporting DB version")
-	}
-	instance, _ := store.Version().InstanceID()
-	backup.Version = map[string]string{
-		"DB_VERSION":  strconv.Itoa(v),
-		"INSTANCE_ID": instance,
+	if version, err := store.Version().Version(); err != nil {
+		if !store.IsErrObjectNotFound(err) {
+			log.Error().Err(err).Msg("exporting Version")
+		}
+	} else {
+		backup.Version = *version
 	}
 
 	backup.Metadata, err = store.connection.BackupMetadata()
@@ -622,19 +607,7 @@ func (store *Store) Import(filename string) (err error) {
 		return err
 	}
 
-	// TODO: yup, this is bad, and should be in a version struct...
-	if dbversion, ok := backup.Version["DB_VERSION"]; ok {
-		if v, err := strconv.Atoi(dbversion); err == nil {
-			if err := store.Version().StoreDBVersion(v); err != nil {
-				log.Error().Err(err).Msg("DB_VERSION import issue")
-			}
-		}
-	}
-	if instanceID, ok := backup.Version["INSTANCE_ID"]; ok {
-		if err := store.Version().StoreInstanceID(instanceID); err != nil {
-			log.Error().Err(err).Msg("INSTANCE_ID import issue")
-		}
-	}
+	store.Version().UpdateVersion(&backup.Version)
 
 	for _, v := range backup.CustomTemplate {
 		store.CustomTemplate().UpdateCustomTemplate(v.ID, &v)

api/datastore/test_data/output_24_to_latest.json

@@ -66,6 +66,9 @@
       },
       "LastCheckInDate": 0,
       "Name": "local",
+      "PostInitMigrations": {
+        "MigrateIngresses": true
+      },
       "PublicURL": "",
       "QueryDate": 0,
       "SecuritySettings": {
@@ -637,6 +640,7 @@
       "Scopes": "",
       "UserIdentifier": ""
     },
+    "ShowKomposeBuildOption": false,
     "SnapshotInterval": "5m",
     "TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
     "TrustOnFirstConnect": false,
@@ -927,8 +931,6 @@
     }
   ],
   "version": {
-    "DB_UPDATING": "false",
-    "DB_VERSION": "80",
-    "INSTANCE_ID": "null"
+    "VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/datastore/teststore.go

@@ -5,6 +5,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/filesystem"
 
 	"github.com/pkg/errors"
@@ -21,7 +22,9 @@ func MustNewTestStore(t *testing.T, init, secure bool) (bool, *Store, func()) {
 	newStore, store, teardown, err := NewTestStore(t, init, secure)
 	if err != nil {
 		if !errors.Is(err, errTempDir) {
-			teardown()
+			if teardown != nil {
+				teardown()
+			}
 		}
 
 		log.Fatal().Err(err).Msg("")
@@ -33,6 +36,7 @@ func MustNewTestStore(t *testing.T, init, secure bool) (bool, *Store, func()) {
 func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error) {
 	// Creates unique temp directory in a concurrency friendly manner.
 	storePath := t.TempDir()
+
 	fileService, err := filesystem.NewService(storePath, "")
 	if err != nil {
 		return false, nil, nil, err
@@ -54,8 +58,6 @@ func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error)
 		return newStore, nil, nil, err
 	}
 
-	log.Debug().Msg("opened")
-
 	if init {
 		err = store.Init()
 		if err != nil {
@@ -63,11 +65,13 @@ func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error)
 		}
 	}
 
-	log.Debug().Msg("initialised")
-
 	if newStore {
 		// from MigrateData
-		store.VersionService.StoreDBVersion(portainer.DBVersion)
+		v := models.Version{
+			SchemaVersion: portainer.APIVersion,
+			Edition:       int(portainer.PortainerCE),
+		}
+		err = store.VersionService.UpdateVersion(&v)
 		if err != nil {
 			return newStore, nil, nil, err
 		}

api/edgetypes/edgetypes.go

@@ -1,102 +0,0 @@
-package edgetypes
-
-import portainer "github.com/portainer/portainer/api"
-
-const (
-	// PortainerAgentUpdateScheduleIDHeader represents the name of the header containing the update schedule id
-	PortainerAgentUpdateScheduleIDHeader = "X-Portainer-Update-Schedule-ID"
-	// PortainerAgentUpdateStatusHeader is the name of the header that will have the update status
-	PortainerAgentUpdateStatusHeader = "X-Portainer-Update-Status"
-	// PortainerAgentUpdateErrorHeader is the name of the header that will have the update error
-	PortainerAgentUpdateErrorHeader = "X-Portainer-Update-Error"
-)
-
-type (
-
-	// UpdateScheduleID represents an Edge schedule identifier
-	UpdateScheduleID int
-
-	// UpdateSchedule represents a schedule for update/rollback of edge devices
-	UpdateSchedule struct {
-		// EdgeUpdateSchedule Identifier
-		ID UpdateScheduleID `json:"id" example:"1"`
-		// Name of the schedule
-		Name string `json:"name" example:"Update Schedule"`
-		// Type of the schedule
-		Time int64 `json:"time" example:"1564897200"`
-		// EdgeGroups to be updated
-		GroupIDs []portainer.EdgeGroupID `json:"groupIds" example:"1"`
-		// Type of the update (1 - update, 2 - rollback)
-		Type UpdateScheduleType `json:"type" example:"1" enums:"1,2"`
-		// Status of the schedule, grouped by environment id
-		Status map[portainer.EndpointID]UpdateScheduleStatus `json:"status"`
-		// Created timestamp
-		Created int64 `json:"created" example:"1564897200"`
-		// Created by user id
-		CreatedBy portainer.UserID `json:"createdBy" example:"1"`
-	}
-
-	// UpdateScheduleType represents type of an Edge update schedule
-	UpdateScheduleType int
-
-	// UpdateScheduleStatus represents status of an Edge update schedule
-	UpdateScheduleStatus struct {
-		// Status of the schedule (0 - pending, 1 - failed, 2 - success)
-		Status UpdateScheduleStatusType `json:"status" example:"1" enums:"1,2,3"`
-		// Error message if status is failed
-		Error string `json:"error" example:""`
-		// Target version of the edge agent
-		TargetVersion string `json:"targetVersion" example:"1"`
-		// Current version of the edge agent
-		CurrentVersion string `json:"currentVersion" example:"1"`
-	}
-
-	// UpdateScheduleStatusType represents status type of an Edge update schedule
-	UpdateScheduleStatusType int
-
-	VersionUpdateRequest struct {
-		// Target version
-		Version string
-		// Scheduled time
-		ScheduledTime int64
-		// If need to update
-		Active bool
-		// Update schedule ID
-		ScheduleID UpdateScheduleID
-	}
-
-	// VersionUpdateStatus represents the status of an agent version update
-	VersionUpdateStatus struct {
-		Status     UpdateScheduleStatusType
-		ScheduleID UpdateScheduleID
-		Error      string
-	}
-
-	// EndpointUpdateScheduleRelation represents the relation between an environment(endpoint) and an update schedule
-	EndpointUpdateScheduleRelation struct {
-		EnvironmentID portainer.EndpointID     `json:"environmentId"`
-		ScheduleID    UpdateScheduleID         `json:"scheduleId"`
-		TargetVersion string                   `json:"targetVersion"`
-		Status        UpdateScheduleStatusType `json:"status"`
-		Error         string                   `json:"error"`
-		Type          UpdateScheduleType       `json:"type"`
-		ScheduledTime int64                    `json:"scheduledTime"`
-	}
-)
-
-const (
-	_ UpdateScheduleType = iota
-	// UpdateScheduleUpdate represents an edge device scheduled for an update
-	UpdateScheduleUpdate
-	// UpdateScheduleRollback represents an edge device scheduled for a rollback
-	UpdateScheduleRollback
-)
-
-const (
-	// UpdateScheduleStatusPending represents a pending edge update schedule
-	UpdateScheduleStatusPending UpdateScheduleStatusType = iota
-	// UpdateScheduleStatusError represents a failed edge update schedule
-	UpdateScheduleStatusError
-	// UpdateScheduleStatusSuccess represents a successful edge update schedule
-	UpdateScheduleStatusSuccess
-)

api/exec/compose_stack.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	libstack "github.com/portainer/docker-compose-wrapper"
-	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
@@ -25,11 +24,7 @@ type ComposeStackManager struct {
 }
 
 // NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
-	deployer, err := compose.NewComposeDeployer(binaryPath, configPath)
-	if err != nil {
-		return nil, err
-	}
+func NewComposeStackManager(deployer libstack.Deployer, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
 
 	return &ComposeStackManager{
 		deployer:     deployer,
@@ -43,7 +38,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 }
 
 // Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
-func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, forceRereate bool) error {
+func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, forceRecreate bool) error {
 	url, proxy, err := manager.fetchEndpointProxy(endpoint)
 	if err != nil {
 		return errors.Wrap(err, "failed to fetch environment proxy")
@@ -53,13 +48,21 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 		defer proxy.Close()
 	}
 
-	envFile, err := createEnvFile(stack)
+	envFilePath, err := createEnvFile(stack)
 	if err != nil {
 		return errors.Wrap(err, "failed to create env file")
 	}
 
 	filePaths := stackutils.GetStackFilePaths(stack, false)
-	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile, forceRereate)
+	err = manager.deployer.Deploy(ctx, filePaths, libstack.DeployOptions{
+		Options: libstack.Options{
+			WorkingDir:  stack.ProjectPath,
+			EnvFilePath: envFilePath,
+			Host:        url,
+			ProjectName: stack.Name,
+		},
+		ForceRecreate: forceRecreate,
+	})
 	return errors.Wrap(err, "failed to deploy a stack")
 }
 
@@ -73,14 +76,19 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		defer proxy.Close()
 	}
 
-	envFile, err := createEnvFile(stack)
+	envFilePath, err := createEnvFile(stack)
 	if err != nil {
 		return errors.Wrap(err, "failed to create env file")
 	}
 
 	filePaths := stackutils.GetStackFilePaths(stack, false)
 
-	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
+	err = manager.deployer.Remove(ctx, filePaths, libstack.Options{
+		WorkingDir:  stack.ProjectPath,
+		EnvFilePath: envFilePath,
+		Host:        url,
+		ProjectName: stack.Name,
+	})
 	return errors.Wrap(err, "failed to remove a stack")
 }
 
@@ -95,13 +103,18 @@ func (manager *ComposeStackManager) Pull(ctx context.Context, stack *portainer.S
 		defer proxy.Close()
 	}
 
-	envFile, err := createEnvFile(stack)
+	envFilePath, err := createEnvFile(stack)
 	if err != nil {
 		return errors.Wrap(err, "failed to create env file")
 	}
 
 	filePaths := stackutils.GetStackFilePaths(stack, false)
-	err = manager.deployer.Pull(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
+	err = manager.deployer.Pull(ctx, filePaths, libstack.Options{
+		WorkingDir:  stack.ProjectPath,
+		EnvFilePath: envFilePath,
+		Host:        url,
+		ProjectName: stack.Name,
+	})
 	return errors.Wrap(err, "failed to pull images of the stack")
 }
 

api/exec/compose_stack_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/testhelpers"
 
@@ -47,7 +48,12 @@ func Test_UpAndDown(t *testing.T) {
 
 	stack, endpoint := setup(t)
 
-	w, err := NewComposeStackManager("", "", nil)
+	deployer, err := compose.NewComposeDeployer("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w, err := NewComposeStackManager(deployer, nil)
 	if err != nil {
 		t.Fatalf("Failed creating manager: %s", err)
 	}

api/exec/kubernetes_deploy.go

@@ -49,7 +49,7 @@ func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *
 		return "", err
 	}
 
-	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(int(endpoint.ID))
+	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(endpoint.ID)
 
 	tokenManager, err := kubernetes.NewTokenManager(kubeCLI, deployer.dataStore, tokenCache, setLocalAdminToken)
 	if err != nil {

api/filesystem/filesystem.go

@@ -62,6 +62,10 @@ const (
 	SSLKeyFilename = "key.pem"
 	// SSLCACertFilename represents the CA ssl certificate file name for mTLS
 	SSLCACertFilename = "ca-cert.pem"
+
+	MTLSCertFilename   = "mtls-cert.pem"
+	MTLSCACertFilename = "mtls-ca-cert.pem"
+	MTLSKeyFilename    = "mtls-key.pem"
 )
 
 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -489,14 +493,7 @@ func (service *Service) createDirectoryInStore(name string) error {
 func (service *Service) createFileInStore(filePath string, r io.Reader) error {
 	path := service.wrapFileStore(filePath)
 
-	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	_, err = io.Copy(out, r)
-	return err
+	return CreateFile(path, r)
 }
 
 // createBackupFileInStore makes a copy in the file store.
@@ -670,6 +667,14 @@ func (service *Service) GetDefaultSSLCertsPath() (string, string) {
 	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath)
 }
 
+func defaultMTLSCertPathUnderFileStore() (string, string, string) {
+	certPath := JoinPaths(SSLCertPath, MTLSCertFilename)
+	caCertPath := JoinPaths(SSLCertPath, MTLSCACertFilename)
+	keyPath := JoinPaths(SSLCertPath, MTLSKeyFilename)
+
+	return certPath, caCertPath, keyPath
+}
+
 // StoreSSLCertPair stores a ssl certificate pair
 func (service *Service) StoreSSLCertPair(cert, key []byte) (string, string, error) {
 	certPath, keyPath := defaultCertPathUnderFileStore()
@@ -762,3 +767,39 @@ func (service *Service) StoreFDOProfileFileFromBytes(fdoProfileIdentifier string
 
 	return service.wrapFileStore(filePath), nil
 }
+
+func CreateFile(path string, r io.Reader) error {
+	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	_, err = io.Copy(out, r)
+	return err
+}
+
+func (service *Service) StoreMTLSCertificates(cert, caCert, key []byte) (string, string, string, error) {
+	certPath, caCertPath, keyPath := defaultMTLSCertPathUnderFileStore()
+
+	r := bytes.NewReader(cert)
+	err := service.createFileInStore(certPath, r)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	r = bytes.NewReader(caCert)
+	err = service.createFileInStore(caCertPath, r)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	r = bytes.NewReader(key)
+	err = service.createFileInStore(keyPath, r)
+	if err != nil {
+		return "", "", "", err
+	}
+
+	return service.wrapFileStore(certPath), service.wrapFileStore(caCertPath), service.wrapFileStore(keyPath), nil
+}

api/go.mod

@@ -3,11 +3,13 @@ module github.com/portainer/portainer/api
 go 1.18
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/go-winio v0.5.1
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/aws/aws-sdk-go-v2 v1.11.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.6.2
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.10.1
+	github.com/cbroglie/mustache v1.4.0
 	github.com/coreos/go-semver v0.3.0
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
 	github.com/docker/cli v20.10.9+incompatible
@@ -33,29 +35,30 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021
+	github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27
 	github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a
-	github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9
-	github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a
-	github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777
+	github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9
+	github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724
+	github.com/portainer/portainer/third_party/digest v0.0.0-20221201002639-8fd0efa34f73
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.28.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.1
 	github.com/viney-shih/go-lock v1.1.1
 	go.etcd.io/bbolt v1.3.6
-	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
-	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
+	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.22.5
-	k8s.io/apimachinery v0.22.5
-	k8s.io/client-go v0.22.5
+	k8s.io/api v0.25.3
+	k8s.io/apimachinery v0.25.3
+	k8s.io/client-go v0.25.3
 	software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78
 )
 
 require (
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
 	github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 // indirect
@@ -64,32 +67,38 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 // indirect
 	github.com/aws/smithy-go v1.9.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.0+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.3.1 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.1.0 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/ansi v1.0.2 // indirect
 	github.com/jpillora/requestlog v1.0.0 // indirect
 	github.com/jpillora/sizestr v1.0.0 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -99,8 +108,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.15.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -113,20 +121,21 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/net v0.2.0 // indirect
+	golang.org/x/sys v0.2.0 // indirect
+	golang.org/x/term v0.2.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect
-	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	k8s.io/klog/v2 v2.70.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

api/go.sum

@@ -33,21 +33,17 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
@@ -61,7 +57,6 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/aws/aws-sdk-go-v2 v1.11.1 h1:GzvOVAdTbWxhEMRK4FfiblkGverOkAT0UodDxC1jHQM=
@@ -80,6 +75,8 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.6.1/go.mod h1:/73aFBwUl60wKBKhdth2pE
 github.com/aws/aws-sdk-go-v2/service/sts v1.10.1/go.mod h1:+BmlPeQ1Y+PuIho93MMKDby12PoUnt1SZXQdEHCzSlw=
 github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58=
 github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
+github.com/cbroglie/mustache v1.4.0 h1:Azg0dVhxTml5me+7PsZ7WPrQq1Gkf3WApcHMjMprYoU=
+github.com/cbroglie/mustache v1.4.0/go.mod h1:SS1FTIghy0sjse4DUVGV1k/40B1qE1XkD9DtDsHo9iM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -98,8 +95,8 @@ github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfD
 github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4=
 github.com/docker/cli v20.10.9+incompatible h1:OJ7YkwQA+k2Oi51lmCojpjiygKpi76P7bg91b2eJxYU=
 github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v2.8.0+incompatible h1:l9EaZDICImO1ngI+uTifW+ZYvvz7fKISBAKpg+MbWbY=
-github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
+github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ=
 github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@@ -108,23 +105,20 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
+github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
@@ -152,22 +146,25 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
 github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
 github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig=
 github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -179,7 +176,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -205,7 +201,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
+github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -219,7 +216,6 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -234,13 +230,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
@@ -251,14 +242,11 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -270,6 +258,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/ansi v0.0.0-20170202005112-f496b27cd669/go.mod h1:kOeLNvjNBGSV3uYtFjvb72+fnZCMFJF1XDvRIjdom0g=
 github.com/jpillora/ansi v1.0.2 h1:+Ei5HCAH0xsrQRCT2PDr4mq9r4Gm4tg+arNdXRkB22s=
 github.com/jpillora/ansi v1.0.2/go.mod h1:D2tT+6uzJvN1nBVQILYWkIdq7zG+b5gcFN5WI/VyjMY=
@@ -282,8 +272,6 @@ github.com/jpillora/requestlog v1.0.0/go.mod h1:HTWQb7QfDc2jtHnWe2XEIEeJB7gJPnVd
 github.com/jpillora/sizestr v0.0.0-20160130011556-e2ea2fa42fb9/go.mod h1:1ffp+CRe0eAwwRb0/BownUAjMBsmTLwgAvRbfj9dRwE=
 github.com/jpillora/sizestr v1.0.0 h1:4tr0FLxs1Mtq3TnsLDV+GYUWG7Q26a6s+tV5Zfw2ygw=
 github.com/jpillora/sizestr v1.0.0/go.mod h1:bUhLv4ctkknatr6gR42qPxirmd5+ds1u7mzD+MZ33f0=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -308,6 +296,8 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
@@ -323,29 +313,15 @@ github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6U
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
-github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
+github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
@@ -354,26 +330,25 @@ github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THr
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021 h1:GFTn2e5AyIoBuK6hXbdVNkuV2m450DQnYmgQDZRU3x8=
-github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
+github.com/portainer/docker-compose-wrapper v0.0.0-20221122145319-915b021aea84 h1:d1P8i0pCPvAfxH6nSLUFm6NYoi8tMrIpafaZXSV8Lac=
+github.com/portainer/docker-compose-wrapper v0.0.0-20221122145319-915b021aea84/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
+github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27 h1:PceCpp86SDYb3lZHT4KpuBCkmcJMW5x1qrdFNEfAdUo=
+github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a h1:B0z3skIMT+OwVNJPQhKp52X+9OWW6A9n5UWig3lHBJk=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
-github.com/portainer/libhelm v0.0.0-20221017001602-b619e0ddf8fc h1:Ms8ZDGvSydX7aoCfu06Rk8II2N4tlX3cn7whtGC5fDA=
-github.com/portainer/libhelm v0.0.0-20221017001602-b619e0ddf8fc/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
-github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9 h1:PMw+45hocpYsMQ7WLIlzsf4854BhDM8C631kr0DVJtc=
-github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
-github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a h1:BJ5V4EDNhg3ImYbmXnGS8vrMhq6rzsEneIXyJh0g4dc=
-github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a/go.mod h1:ckuHnoLA5kLuE5WkvPBXmrw63LUMdSH4aX71QRi9y10=
+github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9 h1:L7o0L+1qq+LzKjzgRB6bDIh5ZrZ5A1oSS+WgWzDgJIo=
+github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9/go.mod h1:H49JLiywwLt2rrJVroafEWy8fIs0i7mThAThK40sbb8=
+github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724 h1:FZrRVMpxXdUV+p5VSCAy9Uz7RzAeEJr2ytlctvMrsHY=
+github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724/go.mod h1:WUdwNVH9GMffP4qf4U2ea2qCYfti2V7S+IhGpO8Sxv0=
+github.com/portainer/portainer/third_party/digest v0.0.0-20221201002639-8fd0efa34f73 h1:7bPOnwucE0nor0so1BQJxQKCL5t+vCWO4nAz/S0lci0=
+github.com/portainer/portainer/third_party/digest v0.0.0-20221201002639-8fd0efa34f73/go.mod h1:E2w/A6qsKuG2VyiUubPdXpDyPykWfQqxuCs0YNS0MhM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777 h1:rDj3WeO+TiWyxfcydUnKegWAZoR5kQsnW0wzhggdOrw=
-github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777/go.mod h1:xRVvTK+cS/dJSvrOufGUQFWfgvE7yXExeng96n8377o=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -390,20 +365,24 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
 github.com/viney-shih/go-lock v1.1.1 h1:SwzDPPAiHpcwGCr5k8xD15d2gQSo8d4roRYd7TDV2eI=
@@ -436,11 +415,11 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -451,8 +430,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0=
-golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -475,7 +452,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181017193950-04a2e542c03f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -498,7 +474,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -506,18 +481,16 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -531,7 +504,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181019160139-8e24a49d80f8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -541,12 +513,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -560,13 +529,11 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -576,26 +543,26 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -638,7 +605,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -722,8 +688,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -733,14 +699,10 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -749,7 +711,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -763,28 +724,26 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.22.5 h1:xk7C+rMjF/EGELiD560jdmwzrB788mfcHiNbMQLIVI8=
-k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs=
-k8s.io/apimachinery v0.22.5 h1:cIPwldOYm1Slq9VLBRPtEYpyhjIm1C6aAMAoENuvN9s=
-k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U=
-k8s.io/client-go v0.22.5 h1:I8Zn/UqIdi2r02aZmhaJ1hqMxcpfJ3t5VqvHtctHYFo=
-k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
+k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
+k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
+k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
+k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=
+k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80=
-k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
-k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
+k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
+k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
+k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
+k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 h1:SqYE5+A2qvRhErbsXFfUEUmpWEKxxRSMgGLkvRAFOV4=

api/hostmanagement/fdo/owner_client.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/rkl-/digest"
+	"github.com/portainer/portainer/third_party/digest"
 )
 
 type FDOOwnerClient struct {

api/http/handler/auth/logout.go

@@ -23,7 +23,7 @@ func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperro
 		return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
 	}
 
-	handler.KubernetesTokenCacheManager.RemoveUserFromCache(int(tokenData.ID))
+	handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
 
 	return response.Empty(w)
 }

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,6 +3,7 @@ package customtemplates
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"regexp"
@@ -17,7 +18,6 @@ import (
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/portainer/portainer/api/stacks/stackutils"
 	"github.com/rs/zerolog/log"
 )
 
@@ -213,8 +213,6 @@ type customTemplateFromGitRepositoryPayload struct {
 	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
-	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
-	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -234,10 +232,14 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
 	}
 
-	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
+	if payload.Type == portainer.KubernetesStack {
+		return errors.New("Creating a Kubernetes custom template from git is not supported")
+	}
+
+	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
-	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack {
+	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
 	if !isValidNote(payload.Note) {
@@ -256,44 +258,35 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:              portainer.CustomTemplateID(customTemplateID),
-		Title:           payload.Title,
-		Description:     payload.Description,
-		Note:            payload.Note,
-		Platform:        payload.Platform,
-		Type:            payload.Type,
-		Logo:            payload.Logo,
-		Variables:       payload.Variables,
-		IsComposeFormat: payload.IsComposeFormat,
+		ID:          portainer.CustomTemplateID(customTemplateID),
+		Title:       payload.Title,
+		EntryPoint:  payload.ComposeFilePathInRepository,
+		Description: payload.Description,
+		Note:        payload.Note,
+		Platform:    payload.Platform,
+		Type:        payload.Type,
+		Logo:        payload.Logo,
+		Variables:   payload.Variables,
 	}
 
-	getProjectPath := func() string {
-		return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
-	}
-	projectPath := getProjectPath()
+	projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
 	customTemplate.ProjectPath = projectPath
 
-	gitConfig := &gittypes.RepoConfig{
-		URL:            payload.RepositoryURL,
-		ReferenceName:  payload.RepositoryReferenceName,
-		ConfigFilePath: payload.ComposeFilePathInRepository,
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	if payload.RepositoryAuthentication {
-		gitConfig.Authentication = &gittypes.GitAuthentication{
-			Username: payload.RepositoryUsername,
-			Password: payload.RepositoryPassword,
-		}
-	}
-
-	commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath)
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
+		if err == gittypes.ErrAuthenticationFailure {
+			return nil, fmt.Errorf("invalid git credential")
+		}
 		return nil, err
 	}
 
-	gitConfig.ConfigHash = commitHash
-	customTemplate.GitConfig = gitConfig
-
 	isValidProject := true
 	defer func() {
 		if !isValidProject {
@@ -303,7 +296,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		}
 	}()
 
-	entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath)
+	entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint)
 
 	exists, err := handler.FileService.FileExists(entryPath)
 	if err != nil || !exists {
@@ -315,9 +308,6 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 	}
 
 	if !exists {
-		if payload.Type == portainer.KubernetesStack {
-			return nil, errors.New("Invalid Manifest file, ensure that the Manifest file path is correct")
-		}
 		return nil, errors.New("Invalid Compose file, ensure that the Compose file path is correct")
 	}
 

api/http/handler/customtemplates/customtemplate_file.go

@@ -40,11 +40,7 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques
 		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
 	}
 
-	entryPath := customTemplate.EntryPoint
-	if customTemplate.GitConfig != nil {
-		entryPath = customTemplate.GitConfig.ConfigFilePath
-	}
-	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath)
+	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint)
 	if err != nil {
 		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
 	}

api/http/handler/customtemplates/customtemplate_git_fetch.go

@@ -1,119 +0,0 @@
-package customtemplates
-
-import (
-	"fmt"
-	"net/http"
-	"os"
-	"sync"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/stacks/stackutils"
-)
-
-// @id CustomTemplateGitFetch
-// @summary Fetch the latest config file content based on custom template's git repository configuration
-// @description Retrieve details about a template created from git repository method.
-// @description **Access policy**: authenticated
-// @tags custom_templates
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @param id path int true "Template identifier"
-// @success 200 {object} fileResponse "Success"
-// @failure 400 "Invalid request"
-// @failure 404 "Custom template not found"
-// @failure 500 "Server error"
-// @router /custom_templates/{id}/git_fetch [put]
-func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest("Invalid Custom template identifier route variable", err)
-	}
-
-	customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
-	}
-
-	if customTemplate.GitConfig == nil {
-		return httperror.BadRequest("Git configuration does not exist in this custom template", err)
-	}
-
-	// If multiple users are trying to fetch the same custom template simultaneously, a lock needs to be added
-	mu, ok := handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)]
-	if !ok {
-		mu = &sync.Mutex{}
-		handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] = mu
-	}
-	mu.Lock()
-	defer mu.Unlock()
-
-	// back up the current custom template folder
-	backupPath, err := backupCustomTemplate(customTemplate.ProjectPath)
-	if err != nil {
-		return httperror.InternalServerError("Failed to backup the custom template folder", err)
-	}
-
-	// remove backup custom template folder
-	defer cleanUpBackupCustomTemplate(backupPath)
-
-	commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string {
-		return customTemplate.ProjectPath
-	})
-	if err != nil {
-		err = rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
-		if err != nil {
-			return httperror.InternalServerError("Failed to rollback the custom template folder", err)
-		}
-		return httperror.InternalServerError(err.Error(), err)
-	}
-
-	if customTemplate.GitConfig.ConfigHash != commitHash {
-		customTemplate.GitConfig.ConfigHash = commitHash
-
-		err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
-		if err != nil {
-			return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
-		}
-	}
-
-	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.GitConfig.ConfigFilePath)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
-	}
-
-	return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
-}
-
-func backupCustomTemplate(projectPath string) (string, error) {
-	stat, err := os.Stat(projectPath)
-	if err != nil {
-		return "", err
-	}
-
-	backupPath := fmt.Sprintf("%s-backup", projectPath)
-	err = os.Rename(projectPath, backupPath)
-	if err != nil {
-		return "", err
-	}
-
-	err = os.Mkdir(projectPath, stat.Mode())
-	if err != nil {
-		return backupPath, err
-	}
-	return backupPath, nil
-}
-
-func rollbackCustomTemplate(backupPath, projectPath string) error {
-	os.RemoveAll(projectPath)
-	return os.Rename(backupPath, projectPath)
-}
-
-func cleanUpBackupCustomTemplate(backupPath string) error {
-	return os.RemoveAll(backupPath)
-}

api/http/handler/customtemplates/customtemplate_git_fetch_test.go

@@ -1,174 +0,0 @@
-package customtemplates
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"sync"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/datastore"
-	gittypes "github.com/portainer/portainer/api/git/types"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/portainer/portainer/api/jwt"
-	"github.com/stretchr/testify/assert"
-)
-
-var testFileContent string = "abcdefg"
-
-type TestGitService struct {
-	portainer.GitService
-	targetFilePath string
-}
-
-func (g *TestGitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string) error {
-	time.Sleep(100 * time.Millisecond)
-	return createTestFile(g.targetFilePath)
-}
-
-func (g *TestGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	return "", nil
-}
-
-type TestFileService struct {
-	portainer.FileService
-}
-
-func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]byte, error) {
-	return os.ReadFile(filepath.Join(projectPath, configFilePath))
-}
-
-func createTestFile(targetPath string) error {
-	f, err := os.Create(targetPath)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	_, err = f.WriteString(testFileContent)
-	return err
-}
-
-func prepareTestFolder(projectPath, filename string) error {
-	err := os.MkdirAll(projectPath, fs.ModePerm)
-	if err != nil {
-		return err
-	}
-
-	return createTestFile(filepath.Join(projectPath, filename))
-}
-
-func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect string) {
-	type response struct {
-		FileContent string
-	}
-
-	req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
-	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
-
-	rr := httptest.NewRecorder()
-	h.ServeHTTP(rr, req)
-
-	is.Equal(http.StatusOK, rr.Code)
-
-	body, err := io.ReadAll(rr.Body)
-	is.NoError(err, "ReadAll should not return error")
-
-	var resp response
-	err = json.Unmarshal(body, &resp)
-	is.NoError(err, "response should be list json")
-	is.Equal(resp.FileContent, expect)
-}
-
-func Test_customTemplateGitFetch(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
-
-	// create user(s)
-	user1 := &portainer.User{ID: 1, Username: "user-1", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
-	err := store.User().Create(user1)
-	is.NoError(err, "error creating user 1")
-
-	user2 := &portainer.User{ID: 2, Username: "user-2", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
-	err = store.User().Create(user2)
-	is.NoError(err, "error creating user 2")
-
-	dir, err := os.Getwd()
-	is.NoError(err, "error to get working directory")
-
-	template1 := &portainer.CustomTemplate{ID: 1, Title: "custom-template-1", ProjectPath: filepath.Join(dir, "fixtures/custom_template_1"), GitConfig: &gittypes.RepoConfig{ConfigFilePath: "test-config-path.txt"}}
-	err = store.CustomTemplateService.Create(template1)
-	is.NoError(err, "error creating custom template 1")
-
-	// prepare testing folder
-	err = prepareTestFolder(template1.ProjectPath, template1.GitConfig.ConfigFilePath)
-	is.NoError(err, "error creating testing folder")
-
-	defer os.RemoveAll(filepath.Join(dir, "fixtures"))
-
-	// setup services
-	jwtService, err := jwt.NewService("1h", store)
-	is.NoError(err, "Error initiating jwt service")
-	requestBouncer := security.NewRequestBouncer(store, jwtService, nil)
-
-	gitService := &TestGitService{
-		targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath),
-	}
-	fileService := &TestFileService{}
-
-	h := NewHandler(requestBouncer, store, fileService, gitService)
-
-	// generate standard and admin user tokens
-	jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
-	jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
-
-	t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
-		singleAPIRequest(h, jwt1, is, "abcdefg")
-	})
-
-	t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) {
-		var wg sync.WaitGroup
-		wg.Add(5)
-		for i := 0; i < 5; i++ {
-			go func() {
-				singleAPIRequest(h, jwt1, is, "abcdefg")
-				wg.Done()
-			}()
-		}
-		wg.Wait()
-	})
-
-	t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) {
-		var wg sync.WaitGroup
-		wg.Add(10)
-		for i := 0; i < 10; i++ {
-			go func(j int) {
-				if j%1 == 0 {
-					singleAPIRequest(h, jwt1, is, "abcdefg")
-				} else {
-					singleAPIRequest(h, jwt2, is, "abcdefg")
-				}
-				wg.Done()
-			}(i)
-		}
-		wg.Wait()
-	})
-
-	t.Run("can return the expected file content after a new commit is made", func(t *testing.T) {
-		singleAPIRequest(h, jwt1, is, "abcdefg")
-
-		testFileContent = "gfedcba"
-
-		singleAPIRequest(h, jwt2, is, "gfedcba")
-	})
-}

api/http/handler/customtemplates/customtemplate_update.go

@@ -3,17 +3,15 @@ package customtemplates
 import (
 	"errors"
 	"net/http"
+	"strconv"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
-	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/stacks/stackutils"
 )
 
 type customTemplateUpdatePayload struct {
@@ -31,37 +29,18 @@ type customTemplateUpdatePayload struct {
 	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
 	// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
 	Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
-	// URL of a Git repository hosting the Stack file
-	RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
-	// Reference name of a Git repository hosting the Stack file
-	RepositoryReferenceName string `example:"refs/heads/master"`
-	// Use basic authentication to clone the Git repository
-	RepositoryAuthentication bool `example:"true"`
-	// Username used in basic authentication. Required when RepositoryAuthentication is true
-	// and RepositoryGitCredentialID is 0
-	RepositoryUsername string `example:"myGitUsername"`
-	// Password used in basic authentication. Required when RepositoryAuthentication is true
-	// and RepositoryGitCredentialID is 0
-	RepositoryPassword string `example:"myGitPassword"`
-	// GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication
-	// is true and RepositoryUsername/RepositoryPassword are not provided
-	RepositoryGitCredentialID int `example:"0"`
-	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Content of stack file
 	FileContent string `validate:"required"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
-	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
-	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Title) {
 		return errors.New("Invalid custom template title")
 	}
-	if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) {
-		return errors.New("Either file content or git repository url need to be provided")
+	if govalidator.IsNull(payload.FileContent) {
+		return errors.New("Invalid file content")
 	}
 	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
@@ -76,13 +55,6 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
-	}
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
-	}
-
 	return validateVariablesDefinitions(payload.Variables)
 }
 
@@ -143,6 +115,12 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 		return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
 	}
 
+	templateFolder := strconv.Itoa(customTemplateID)
+	_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
+	if err != nil {
+		return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
+	}
+
 	customTemplate.Title = payload.Title
 	customTemplate.Logo = payload.Logo
 	customTemplate.Description = payload.Description
@@ -150,36 +128,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.Platform = payload.Platform
 	customTemplate.Type = payload.Type
 	customTemplate.Variables = payload.Variables
-	customTemplate.IsComposeFormat = payload.IsComposeFormat
-
-	if payload.RepositoryURL != "" {
-		if !govalidator.IsURL(payload.RepositoryURL) {
-			return httperror.BadRequest("Invalid repository URL. Must correspond to a valid URL format", err)
-		}
-
-		gitConfig := &gittypes.RepoConfig{
-			URL:            payload.RepositoryURL,
-			ReferenceName:  payload.RepositoryReferenceName,
-			ConfigFilePath: payload.ComposeFilePathInRepository,
-		}
-
-		if payload.RepositoryAuthentication {
-			gitConfig.Authentication = &gittypes.GitAuthentication{
-				Username: payload.RepositoryUsername,
-				Password: payload.RepositoryPassword,
-			}
-		}
-
-		commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string {
-			return customTemplate.ProjectPath
-		})
-		if err != nil {
-			return httperror.InternalServerError(err.Error(), err)
-		}
-
-		gitConfig.ConfigHash = commitHash
-		customTemplate.GitConfig = gitConfig
-	}
 
 	err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
 	if err != nil {

api/http/handler/customtemplates/handler.go

@@ -2,7 +2,6 @@ package customtemplates
 
 import (
 	"net/http"
-	"sync"
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
@@ -14,20 +13,15 @@ import (
 // Handler is the HTTP handler used to handle environment(endpoint) group operations.
 type Handler struct {
 	*mux.Router
-	DataStore      dataservices.DataStore
-	FileService    portainer.FileService
-	GitService     portainer.GitService
-	gitFetchMutexs map[portainer.TemplateID]*sync.Mutex
+	DataStore   dataservices.DataStore
+	FileService portainer.FileService
+	GitService  portainer.GitService
 }
 
 // NewHandler creates a handler to manage environment(endpoint) group operations.
-func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, fileService portainer.FileService, gitService portainer.GitService) *Handler {
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h := &Handler{
-		Router:         mux.NewRouter(),
-		DataStore:      dataStore,
-		FileService:    fileService,
-		GitService:     gitService,
-		gitFetchMutexs: make(map[portainer.TemplateID]*sync.Mutex),
+		Router: mux.NewRouter(),
 	}
 	h.Handle("/custom_templates",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateCreate))).Methods(http.MethodPost)
@@ -41,8 +35,6 @@ func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataSto
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut)
 	h.Handle("/custom_templates/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete)
-	h.Handle("/custom_templates/{id}/git_fetch",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut)
 	return h
 }
 

api/http/handler/docker/containers/container_gpus_inspect.go

@@ -10,7 +10,7 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/middlewares"
-	"golang.org/x/exp/slices"
+	"github.com/portainer/portainer/api/internal/slices"
 )
 
 type containerGpusResponse struct {

api/http/handler/edgegroups/edgegroup_create.go

@@ -23,13 +23,13 @@ type edgeGroupCreatePayload struct {
 
 func (payload *edgeGroupCreatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid Edge group name")
+		return errors.New("invalid Edge group name")
 	}
 	if payload.Dynamic && (payload.TagIDs == nil || len(payload.TagIDs) == 0) {
-		return errors.New("TagIDs is mandatory for a dynamic Edge group")
+		return errors.New("tagIDs is mandatory for a dynamic Edge group")
 	}
 	if !payload.Dynamic && (payload.Endpoints == nil || len(payload.Endpoints) == 0) {
-		return errors.New("Environment is mandatory for a static Edge group")
+		return errors.New("environment is mandatory for a static Edge group")
 	}
 	return nil
 }
@@ -61,7 +61,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 
 	for _, edgeGroup := range edgeGroups {
 		if edgeGroup.Name == payload.Name {
-			return httperror.BadRequest("Edge group name must be unique", errors.New("Edge group name must be unique"))
+			return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
 		}
 	}
 

api/http/handler/edgegroups/edgegroup_delete.go

@@ -42,7 +42,20 @@ func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request)
 	for _, edgeStack := range edgeStacks {
 		for _, groupID := range edgeStack.EdgeGroups {
 			if groupID == portainer.EdgeGroupID(edgeGroupID) {
-				return httperror.Forbidden("Edge group is used by an Edge stack", errors.New("Edge group is used by an Edge stack"))
+				return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge stack", errors.New("edge group is used by an Edge stack"))
+			}
+		}
+	}
+
+	edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err)
+	}
+
+	for _, edgeJob := range edgeJobs {
+		for _, groupID := range edgeJob.EdgeGroups {
+			if groupID == portainer.EdgeGroupID(edgeGroupID) {
+				return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge job", errors.New("edge group is used by an Edge job"))
 			}
 		}
 	}

api/http/handler/edgegroups/edgegroup_list.go

@@ -8,11 +8,13 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/slices"
 )
 
 type decoratedEdgeGroup struct {
 	portainer.EdgeGroup
 	HasEdgeStack  bool `json:"HasEdgeStack"`
+	HasEdgeGroup  bool `json:"HasEdgeGroup"`
 	EndpointTypes []portainer.EndpointType
 }
 
@@ -46,8 +48,21 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h
 		}
 	}
 
+	edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err)
+	}
+
 	decoratedEdgeGroups := []decoratedEdgeGroup{}
 	for _, orgEdgeGroup := range edgeGroups {
+		usedByEdgeJob := false
+		for _, edgeJob := range edgeJobs {
+			if slices.Contains(edgeJob.EdgeGroups, portainer.EdgeGroupID(orgEdgeGroup.ID)) {
+				usedByEdgeJob = true
+				break
+			}
+		}
+
 		edgeGroup := decoratedEdgeGroup{
 			EdgeGroup:     orgEdgeGroup,
 			EndpointTypes: []portainer.EndpointType{},
@@ -63,13 +78,15 @@ func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *h
 
 		endpointTypes, err := getEndpointTypes(handler.DataStore.Endpoint(), edgeGroup.Endpoints)
 		if err != nil {
-			return httperror.InternalServerError("Unable to retrieve endpoint types for Edge group", err)
+			return httperror.InternalServerError("Unable to retrieve environment types for Edge group", err)
 		}
 
 		edgeGroup.EndpointTypes = endpointTypes
 
 		edgeGroup.HasEdgeStack = usedEdgeGroups[edgeGroup.ID]
 
+		edgeGroup.HasEdgeGroup = usedByEdgeJob
+
 		decoratedEdgeGroups = append(decoratedEdgeGroups, edgeGroup)
 	}
 

api/http/handler/edgegroups/edgegroup_update.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/internal/slices"
 
 	"github.com/asaskevich/govalidator"
 )
@@ -24,13 +25,13 @@ type edgeGroupUpdatePayload struct {
 
 func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid Edge group name")
+		return errors.New("invalid Edge group name")
 	}
 	if payload.Dynamic && (payload.TagIDs == nil || len(payload.TagIDs) == 0) {
-		return errors.New("TagIDs is mandatory for a dynamic Edge group")
+		return errors.New("tagIDs is mandatory for a dynamic Edge group")
 	}
 	if !payload.Dynamic && (payload.Endpoints == nil || len(payload.Endpoints) == 0) {
-		return errors.New("Environments is mandatory for a static Edge group")
+		return errors.New("environments is mandatory for a static Edge group")
 	}
 	return nil
 }
@@ -75,7 +76,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 		}
 		for _, edgeGroup := range edgeGroups {
 			if edgeGroup.Name == payload.Name && edgeGroup.ID != portainer.EdgeGroupID(edgeGroupID) {
-				return httperror.BadRequest("Edge group name must be unique", errors.New("Edge group name must be unique"))
+				return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
 			}
 		}
 
@@ -123,17 +124,45 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 	newRelatedEndpoints := edge.EdgeGroupRelatedEndpoints(edgeGroup, endpoints, endpointGroups)
 	endpointsToUpdate := append(newRelatedEndpoints, oldRelatedEndpoints...)
 
+	edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return httperror.InternalServerError("Unable to fetch Edge jobs", err)
+	}
+
 	for _, endpointID := range endpointsToUpdate {
-		err = handler.updateEndpoint(endpointID)
+		err = handler.updateEndpointStacks(endpointID)
 		if err != nil {
 			return httperror.InternalServerError("Unable to persist Environment relation changes inside the database", err)
 		}
+
+		endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Environment from database", err)
+		}
+
+		if !endpointutils.IsEdgeEndpoint(endpoint) {
+			continue
+		}
+
+		var operation string
+		if slices.Contains(newRelatedEndpoints, endpointID) {
+			operation = "add"
+		} else if slices.Contains(oldRelatedEndpoints, endpointID) {
+			operation = "remove"
+		} else {
+			continue
+		}
+
+		err = handler.updateEndpointEdgeJobs(edgeGroup.ID, endpointID, edgeJobs, operation)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist Environment Edge Jobs changes inside the database", err)
+		}
 	}
 
 	return response.JSON(w, edgeGroup)
 }
 
-func (handler *Handler) updateEndpoint(endpointID portainer.EndpointID) error {
+func (handler *Handler) updateEndpointStacks(endpointID portainer.EndpointID) error {
 	relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 	if err != nil {
 		return err
@@ -170,3 +199,20 @@ func (handler *Handler) updateEndpoint(endpointID portainer.EndpointID) error {
 
 	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 }
+
+func (handler *Handler) updateEndpointEdgeJobs(edgeGroupID portainer.EdgeGroupID, endpointID portainer.EndpointID, edgeJobs []portainer.EdgeJob, operation string) error {
+	for _, edgeJob := range edgeJobs {
+		if !slices.Contains(edgeJob.EdgeGroups, edgeGroupID) {
+			continue
+		}
+
+		switch operation {
+		case "add":
+			handler.ReverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
+		case "remove":
+			handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
+		}
+	}
+
+	return nil
+}

api/http/handler/edgegroups/handler.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/security"
 )
@@ -12,7 +13,8 @@ import (
 // Handler is the HTTP handler used to handle environment(endpoint) group operations.
 type Handler struct {
 	*mux.Router
-	DataStore dataservices.DataStore
+	DataStore            dataservices.DataStore
+	ReverseTunnelService portainer.ReverseTunnelService
 }
 
 // NewHandler creates a handler to manage environment(endpoint) group operations.

api/http/handler/edgejobs/edgejob_create.go

@@ -12,7 +12,9 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/internal/maps"
 )
 
 // @id EdgeJobCreate
@@ -41,7 +43,7 @@ func (handler *Handler) edgeJobCreate(w http.ResponseWriter, r *http.Request) *h
 	case "file":
 		return handler.createEdgeJobFromFile(w, r)
 	default:
-		return httperror.BadRequest("Invalid query parameter: method. Valid values are: file or string", errors.New(request.ErrInvalidQueryParameter))
+		return httperror.BadRequest("Invalid query parameter: method. Valid values are: file or string", errors.New(strings.ToLower(request.ErrInvalidQueryParameter)))
 	}
 }
 
@@ -50,28 +52,29 @@ type edgeJobCreateFromFileContentPayload struct {
 	CronExpression string
 	Recurring      bool
 	Endpoints      []portainer.EndpointID
+	EdgeGroups     []portainer.EdgeGroupID
 	FileContent    string
 }
 
 func (payload *edgeJobCreateFromFileContentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid Edge job name")
+		return errors.New("invalid Edge job name")
 	}
 
 	if !govalidator.Matches(payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) {
-		return errors.New("Invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
+		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 
 	if govalidator.IsNull(payload.CronExpression) {
-		return errors.New("Invalid cron expression")
+		return errors.New("invalid cron expression")
 	}
 
-	if payload.Endpoints == nil || len(payload.Endpoints) == 0 {
-		return errors.New("Invalid environment payload")
+	if (payload.Endpoints == nil || len(payload.Endpoints) == 0) && (payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0) {
+		return errors.New("no environments or groups have been provided")
 	}
 
 	if govalidator.IsNull(payload.FileContent) {
-		return errors.New("Invalid script file content")
+		return errors.New("invalid script file content")
 	}
 
 	return nil
@@ -86,7 +89,16 @@ func (handler *Handler) createEdgeJobFromFileContent(w http.ResponseWriter, r *h
 
 	edgeJob := handler.createEdgeJobObjectFromFileContentPayload(&payload)
 
-	err = handler.addAndPersistEdgeJob(edgeJob, []byte(payload.FileContent))
+	var endpoints []portainer.EndpointID
+	if len(edgeJob.EdgeGroups) > 0 {
+		endpoints, err = edge.GetEndpointsFromEdgeGroups(payload.EdgeGroups, handler.DataStore)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
+	}
+
+	err = handler.addAndPersistEdgeJob(edgeJob, []byte(payload.FileContent), endpoints)
+
 	if err != nil {
 		return httperror.InternalServerError("Unable to schedule Edge job", err)
 	}
@@ -99,36 +111,48 @@ type edgeJobCreateFromFilePayload struct {
 	CronExpression string
 	Recurring      bool
 	Endpoints      []portainer.EndpointID
+	EdgeGroups     []portainer.EdgeGroupID
 	File           []byte
 }
 
 func (payload *edgeJobCreateFromFilePayload) Validate(r *http.Request) error {
 	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
 	if err != nil {
-		return errors.New("Invalid Edge job name")
+		return errors.New("invalid Edge job name")
 	}
 
 	if !govalidator.Matches(name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
-		return errors.New("Invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
+		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 	payload.Name = name
 
 	cronExpression, err := request.RetrieveMultiPartFormValue(r, "CronExpression", false)
 	if err != nil {
-		return errors.New("Invalid cron expression")
+		return errors.New("invalid cron expression")
 	}
 	payload.CronExpression = cronExpression
 
 	var endpoints []portainer.EndpointID
-	err = request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, false)
+	err = request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, true)
 	if err != nil {
-		return errors.New("Invalid environments")
+		return errors.New("invalid environments")
 	}
 	payload.Endpoints = endpoints
 
+	var edgeGroups []portainer.EdgeGroupID
+	err = request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, true)
+	if err != nil {
+		return errors.New("invalid edge groups")
+	}
+	payload.EdgeGroups = edgeGroups
+
+	if (payload.Endpoints == nil || len(payload.Endpoints) == 0) && (payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0) {
+		return errors.New("no environments or groups have been provided")
+	}
+
 	file, _, err := request.RetrieveMultiPartFormFile(r, "file")
 	if err != nil {
-		return errors.New("Invalid script file. Ensure that the file is uploaded correctly")
+		return errors.New("invalid script file. Ensure that the file is uploaded correctly")
 	}
 	payload.File = file
 
@@ -144,7 +168,16 @@ func (handler *Handler) createEdgeJobFromFile(w http.ResponseWriter, r *http.Req
 
 	edgeJob := handler.createEdgeJobObjectFromFilePayload(payload)
 
-	err = handler.addAndPersistEdgeJob(edgeJob, payload.File)
+	var endpoints []portainer.EndpointID
+	if len(edgeJob.EdgeGroups) > 0 {
+		endpoints, err = edge.GetEndpointsFromEdgeGroups(payload.EdgeGroups, handler.DataStore)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
+	}
+
+	err = handler.addAndPersistEdgeJob(edgeJob, payload.File, endpoints)
+
 	if err != nil {
 		return httperror.InternalServerError("Unable to schedule Edge job", err)
 	}
@@ -155,16 +188,18 @@ func (handler *Handler) createEdgeJobFromFile(w http.ResponseWriter, r *http.Req
 func (handler *Handler) createEdgeJobObjectFromFilePayload(payload *edgeJobCreateFromFilePayload) *portainer.EdgeJob {
 	edgeJobIdentifier := portainer.EdgeJobID(handler.DataStore.EdgeJob().GetNextIdentifier())
 
-	endpoints := convertEndpointsToMetaObject(payload.Endpoints)
+	endpoints := handler.convertEndpointsToMetaObject(payload.Endpoints)
 
 	edgeJob := &portainer.EdgeJob{
-		ID:             edgeJobIdentifier,
-		Name:           payload.Name,
-		CronExpression: payload.CronExpression,
-		Recurring:      payload.Recurring,
-		Created:        time.Now().Unix(),
-		Endpoints:      endpoints,
-		Version:        1,
+		ID:                  edgeJobIdentifier,
+		Name:                payload.Name,
+		CronExpression:      payload.CronExpression,
+		Recurring:           payload.Recurring,
+		Created:             time.Now().Unix(),
+		Endpoints:           endpoints,
+		EdgeGroups:          payload.EdgeGroups,
+		Version:             1,
+		GroupLogsCollection: map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{},
 	}
 
 	return edgeJob
@@ -173,22 +208,24 @@ func (handler *Handler) createEdgeJobObjectFromFilePayload(payload *edgeJobCreat
 func (handler *Handler) createEdgeJobObjectFromFileContentPayload(payload *edgeJobCreateFromFileContentPayload) *portainer.EdgeJob {
 	edgeJobIdentifier := portainer.EdgeJobID(handler.DataStore.EdgeJob().GetNextIdentifier())
 
-	endpoints := convertEndpointsToMetaObject(payload.Endpoints)
+	endpoints := handler.convertEndpointsToMetaObject(payload.Endpoints)
 
 	edgeJob := &portainer.EdgeJob{
-		ID:             edgeJobIdentifier,
-		Name:           payload.Name,
-		CronExpression: payload.CronExpression,
-		Recurring:      payload.Recurring,
-		Created:        time.Now().Unix(),
-		Endpoints:      endpoints,
-		Version:        1,
+		ID:                  edgeJobIdentifier,
+		Name:                payload.Name,
+		CronExpression:      payload.CronExpression,
+		Recurring:           payload.Recurring,
+		Created:             time.Now().Unix(),
+		Endpoints:           endpoints,
+		EdgeGroups:          payload.EdgeGroups,
+		Version:             1,
+		GroupLogsCollection: map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{},
 	}
 
 	return edgeJob
 }
 
-func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []byte) error {
+func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []byte, endpointsFromGroups []portainer.EndpointID) error {
 	edgeCronExpression := strings.Split(edgeJob.CronExpression, " ")
 	if len(edgeCronExpression) == 6 {
 		edgeCronExpression = edgeCronExpression[1:]
@@ -206,29 +243,39 @@ func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []
 		}
 	}
 
-	if len(edgeJob.Endpoints) == 0 {
-		return errors.New("Environments are mandatory for an Edge job")
-	}
-
 	scriptPath, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), file)
 	if err != nil {
 		return err
 	}
 	edgeJob.ScriptPath = scriptPath
 
-	for endpointID := range edgeJob.Endpoints {
+	var endpointsMap map[portainer.EndpointID]portainer.EdgeJobEndpointMeta
+	if len(endpointsFromGroups) > 0 {
+		endpointsMap = handler.convertEndpointsToMetaObject(endpointsFromGroups)
+
+		for ID := range endpointsMap {
+			endpoint, err := handler.DataStore.Endpoint().Endpoint(ID)
+			if err != nil {
+				return err
+			}
+
+			if !endpointutils.IsEdgeEndpoint(endpoint) {
+				delete(endpointsMap, ID)
+			}
+		}
+
+		maps.Copy(endpointsMap, edgeJob.Endpoints)
+	} else {
+		endpointsMap = edgeJob.Endpoints
+	}
+
+	if len(endpointsMap) == 0 {
+		return errors.New("environments or edge groups are mandatory for an Edge job")
+	}
+
+	for endpointID := range endpointsMap {
 		handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
 	}
 
 	return handler.DataStore.EdgeJob().Create(edgeJob.ID, edgeJob)
 }
-
-func convertEndpointsToMetaObject(endpoints []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeJobEndpointMeta {
-	endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
-
-	for _, endpointID := range endpoints {
-		endpointsMap[endpointID] = portainer.EdgeJobEndpointMeta{}
-	}
-
-	return endpointsMap
-}

api/http/handler/edgejobs/edgejob_delete.go

@@ -8,6 +8,8 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/maps"
 )
 
 // @id EdgeJobDelete
@@ -43,6 +45,23 @@ func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *h
 
 	handler.ReverseTunnelService.RemoveEdgeJob(edgeJob.ID)
 
+	var endpointsMap map[portainer.EndpointID]portainer.EdgeJobEndpointMeta
+	if len(edgeJob.EdgeGroups) > 0 {
+		endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
+
+		endpointsMap = handler.convertEndpointsToMetaObject(endpoints)
+		maps.Copy(endpointsMap, edgeJob.Endpoints)
+	} else {
+		endpointsMap = edgeJob.Endpoints
+	}
+
+	for endpointID := range endpointsMap {
+		handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
+	}
+
 	err = handler.DataStore.EdgeJob().DeleteEdgeJob(edgeJob.ID)
 	if err != nil {
 		return httperror.InternalServerError("Unable to remove the Edge job from the database", err)

api/http/handler/edgejobs/edgejob_tasklogs_clear.go

@@ -8,6 +8,8 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/slices"
 )
 
 // @id EdgeJobTasksClear
@@ -43,11 +45,22 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
 	}
 
 	endpointID := portainer.EndpointID(taskID)
+	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
+	if err != nil {
+		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+	}
 
-	meta := edgeJob.Endpoints[endpointID]
-	meta.CollectLogs = false
-	meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
-	edgeJob.Endpoints[endpointID] = meta
+	if slices.Contains(endpointsFromGroups, endpointID) {
+		edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+			CollectLogs: false,
+			LogsStatus:  portainer.EdgeJobLogsStatusIdle,
+		}
+	} else {
+		meta := edgeJob.Endpoints[endpointID]
+		meta.CollectLogs = false
+		meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
+		edgeJob.Endpoints[endpointID] = meta
+	}
 
 	err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
 	if err != nil {

api/http/handler/edgejobs/edgejob_tasklogs_collect.go

@@ -7,6 +7,8 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/slices"
 )
 
 // @id EdgeJobTasksCollect
@@ -42,15 +44,25 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
 	}
 
 	endpointID := portainer.EndpointID(taskID)
+	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
+	if err != nil {
+		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+	}
 
-	meta := edgeJob.Endpoints[endpointID]
-	meta.CollectLogs = true
-	meta.LogsStatus = portainer.EdgeJobLogsStatusPending
-	edgeJob.Endpoints[endpointID] = meta
+	err = handler.DataStore.EdgeJob().UpdateEdgeJobFunc(edgeJob.ID, func(edgeJob *portainer.EdgeJob) {
+		if slices.Contains(endpointsFromGroups, endpointID) {
+			edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+				CollectLogs: true,
+				LogsStatus:  portainer.EdgeJobLogsStatusPending,
+			}
+		} else {
+			meta := edgeJob.Endpoints[endpointID]
+			meta.CollectLogs = true
+			meta.LogsStatus = portainer.EdgeJobLogsStatusPending
+			edgeJob.Endpoints[endpointID] = meta
+		}
+	})
 
-	handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
-
-	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
 	if err != nil {
 		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
 	}

api/http/handler/edgejobs/edgejob_tasks_list.go

@@ -8,6 +8,8 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/maps"
 )
 
 type taskContainer struct {
@@ -44,8 +46,20 @@ func (handler *Handler) edgeJobTasksList(w http.ResponseWriter, r *http.Request)
 
 	tasks := make([]taskContainer, 0)
 
-	for endpointID, meta := range edgeJob.Endpoints {
+	endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
+	if len(edgeJob.EdgeGroups) > 0 {
+		endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
 
+		endpointsMap = handler.convertEndpointsToMetaObject(endpoints)
+		maps.Copy(endpointsMap, edgeJob.GroupLogsCollection)
+	}
+
+	maps.Copy(endpointsMap, edgeJob.Endpoints)
+
+	for endpointID, meta := range endpointsMap {
 		cronTask := taskContainer{
 			ID:         fmt.Sprintf("edgejob_task_%d_%d", edgeJob.ID, endpointID),
 			EndpointID: endpointID,

api/http/handler/edgejobs/edgejob_update.go

@@ -9,6 +9,10 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/internal/maps"
+	"github.com/portainer/portainer/api/internal/slices"
 
 	"github.com/asaskevich/govalidator"
 )
@@ -18,12 +22,13 @@ type edgeJobUpdatePayload struct {
 	CronExpression *string
 	Recurring      *bool
 	Endpoints      []portainer.EndpointID
+	EdgeGroups     []portainer.EdgeGroupID
 	FileContent    *string
 }
 
 func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
 	if payload.Name != nil && !govalidator.Matches(*payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
-		return errors.New("Invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
+		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 	return nil
 }
@@ -80,16 +85,26 @@ func (handler *Handler) updateEdgeSchedule(edgeJob *portainer.EdgeJob, payload *
 		edgeJob.Name = *payload.Name
 	}
 
+	endpointsToAdd := map[portainer.EndpointID]bool{}
+	endpointsToRemove := map[portainer.EndpointID]bool{}
+
 	if payload.Endpoints != nil {
 		endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
 
+		newEndpoints := endpointutils.EndpointSet(payload.Endpoints)
+		for endpointID := range edgeJob.Endpoints {
+			if !newEndpoints[endpointID] {
+				endpointsToRemove[endpointID] = true
+			}
+		}
+
 		for _, endpointID := range payload.Endpoints {
 			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 			if err != nil {
 				return err
 			}
 
-			if endpoint.Type != portainer.EdgeAgentOnDockerEnvironment && endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
+			if !endpointutils.IsEdgeEndpoint(endpoint) {
 				continue
 			}
 
@@ -97,12 +112,73 @@ func (handler *Handler) updateEdgeSchedule(edgeJob *portainer.EdgeJob, payload *
 				endpointsMap[endpointID] = meta
 			} else {
 				endpointsMap[endpointID] = portainer.EdgeJobEndpointMeta{}
+				endpointsToAdd[endpointID] = true
 			}
 		}
 
 		edgeJob.Endpoints = endpointsMap
 	}
 
+	if len(payload.EdgeGroups) == 0 && len(edgeJob.EdgeGroups) > 0 {
+		endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
+		if err != nil {
+			return errors.New("unable to get endpoints from edge groups")
+		}
+
+		for _, endpointID := range endpoints {
+			endpointsToRemove[portainer.EndpointID(endpointID)] = true
+		}
+
+		edgeJob.EdgeGroups = nil
+	}
+
+	edgeGroupsToAdd := []portainer.EdgeGroupID{}
+	edgeGroupsToRemove := []portainer.EdgeGroupID{}
+	endpointsFromGroupsToAddMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
+
+	if len(payload.EdgeGroups) > 0 {
+		for _, edgeGroupID := range payload.EdgeGroups {
+			_, err := handler.DataStore.EdgeGroup().EdgeGroup(edgeGroupID)
+			if err != nil {
+				return err
+			}
+
+			if !slices.Contains(edgeJob.EdgeGroups, edgeGroupID) {
+				edgeGroupsToAdd = append(edgeGroupsToAdd, edgeGroupID)
+			}
+		}
+
+		endpointsFromGroupsToAdd, err := edge.GetEndpointsFromEdgeGroups(edgeGroupsToAdd, handler.DataStore)
+		if err != nil {
+			return errors.New("unable to get endpoints from edge groups")
+		}
+		endpointsFromGroupsToAddMap = handler.convertEndpointsToMetaObject(endpointsFromGroupsToAdd)
+
+		for endpointID := range endpointsFromGroupsToAddMap {
+			endpointsToAdd[endpointID] = true
+		}
+
+		newEdgeGroups := edge.EdgeGroupSet(payload.EdgeGroups)
+		for _, edgeGroupID := range edgeJob.EdgeGroups {
+			if !newEdgeGroups[edgeGroupID] {
+				edgeGroupsToRemove = append(edgeGroupsToRemove, edgeGroupID)
+			}
+		}
+
+		endpointsFromGroupsToRemove, err := edge.GetEndpointsFromEdgeGroups(edgeGroupsToRemove, handler.DataStore)
+		if err != nil {
+			return errors.New("unable to get endpoints from edge groups")
+		}
+
+		endpointsToRemoveMap := handler.convertEndpointsToMetaObject(endpointsFromGroupsToRemove)
+
+		for endpointID := range endpointsToRemoveMap {
+			endpointsToRemove[endpointID] = true
+		}
+
+		edgeJob.EdgeGroups = payload.EdgeGroups
+	}
+
 	updateVersion := false
 	if payload.CronExpression != nil && *payload.CronExpression != edgeJob.CronExpression {
 		edgeJob.CronExpression = *payload.CronExpression
@@ -133,9 +209,15 @@ func (handler *Handler) updateEdgeSchedule(edgeJob *portainer.EdgeJob, payload *
 		edgeJob.Version++
 	}
 
-	for endpointID := range edgeJob.Endpoints {
+	maps.Copy(endpointsFromGroupsToAddMap, edgeJob.Endpoints)
+
+	for endpointID := range endpointsFromGroupsToAddMap {
 		handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
 	}
 
+	for endpointID := range endpointsToRemove {
+		handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
+	}
+
 	return nil
 }

api/http/handler/edgejobs/handler.go

@@ -46,3 +46,13 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobTasksClear)))).Methods(http.MethodDelete)
 	return h
 }
+
+func (handler *Handler) convertEndpointsToMetaObject(endpoints []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeJobEndpointMeta {
+	endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
+
+	for _, endpointID := range endpoints {
+		endpointsMap[endpointID] = portainer.EdgeJobEndpointMeta{}
+	}
+
+	return endpointsMap
+}

api/http/handler/edgestacks/edgestack_create.go

@@ -3,22 +3,26 @@ package edgestacks
 import (
 	"fmt"
 	"net/http"
-	"strconv"
-	"strings"
-	"time"
 
+	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
-	"github.com/portainer/portainer/api/internal/edge"
-
-	"github.com/asaskevich/govalidator"
-	"github.com/pkg/errors"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/http/security"
 )
 
+type InvalidPayloadError struct {
+	msg string
+}
+
+func (e *InvalidPayloadError) Error() string {
+	return e.msg
+}
+
 // @id EdgeStackCreate
 // @summary Create an EdgeStack
 // @description **Access policy**: administrator
@@ -39,23 +43,36 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 	if err != nil {
 		return httperror.BadRequest("Invalid query parameter: method", err)
 	}
+	dryrun, _ := request.RetrieveBooleanQueryParameter(r, "dryrun", true)
 
-	edgeStack, err := handler.createSwarmStack(method, r)
+	tokenData, err := security.RetrieveTokenData(r)
 	if err != nil {
-		return httperror.InternalServerError("Unable to create Edge stack", err)
+		return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
+	}
+
+	edgeStack, err := handler.createSwarmStack(method, dryrun, tokenData.ID, r)
+	if err != nil {
+		var payloadError *InvalidPayloadError
+		switch {
+		case errors.As(err, &payloadError):
+			return httperror.BadRequest("Invalid payload", err)
+		default:
+			return httperror.InternalServerError("Unable to create Edge stack", err)
+		}
 	}
 
 	return response.JSON(w, edgeStack)
 }
 
-func (handler *Handler) createSwarmStack(method string, r *http.Request) (*portainer.EdgeStack, error) {
+func (handler *Handler) createSwarmStack(method string, dryrun bool, userID portainer.UserID, r *http.Request) (*portainer.EdgeStack, error) {
+
 	switch method {
 	case "string":
-		return handler.createSwarmStackFromFileContent(r)
+		return handler.createSwarmStackFromFileContent(r, dryrun)
 	case "repository":
-		return handler.createSwarmStackFromGitRepository(r)
+		return handler.createSwarmStackFromGitRepository(r, dryrun, userID)
 	case "file":
-		return handler.createSwarmStackFromFileUpload(r)
+		return handler.createSwarmStackFromFileUpload(r, dryrun)
 	}
 	return nil, errors.New("Invalid value for query parameter: method. Value must be one of: string, repository or file")
 }
@@ -68,104 +85,93 @@ type swarmStackFromFileContentPayload struct {
 	// List of identifiers of EdgeGroups
 	EdgeGroups []portainer.EdgeGroupID `example:"1"`
 	// Deployment type to deploy this stack
-	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
+	// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
 	// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
-	// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
-	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
+	// kubernetes deploy type is enabled only for kubernetes environments(endpoints)
+	// nomad deploy type is enabled only for nomad environments(endpoints)
+	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
+	// List of Registries to use for this stack
+	Registries []portainer.RegistryID
+	// Uses the manifest's namespaces instead of the default one
+	UseManifestNamespaces bool
 }
 
 func (payload *swarmStackFromFileContentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid stack name")
+		return &InvalidPayloadError{msg: "Invalid stack name"}
 	}
 	if govalidator.IsNull(payload.StackFileContent) {
-		return errors.New("Invalid stack file content")
+		return &InvalidPayloadError{msg: "Invalid stack file content"}
 	}
 	if payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0 {
-		return errors.New("Edge Groups are mandatory for an Edge stack")
+		return &InvalidPayloadError{msg: "Edge Groups are mandatory for an Edge stack"}
 	}
 	return nil
 }
 
-func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*portainer.EdgeStack, error) {
+func (handler *Handler) createSwarmStackFromFileContent(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) {
 	var payload swarmStackFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handler.validateUniqueName(payload.Name)
+	stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to create Edge stack object")
 	}
 
-	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
-	stack := &portainer.EdgeStack{
-		ID:             portainer.EdgeStackID(stackID),
-		Name:           payload.Name,
-		DeploymentType: payload.DeploymentType,
-		CreationDate:   time.Now().Unix(),
-		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		Version:        1,
+	if dryrun {
+		return stack, nil
 	}
 
-	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
-	if err != nil {
-		return nil, fmt.Errorf("unable to find environment relations in database: %w", err)
-	}
+	return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
+		return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, []byte(payload.StackFileContent))
+	})
 
-	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
-	if err != nil {
-		return nil, fmt.Errorf("unable to persist environment relation in database: %w", err)
-	}
+}
 
-	stackFolder := strconv.Itoa(int(stack.ID))
-	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
-		stack.EntryPoint = filesystem.ComposeFileDefaultName
+func (handler *Handler) storeFileContent(stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) {
+	if deploymentType == portainer.EdgeStackDeploymentCompose {
+		composePath = filesystem.ComposeFileDefaultName
 
-		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, composePath, fileContent)
 		if err != nil {
-			return nil, err
+			return "", "", "", err
 		}
-		stack.ProjectPath = projectPath
 
-		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
+		manifestPath, err = handler.convertAndStoreKubeManifestIfNeeded(stackFolder, projectPath, composePath, relatedEndpointIds)
 		if err != nil {
-			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
+			return "", "", "", fmt.Errorf("Failed creating and storing kube manifest: %w", err)
 		}
 
-	} else {
-		hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
-		if err != nil {
-			return nil, fmt.Errorf("unable to check for existence of docker endpoint: %w", err)
-		}
+		return composePath, manifestPath, projectPath, nil
 
-		if hasDockerEndpoint {
-			return nil, fmt.Errorf("edge stack with docker endpoint cannot be deployed with kubernetes config")
-		}
-
-		stack.ManifestPath = filesystem.ManifestFileDefaultName
-
-		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
-		if err != nil {
-			return nil, err
-		}
-
-		stack.ProjectPath = projectPath
 	}
 
-	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
+	hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
+		return "", "", "", fmt.Errorf("unable to check for existence of docker environment: %w", err)
 	}
 
-	err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
-	if err != nil {
-		return nil, err
+	if hasDockerEndpoint {
+		return "", "", "", fmt.Errorf("edge stack with docker environment cannot be deployed with kubernetes or nomad config")
 	}
 
-	return stack, nil
+	if deploymentType == portainer.EdgeStackDeploymentKubernetes {
+
+		manifestPath = filesystem.ManifestFileDefaultName
+
+		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, manifestPath, fileContent)
+		if err != nil {
+			return "", "", "", err
+		}
+
+		return "", manifestPath, projectPath, nil
+
+	}
+
+	return "", "", "", fmt.Errorf("invalid deployment type: %d", deploymentType)
 }
 
 type swarmStackFromGitRepositoryPayload struct {
@@ -186,239 +192,180 @@ type swarmStackFromGitRepositoryPayload struct {
 	// List of identifiers of EdgeGroups
 	EdgeGroups []portainer.EdgeGroupID `example:"1"`
 	// Deployment type to deploy this stack
-	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
+	// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
 	// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
-	// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
-	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
+	// kubernetes deploy type is enabled only for kubernetes environments(endpoints)
+	// nomad deploy type is enabled only for nomad environments(endpoints)
+	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
+	// List of Registries to use for this stack
+	Registries []portainer.RegistryID
+	// Uses the manifest's namespaces instead of the default one
+	UseManifestNamespaces bool
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid stack name")
+		return &InvalidPayloadError{msg: "Invalid stack name"}
 	}
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
-		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
+		return &InvalidPayloadError{msg: "Invalid repository URL. Must correspond to a valid URL format"}
 	}
 	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+		return &InvalidPayloadError{msg: "Invalid repository credentials. Password must be specified when authentication is enabled"}
 	}
 	if govalidator.IsNull(payload.FilePathInRepository) {
-		payload.FilePathInRepository = filesystem.ComposeFileDefaultName
+		switch payload.DeploymentType {
+		case portainer.EdgeStackDeploymentCompose:
+			payload.FilePathInRepository = filesystem.ComposeFileDefaultName
+		case portainer.EdgeStackDeploymentKubernetes:
+			payload.FilePathInRepository = filesystem.ManifestFileDefaultName
+		}
 	}
 	if payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0 {
-		return errors.New("Edge Groups are mandatory for an Edge stack")
+		return &InvalidPayloadError{msg: "Edge Groups are mandatory for an Edge stack"}
 	}
 	return nil
 }
 
-func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*portainer.EdgeStack, error) {
+func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request, dryrun bool, userID portainer.UserID) (*portainer.EdgeStack, error) {
 	var payload swarmStackFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handler.validateUniqueName(payload.Name)
+	stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to create edge stack object")
 	}
 
-	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
-	stack := &portainer.EdgeStack{
-		ID:             portainer.EdgeStackID(stackID),
-		Name:           payload.Name,
-		CreationDate:   time.Now().Unix(),
-		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		DeploymentType: payload.DeploymentType,
-		Version:        1,
+	if dryrun {
+		return stack, nil
 	}
 
-	projectPath := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(stack.ID)))
-	stack.ProjectPath = projectPath
-
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
+	repoConfig := gittypes.RepoConfig{
+		URL:            payload.RepositoryURL,
+		ReferenceName:  payload.RepositoryReferenceName,
+		ConfigFilePath: payload.FilePathInRepository,
 	}
 
-	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
-	if err != nil {
-		return nil, fmt.Errorf("failed fetching relations config: %w", err)
-	}
-
-	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
-	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
-	}
-
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		return nil, err
-	}
-
-	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
-		stack.EntryPoint = payload.FilePathInRepository
-
-		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
-		if err != nil {
-			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
+	if payload.RepositoryAuthentication {
+		repoConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
 		}
-	} else {
-		stack.ManifestPath = payload.FilePathInRepository
 	}
 
-	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to update environment relations: %w", err)
-	}
-
-	err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
-	if err != nil {
-		return nil, err
-	}
-
-	return stack, nil
+	return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
+		return handler.storeManifestFromGitRepository(stackFolder, relatedEndpointIds, payload.DeploymentType, userID, repoConfig)
+	})
 }
 
 type swarmStackFromFileUploadPayload struct {
 	Name             string
 	StackFileContent []byte
 	EdgeGroups       []portainer.EdgeGroupID
-	DeploymentType   portainer.EdgeStackDeploymentType
+	// Deployment type to deploy this stack
+	// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
+	// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
+	// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
+	// nomad deploytype is enabled only for nomad environments(endpoints)
+	DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
+	Registries     []portainer.RegistryID
+	// Uses the manifest's namespaces instead of the default one
+	UseManifestNamespaces bool
 }
 
 func (payload *swarmStackFromFileUploadPayload) Validate(r *http.Request) error {
 	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
 	if err != nil {
-		return errors.New("Invalid stack name")
+		return &InvalidPayloadError{msg: "Invalid stack name"}
 	}
 	payload.Name = name
 
 	composeFileContent, _, err := request.RetrieveMultiPartFormFile(r, "file")
 	if err != nil {
-		return errors.New("Invalid Compose file. Ensure that the Compose file is uploaded correctly")
+		return &InvalidPayloadError{msg: "Invalid Compose file. Ensure that the Compose file is uploaded correctly"}
 	}
 	payload.StackFileContent = composeFileContent
 
 	var edgeGroups []portainer.EdgeGroupID
 	err = request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, false)
 	if err != nil || len(edgeGroups) == 0 {
-		return errors.New("Edge Groups are mandatory for an Edge stack")
+		return &InvalidPayloadError{msg: "Edge Groups are mandatory for an Edge stack"}
 	}
 	payload.EdgeGroups = edgeGroups
 
 	deploymentType, err := request.RetrieveNumericMultiPartFormValue(r, "DeploymentType", true)
 	if err != nil {
-		return errors.New("Invalid deployment type")
+		return &InvalidPayloadError{msg: "Invalid deployment type"}
 	}
 	payload.DeploymentType = portainer.EdgeStackDeploymentType(deploymentType)
 
+	var registries []portainer.RegistryID
+	request.RetrieveMultiPartFormJSONValue(r, "Registries", &registries, false)
+	if err != nil {
+		return errors.New("Invalid registry type")
+	}
+	payload.Registries = registries
+
+	useManifestNamespaces, _ := request.RetrieveBooleanMultiPartFormValue(r, "UseManifestNamespaces", true)
+	payload.UseManifestNamespaces = useManifestNamespaces
+
 	return nil
 }
 
-func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portainer.EdgeStack, error) {
+func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) {
 	payload := &swarmStackFromFileUploadPayload{}
 	err := payload.Validate(r)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handler.validateUniqueName(payload.Name)
+	stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to create edge stack object")
 	}
 
-	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
-	stack := &portainer.EdgeStack{
-		ID:             portainer.EdgeStackID(stackID),
-		Name:           payload.Name,
-		DeploymentType: payload.DeploymentType,
-		CreationDate:   time.Now().Unix(),
-		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
-		Version:        1,
+	if dryrun {
+		return stack, nil
 	}
 
-	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
-	if err != nil {
-		return nil, fmt.Errorf("failed fetching relations config: %w", err)
-	}
-
-	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
-	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
-	}
-
-	stackFolder := strconv.Itoa(int(stack.ID))
-	if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
-		stack.EntryPoint = filesystem.ComposeFileDefaultName
-
-		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
-		if err != nil {
-			return nil, err
-		}
-		stack.ProjectPath = projectPath
-
-		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
-		if err != nil {
-			return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
-		}
-
-	} else {
-		stack.ManifestPath = filesystem.ManifestFileDefaultName
-
-		projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
-		if err != nil {
-			return nil, err
-		}
-		stack.ProjectPath = projectPath
-	}
-
-	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to update environment relations: %w", err)
-	}
-
-	err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
-	if err != nil {
-		return nil, err
-	}
-
-	return stack, nil
+	return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
+		return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, payload.StackFileContent)
+	})
 }
 
-func (handler *Handler) validateUniqueName(name string) error {
-	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
+func (handler *Handler) storeManifestFromGitRepository(stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) {
+	projectPath = handler.FileService.GetEdgeStackProjectPath(stackFolder)
+	repositoryUsername := ""
+	repositoryPassword := ""
+	if repositoryConfig.Authentication != nil {
+		if repositoryConfig.Authentication.Password != "" {
+			repositoryUsername = repositoryConfig.Authentication.Username
+			repositoryPassword = repositoryConfig.Authentication.Password
+		}
+	}
+
+	err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
-		return err
+		return "", "", "", err
 	}
 
-	for _, stack := range edgeStacks {
-		if strings.EqualFold(stack.Name, name) {
-			return errors.New("Edge stack name must be unique")
-		}
-	}
-	return nil
-}
+	if deploymentType == portainer.EdgeStackDeploymentCompose {
+		composePath := repositoryConfig.ConfigFilePath
 
-// updateEndpointRelations adds a relation between the Edge Stack to the related environments(endpoints)
-func updateEndpointRelations(endpointRelationService dataservices.EndpointRelationService, edgeStackID portainer.EdgeStackID, relatedEndpointIds []portainer.EndpointID) error {
-	for _, endpointID := range relatedEndpointIds {
-		relation, err := endpointRelationService.EndpointRelation(endpointID)
+		manifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, projectPath, composePath, relatedEndpointIds)
 		if err != nil {
-			return fmt.Errorf("unable to find environment relation in database: %w", err)
+			return "", "", "", fmt.Errorf("Failed creating and storing kube manifest: %w", err)
 		}
 
-		relation.EdgeStacks[edgeStackID] = true
-
-		err = endpointRelationService.UpdateEndpointRelation(endpointID, relation)
-		if err != nil {
-			return fmt.Errorf("unable to persist environment relation in database: %w", err)
-		}
+		return composePath, manifestPath, projectPath, nil
 	}
 
-	return nil
+	if deploymentType == portainer.EdgeStackDeploymentKubernetes {
+		return "", repositoryConfig.ConfigFilePath, projectPath, nil
+	}
+
+	return "", "", "", fmt.Errorf("unknown deployment type: %d", deploymentType)
 }

api/http/handler/edgestacks/edgestack_delete.go

@@ -7,7 +7,6 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/edge"
 )
 
 // @id EdgeStackDelete
@@ -35,33 +34,9 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", err)
 	}
 
-	err = handler.DataStore.EdgeStack().DeleteEdgeStack(portainer.EdgeStackID(edgeStackID))
+	err = handler.edgeStacksService.DeleteEdgeStack(edgeStack.ID, edgeStack.EdgeGroups)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove the edge stack from the database", err)
-	}
-
-	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
-	if err != nil {
-		return httperror.InternalServerError("Unable to find environment relations in database", err)
-	}
-
-	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
-	}
-
-	for _, endpointID := range relatedEndpointIds {
-		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
-		if err != nil {
-			return httperror.InternalServerError("Unable to find environment relation in database", err)
-		}
-
-		delete(relation.EdgeStacks, edgeStack.ID)
-
-		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
-		if err != nil {
-			return httperror.InternalServerError("Unable to persist environment relation in database", err)
-		}
+		return httperror.InternalServerError("Unable to delete edge stack", err)
 	}
 
 	return response.Empty(w)

api/http/handler/edgestacks/edgestack_status_update.go

@@ -4,29 +4,33 @@ import (
 	"errors"
 	"net/http"
 
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
 )
 
 type updateStatusPayload struct {
 	Error      string
 	Status     *portainer.EdgeStackStatusType
-	EndpointID *portainer.EndpointID
+	EndpointID portainer.EndpointID
 }
 
 func (payload *updateStatusPayload) Validate(r *http.Request) error {
 	if payload.Status == nil {
 		return errors.New("Invalid status")
 	}
-	if payload.EndpointID == nil {
+
+	if payload.EndpointID == 0 {
 		return errors.New("Invalid EnvironmentID")
 	}
-	if *payload.Status == portainer.StatusError && govalidator.IsNull(payload.Error) {
+
+	if *payload.Status == portainer.EdgeStackStatusError && govalidator.IsNull(payload.Error) {
 		return errors.New("Error message is mandatory when status is error")
 	}
+
 	return nil
 }
 
@@ -49,20 +53,13 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return httperror.BadRequest("Invalid stack identifier route variable", err)
 	}
 
-	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find a stack with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find a stack with the specified identifier inside the database", err)
-	}
-
 	var payload updateStatusPayload
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(*payload.EndpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(payload.EndpointID)
 	if handler.DataStore.IsErrObjectNotFound(err) {
 		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
 	} else if err != nil {
@@ -74,17 +71,38 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return httperror.Forbidden("Permission denied to access environment", err)
 	}
 
-	stack.Status[*payload.EndpointID] = portainer.EdgeStackStatus{
-		Type:       *payload.Status,
-		Error:      payload.Error,
-		EndpointID: *payload.EndpointID,
-	}
+	var stack portainer.EdgeStack
 
-	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
-	if err != nil {
+	err = handler.DataStore.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) {
+		details := edgeStack.Status[payload.EndpointID].Details
+		details.Pending = false
+
+		switch *payload.Status {
+		case portainer.EdgeStackStatusOk:
+			details.Ok = true
+		case portainer.EdgeStackStatusError:
+			details.Error = true
+		case portainer.EdgeStackStatusAcknowledged:
+			details.Acknowledged = true
+		case portainer.EdgeStackStatusRemove:
+			details.Remove = true
+		case portainer.EdgeStackStatusImagesPulled:
+			details.ImagesPulled = true
+		}
+
+		edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{
+			Details:    details,
+			Error:      payload.Error,
+			EndpointID: payload.EndpointID,
+		}
+
+		stack = *edgeStack
+	})
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return httperror.NotFound("Unable to find a stack with the specified identifier inside the database", err)
+	} else if err != nil {
 		return httperror.InternalServerError("Unable to persist the stack changes inside the database", err)
 	}
 
 	return response.JSON(w, stack)
-
 }

api/http/handler/edgestacks/edgestack_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/edge/edgestacks"
 	"github.com/portainer/portainer/api/jwt"
 
 	"github.com/pkg/errors"
@@ -69,9 +70,12 @@ func setupHandler(t *testing.T) (*Handler, string, func()) {
 		t.Fatal(err)
 	}
 
+	edgeStacksService := edgestacks.NewService(store)
+
 	handler := NewHandler(
 		security.NewRequestBouncer(store, jwtService, apiKeyService),
 		store,
+		edgeStacksService,
 	)
 
 	tmpDir := t.TempDir()
@@ -99,10 +103,9 @@ func setupHandler(t *testing.T) (*Handler, string, func()) {
 	return handler, rawAPIKey, storeTeardown
 }
 
-func createEndpoint(t *testing.T, store dataservices.DataStore) portainer.Endpoint {
+func createEndpointWithId(t *testing.T, store dataservices.DataStore, endpointID portainer.EndpointID) portainer.Endpoint {
 	t.Helper()
 
-	endpointID := portainer.EndpointID(5)
 	endpoint := portainer.Endpoint{
 		ID:              endpointID,
 		Name:            "test-endpoint-" + strconv.Itoa(int(endpointID)),
@@ -120,6 +123,10 @@ func createEndpoint(t *testing.T, store dataservices.DataStore) portainer.Endpoi
 	return endpoint
 }
 
+func createEndpoint(t *testing.T, store dataservices.DataStore) portainer.Endpoint {
+	return createEndpointWithId(t, store, 5)
+}
+
 func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID portainer.EndpointID) portainer.EdgeStack {
 	t.Helper()
 
@@ -142,7 +149,7 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
 		ID:   edgeStackID,
 		Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)),
 		Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
-			endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpointID},
+			endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpointID},
 		},
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{edgeGroup.ID},
@@ -199,7 +206,7 @@ func TestInspectInvalidEdgeID(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -259,7 +266,7 @@ func TestCreateAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	data := portainer.EdgeStack{}
@@ -279,7 +286,7 @@ func TestCreateAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	data = portainer.EdgeStack{}
@@ -289,7 +296,7 @@ func TestCreateAndInspect(t *testing.T) {
 	}
 
 	if payload.Name != data.Name {
-		t.Fatalf(fmt.Sprintf("expected EdgeStack Name %s, found %s", payload.Name, data.Name))
+		t.Fatalf("expected EdgeStack Name %s, found %s", payload.Name, data.Name)
 	}
 }
 
@@ -322,19 +329,19 @@ func TestCreateWithInvalidPayload(t *testing.T) {
 			Name:               "Empty swarmStackFromFileContentPayload with string method",
 			Payload:            swarmStackFromFileContentPayload{},
 			QueryString:        "method=string",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: 400,
 		},
 		{
 			Name:               "Empty swarmStackFromFileContentPayload with repository method",
 			Payload:            swarmStackFromFileContentPayload{},
 			QueryString:        "method=repository",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: 400,
 		},
 		{
 			Name:               "Empty swarmStackFromFileContentPayload with file method",
 			Payload:            swarmStackFromFileContentPayload{},
 			QueryString:        "method=file",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: 400,
 		},
 		{
 			Name: "Duplicated EdgeStack Name",
@@ -356,7 +363,7 @@ func TestCreateWithInvalidPayload(t *testing.T) {
 				DeploymentType:   edgeStack.DeploymentType,
 			},
 			QueryString:        "method=string",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: 400,
 		},
 		{
 			Name: "EdgeStackDeploymentKubernetes with Docker endpoint",
@@ -378,7 +385,7 @@ func TestCreateWithInvalidPayload(t *testing.T) {
 				DeploymentType:   portainer.EdgeStackDeploymentCompose,
 			},
 			QueryString:        "method=string",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: 400,
 		},
 		{
 			Name: "Clone Git respository error",
@@ -417,7 +424,7 @@ func TestCreateWithInvalidPayload(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -443,7 +450,7 @@ func TestDeleteAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	data := portainer.EdgeStack{}
@@ -453,7 +460,7 @@ func TestDeleteAndInspect(t *testing.T) {
 	}
 
 	if data.ID != edgeStack.ID {
-		t.Fatalf(fmt.Sprintf("expected EdgeStackID %d, found %d", int(edgeStack.ID), data.ID))
+		t.Fatalf("expected EdgeStackID %d, found %d", int(edgeStack.ID), data.ID)
 	}
 
 	// Delete
@@ -467,7 +474,7 @@ func TestDeleteAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusNoContent {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusNoContent, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusNoContent, rec.Code)
 	}
 
 	// Inspect
@@ -481,7 +488,7 @@ func TestDeleteAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusNotFound {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusNotFound, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusNotFound, rec.Code)
 	}
 }
 
@@ -510,7 +517,7 @@ func TestDeleteInvalidEdgeStack(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -526,14 +533,7 @@ func TestUpdateAndInspect(t *testing.T) {
 
 	// Update edge stack: create new Endpoint, EndpointRelation and EdgeGroup
 	endpointID := portainer.EndpointID(6)
-	newEndpoint := portainer.Endpoint{
-		ID:              endpointID,
-		Name:            "test-endpoint-" + strconv.Itoa(int(endpointID)),
-		Type:            portainer.EdgeAgentOnDockerEnvironment,
-		URL:             "https://portainer.io:9443",
-		EdgeID:          "edge-id",
-		LastCheckInDate: time.Now().Unix(),
-	}
+	newEndpoint := createEndpointWithId(t, handler.DataStore, endpointID)
 
 	err := handler.DataStore.Endpoint().Create(&newEndpoint)
 	if err != nil {
@@ -590,7 +590,7 @@ func TestUpdateAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	// Get updated edge stack
@@ -604,7 +604,7 @@ func TestUpdateAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	data := portainer.EdgeStack{}
@@ -614,11 +614,11 @@ func TestUpdateAndInspect(t *testing.T) {
 	}
 
 	if data.Version != *payload.Version {
-		t.Fatalf(fmt.Sprintf("expected EdgeStackID %d, found %d", edgeStack.Version, data.Version))
+		t.Fatalf("expected EdgeStackID %d, found %d", edgeStack.Version, data.Version)
 	}
 
 	if data.DeploymentType != payload.DeploymentType {
-		t.Fatalf(fmt.Sprintf("expected DeploymentType %d, found %d", edgeStack.DeploymentType, data.DeploymentType))
+		t.Fatalf("expected DeploymentType %d, found %d", edgeStack.DeploymentType, data.DeploymentType)
 	}
 
 	if !reflect.DeepEqual(data.EdgeGroups, payload.EdgeGroups) {
@@ -701,7 +701,7 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -760,7 +760,7 @@ func TestUpdateWithInvalidPayload(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -775,11 +775,11 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
 
 	// Update edge stack status
-	newStatus := portainer.StatusError
+	newStatus := portainer.EdgeStackStatusError
 	payload := updateStatusPayload{
 		Error:      "test-error",
 		Status:     &newStatus,
-		EndpointID: &endpoint.ID,
+		EndpointID: endpoint.ID,
 	}
 
 	jsonPayload, err := json.Marshal(payload)
@@ -798,7 +798,7 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	// Get updated edge stack
@@ -812,7 +812,7 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 
 	data := portainer.EdgeStack{}
@@ -821,16 +821,16 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 		t.Fatal("error decoding response:", err)
 	}
 
-	if data.Status[endpoint.ID].Type != *payload.Status {
-		t.Fatalf(fmt.Sprintf("expected EdgeStackStatusType %d, found %d", payload.Status, data.Status[endpoint.ID].Type))
+	if !data.Status[endpoint.ID].Details.Error {
+		t.Fatalf("expected EdgeStackStatusType %d, found %t", payload.Status, data.Status[endpoint.ID].Details.Error)
 	}
 
 	if data.Status[endpoint.ID].Error != payload.Error {
-		t.Fatalf(fmt.Sprintf("expected EdgeStackStatusError %s, found %s", payload.Error, data.Status[endpoint.ID].Error))
+		t.Fatalf("expected EdgeStackStatusError %s, found %s", payload.Error, data.Status[endpoint.ID].Error)
 	}
 
-	if data.Status[endpoint.ID].EndpointID != *payload.EndpointID {
-		t.Fatalf(fmt.Sprintf("expected EndpointID %d, found %d", payload.EndpointID, data.Status[endpoint.ID].EndpointID))
+	if data.Status[endpoint.ID].EndpointID != payload.EndpointID {
+		t.Fatalf("expected EndpointID %d, found %d", payload.EndpointID, data.Status[endpoint.ID].EndpointID)
 	}
 }
 func TestUpdateStatusWithInvalidPayload(t *testing.T) {
@@ -841,8 +841,8 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 	edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
 
 	// Update edge stack status
-	statusError := portainer.StatusError
-	statusOk := portainer.StatusOk
+	statusError := portainer.EdgeStackStatusError
+	statusOk := portainer.EdgeStackStatusOk
 	cases := []struct {
 		Name                 string
 		Payload              updateStatusPayload
@@ -854,7 +854,7 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 			updateStatusPayload{
 				Error:      "test-error",
 				Status:     nil,
-				EndpointID: &endpoint.ID,
+				EndpointID: endpoint.ID,
 			},
 			"Invalid status",
 			400,
@@ -864,17 +864,17 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 			updateStatusPayload{
 				Error:      "",
 				Status:     &statusError,
-				EndpointID: &endpoint.ID,
+				EndpointID: endpoint.ID,
 			},
 			"Error message is mandatory when status is error",
 			400,
 		},
 		{
-			"Update with nil EndpointID",
+			"Update with missing EndpointID",
 			updateStatusPayload{
 				Error:      "",
 				Status:     &statusOk,
-				EndpointID: nil,
+				EndpointID: 0,
 			},
 			"Invalid EnvironmentID",
 			400,
@@ -899,7 +899,7 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 			handler.ServeHTTP(rec, req)
 
 			if rec.Code != tc.ExpectedStatusCode {
-				t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code))
+				t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
 			}
 		})
 	}
@@ -923,6 +923,6 @@ func TestDeleteStatus(t *testing.T) {
 	handler.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		t.Fatalf(fmt.Sprintf("expected a %d response, found: %d", http.StatusOK, rec.Code))
+		t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
 	}
 }

api/http/handler/edgestacks/edgestack_update.go

@@ -19,6 +19,8 @@ type updateEdgeStackPayload struct {
 	Version          *int
 	EdgeGroups       []portainer.EdgeGroupID
 	DeploymentType   portainer.EdgeStackDeploymentType
+	// Uses the manifest's namespaces instead of the default one
+	UseManifestNamespaces bool
 }
 
 func (payload *updateEdgeStackPayload) Validate(r *http.Request) error {
@@ -65,18 +67,20 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
+	relationConfig, err := edge.FetchEndpointRelationsConfig(handler.DataStore)
 	if err != nil {
 		return httperror.InternalServerError("Unable to retrieve environments relations config from database", err)
 	}
 
-	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
+	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
 	if err != nil {
 		return httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
 	}
 
+	endpointsToAdd := map[portainer.EndpointID]bool{}
+
 	if payload.EdgeGroups != nil {
-		newRelated, err := edge.EdgeStackRelatedEndpoints(payload.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
+		newRelated, err := edge.EdgeStackRelatedEndpoints(payload.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
 		if err != nil {
 			return httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
 		}
@@ -105,7 +109,6 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 			}
 		}
 
-		endpointsToAdd := map[portainer.EndpointID]bool{}
 		for endpointID := range newRelatedSet {
 			if !oldRelatedSet[endpointID] {
 				endpointsToAdd[endpointID] = true
@@ -143,26 +146,32 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 	}
 
 	stackFolder := strconv.Itoa(int(stack.ID))
+
 	if payload.DeploymentType == portainer.EdgeStackDeploymentCompose {
 		if stack.EntryPoint == "" {
 			stack.EntryPoint = filesystem.ComposeFileDefaultName
 		}
 
-		_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+		_, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
 		if err != nil {
 			return httperror.InternalServerError("Unable to persist updated Compose file on disk", err)
 		}
 
-		err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
+		manifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, stack.ProjectPath, stack.EntryPoint, relatedEndpointIds)
 		if err != nil {
 			return httperror.InternalServerError("Unable to convert and persist updated Kubernetes manifest file on disk", err)
 		}
 
-	} else {
+		stack.ManifestPath = manifestPath
+	}
+
+	if payload.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
 		if stack.ManifestPath == "" {
 			stack.ManifestPath = filesystem.ManifestFileDefaultName
 		}
 
+		stack.UseManifestNamespaces = payload.UseManifestNamespaces
+
 		hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
 		if err != nil {
 			return httperror.InternalServerError("Unable to check for existence of docker environment", err)
@@ -174,15 +183,19 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 
 		_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
 		if err != nil {
-			return httperror.InternalServerError("Unable to persist updated Compose file on disk", err)
+			return httperror.InternalServerError("Unable to persist updated Kubernetes manifest file on disk", err)
 		}
 	}
 
-	if payload.Version != nil && *payload.Version != stack.Version {
+	versionUpdated := payload.Version != nil && *payload.Version != stack.Version
+	if versionUpdated {
 		stack.Version = *payload.Version
 		stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
 	}
 
+	stack.NumDeployments = len(relatedEndpointIds)
+	stack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
+
 	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
 	if err != nil {
 		return httperror.InternalServerError("Unable to persist the stack changes inside the database", err)

api/http/handler/edgestacks/endpoints.go

@@ -30,32 +30,3 @@ func hasEndpointPredicate(endpointService dataservices.EndpointService, endpoint
 
 	return false, nil
 }
-
-type endpointRelationsConfig struct {
-	endpoints      []portainer.Endpoint
-	endpointGroups []portainer.EndpointGroup
-	edgeGroups     []portainer.EdgeGroup
-}
-
-func fetchEndpointRelationsConfig(dataStore dataservices.DataStore) (*endpointRelationsConfig, error) {
-	endpoints, err := dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve environments from database: %w", err)
-	}
-
-	endpointGroups, err := dataStore.EndpointGroup().EndpointGroups()
-	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve environment groups from database: %w", err)
-	}
-
-	edgeGroups, err := dataStore.EdgeGroup().EdgeGroups()
-	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve edge groups from database: %w", err)
-	}
-
-	return &endpointRelationsConfig{
-		endpoints:      endpoints,
-		endpointGroups: endpointGroups,
-		edgeGroups:     edgeGroups,
-	}, nil
-}

api/http/handler/edgestacks/handler.go

@@ -3,7 +3,6 @@ package edgestacks
 import (
 	"fmt"
 	"net/http"
-	"strconv"
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
@@ -12,6 +11,7 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
+	edgestackservice "github.com/portainer/portainer/api/internal/edge/edgestacks"
 )
 
 // Handler is the HTTP handler used to handle environment(endpoint) group operations.
@@ -21,16 +21,19 @@ type Handler struct {
 	DataStore          dataservices.DataStore
 	FileService        portainer.FileService
 	GitService         portainer.GitService
+	edgeStacksService  *edgestackservice.Service
 	KubernetesDeployer portainer.KubernetesDeployer
 }
 
 // NewHandler creates a handler to manage environment(endpoint) group operations.
-func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore) *Handler {
+func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service) *Handler {
 	h := &Handler{
-		Router:         mux.NewRouter(),
-		requestBouncer: bouncer,
-		DataStore:      dataStore,
+		Router:            mux.NewRouter(),
+		requestBouncer:    bouncer,
+		DataStore:         dataStore,
+		edgeStacksService: edgeStacksService,
 	}
+
 	h.Handle("/edge_stacks",
 		bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackCreate)))).Methods(http.MethodPost)
 	h.Handle("/edge_stacks",
@@ -54,33 +57,31 @@ func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataSto
 	return h
 }
 
-func (handler *Handler) convertAndStoreKubeManifestIfNeeded(edgeStack *portainer.EdgeStack, relatedEndpointIds []portainer.EndpointID) error {
+func (handler *Handler) convertAndStoreKubeManifestIfNeeded(stackFolder string, projectPath, composePath string, relatedEndpointIds []portainer.EndpointID) (manifestPath string, err error) {
 	hasKubeEndpoint, err := hasKubeEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
 	if err != nil {
-		return fmt.Errorf("unable to check if edge stack has kube environments: %w", err)
+		return "", fmt.Errorf("unable to check if edge stack has kube environments: %w", err)
 	}
 
 	if !hasKubeEndpoint {
-		return nil
+		return "", nil
 	}
 
-	composeConfig, err := handler.FileService.GetFileContent(edgeStack.ProjectPath, edgeStack.EntryPoint)
+	composeConfig, err := handler.FileService.GetFileContent(projectPath, composePath)
 	if err != nil {
-		return fmt.Errorf("unable to retrieve Compose file from disk: %w", err)
+		return "", fmt.Errorf("unable to retrieve Compose file from disk: %w", err)
 	}
 
 	kompose, err := handler.KubernetesDeployer.ConvertCompose(composeConfig)
 	if err != nil {
-		return fmt.Errorf("failed converting compose file to kubernetes manifest: %w", err)
+		return "", fmt.Errorf("failed converting compose file to kubernetes manifest: %w", err)
 	}
 
 	komposeFileName := filesystem.ManifestFileDefaultName
-	_, err = handler.FileService.StoreEdgeStackFileFromBytes(strconv.Itoa(int(edgeStack.ID)), komposeFileName, kompose)
+	_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, komposeFileName, kompose)
 	if err != nil {
-		return fmt.Errorf("failed to store kube manifest file: %w", err)
+		return "", fmt.Errorf("failed to store kube manifest file: %w", err)
 	}
 
-	edgeStack.ManifestPath = komposeFileName
-
-	return nil
+	return komposeFileName, nil
 }

api/http/handler/edgeupdateschedules/agentversions.go

@@ -1,32 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-)
-
-// @id AgentVersions
-// @summary Fetches the supported versions of the agent to update/rollback
-// @description
-// @description **Access policy**: authenticated
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @success 200 {array} string
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @router /edge_update_schedules/agent_versions [get]
-func (h *Handler) agentVersions(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	return response.JSON(w, []string{
-		"2.13.0",
-		"2.13.1",
-		"2.14.0",
-		"2.14.1",
-		"2.14.2",
-		"2.15",    // for develop only
-		"develop", // for develop only
-	})
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_activeschedules.go

@@ -1,42 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-)
-
-type activeSchedulePayload struct {
-	EnvironmentIDs []portainer.EndpointID
-}
-
-func (payload *activeSchedulePayload) Validate(r *http.Request) error {
-	return nil
-}
-
-// @id EdgeUpdateScheduleActiveSchedulesList
-// @summary Fetches the list of Active Edge Update Schedules
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @accept json
-// @param body body activeSchedulePayload true "Active schedule query"
-// @produce json
-// @success 200 {array} edgetypes.EdgeUpdateSchedule
-// @failure 500
-// @router /edge_update_schedules/active [get]
-func (handler *Handler) activeSchedules(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload activeSchedulePayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
-	}
-
-	list := handler.dataStore.EdgeUpdateSchedule().ActiveSchedules(payload.EnvironmentIDs)
-
-	return response.JSON(w, list)
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_create.go

@@ -1,134 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"errors"
-	"net/http"
-	"time"
-
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-type createPayload struct {
-	Name         string
-	GroupIDs     []portainer.EdgeGroupID
-	Type         edgetypes.UpdateScheduleType
-	Environments map[portainer.EndpointID]string
-	Time         int64
-}
-
-func (payload *createPayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid tag name")
-	}
-
-	if len(payload.GroupIDs) == 0 {
-		return errors.New("Required to choose at least one group")
-	}
-
-	if len(payload.Environments) == 0 {
-		return errors.New("No Environment is scheduled for update")
-	}
-
-	if payload.Type != edgetypes.UpdateScheduleRollback && payload.Type != edgetypes.UpdateScheduleUpdate {
-		return errors.New("Invalid schedule type")
-	}
-
-	if payload.Time < time.Now().Unix() {
-		return errors.New("Invalid time")
-	}
-
-	return nil
-}
-
-// @id EdgeUpdateScheduleCreate
-// @summary Creates a new Edge Update Schedule
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @accept json
-// @param body body createPayload true "Schedule details"
-// @produce json
-// @success 200 {object} edgetypes.UpdateSchedule
-// @failure 500
-// @router /edge_update_schedules [post]
-func (handler *Handler) create(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-
-	var payload createPayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
-	}
-
-	err = handler.validateUniqueName(payload.Name, 0)
-	if err != nil {
-		return httperror.NewError(http.StatusConflict, "Edge update schedule name already in use", err)
-
-	}
-
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve user information from token", err)
-	}
-
-	item := &edgetypes.UpdateSchedule{
-		Name:      payload.Name,
-		Time:      payload.Time,
-		GroupIDs:  payload.GroupIDs,
-		Status:    map[portainer.EndpointID]edgetypes.UpdateScheduleStatus{},
-		Created:   time.Now().Unix(),
-		CreatedBy: tokenData.ID,
-		Type:      payload.Type,
-	}
-
-	schedules, err := handler.dataStore.EdgeUpdateSchedule().List()
-	if err != nil {
-		return httperror.InternalServerError("Unable to list edge update schedules", err)
-	}
-
-	prevVersions := map[portainer.EndpointID]string{}
-	if item.Type == edgetypes.UpdateScheduleRollback {
-		prevVersions = previousVersions(schedules)
-	}
-
-	for environmentID, version := range payload.Environments {
-		environment, err := handler.dataStore.Endpoint().Endpoint(environmentID)
-		if err != nil {
-			return httperror.InternalServerError("Unable to retrieve environment from the database", err)
-		}
-
-		// TODO check that env is standalone (snapshots)
-		if environment.Type != portainer.EdgeAgentOnDockerEnvironment {
-			return httperror.BadRequest("Only standalone docker Environments are supported for remote update", nil)
-		}
-
-		// validate version id is valid for rollback
-		if item.Type == edgetypes.UpdateScheduleRollback {
-			if prevVersions[environmentID] == "" {
-				return httperror.BadRequest("No previous version found for environment", nil)
-			}
-
-			if version != prevVersions[environmentID] {
-				return httperror.BadRequest("Rollback version must match previous version", nil)
-			}
-		}
-
-		item.Status[environmentID] = edgetypes.UpdateScheduleStatus{
-			TargetVersion:  version,
-			CurrentVersion: environment.Agent.Version,
-		}
-	}
-
-	err = handler.dataStore.EdgeUpdateSchedule().Create(item)
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist the edge update schedule", err)
-	}
-
-	return response.JSON(w, item)
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_delete.go

@@ -1,33 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/portainer/portainer/api/http/middlewares"
-)
-
-// @id EdgeUpdateScheduleDelete
-// @summary Deletes an Edge Update Schedule
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @success 204
-// @failure 500
-// @router /edge_update_schedules/{id} [delete]
-func (handler *Handler) delete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	item, err := middlewares.FetchItem[edgetypes.UpdateSchedule](r, contextKey)
-	if err != nil {
-		return httperror.InternalServerError(err.Error(), err)
-	}
-
-	err = handler.dataStore.EdgeUpdateSchedule().Delete(item.ID)
-	if err != nil {
-		return httperror.InternalServerError("Unable to delete the edge update schedule", err)
-	}
-
-	return response.Empty(w)
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_inspect.go

@@ -1,29 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/portainer/portainer/api/http/middlewares"
-)
-
-// @id EdgeUpdateScheduleInspect
-// @summary Returns the Edge Update Schedule with the given ID
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @success 200 {object} edgetypes.UpdateSchedule
-// @failure 500
-// @router /edge_update_schedules/{id} [get]
-func (handler *Handler) inspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	item, err := middlewares.FetchItem[edgetypes.UpdateSchedule](r, contextKey)
-	if err != nil {
-		return httperror.InternalServerError(err.Error(), err)
-	}
-
-	return response.JSON(w, item)
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_list.go

@@ -1,27 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-)
-
-// @id EdgeUpdateScheduleList
-// @summary Fetches the list of Edge Update Schedules
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @success 200 {array} edgetypes.UpdateSchedule
-// @failure 500
-// @router /edge_update_schedules [get]
-func (handler *Handler) list(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	list, err := handler.dataStore.EdgeUpdateSchedule().List()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve the edge update schedules list", err)
-	}
-
-	return response.JSON(w, list)
-}

api/http/handler/edgeupdateschedules/edgeupdateschedule_update.go

@@ -1,108 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"errors"
-	"net/http"
-	"time"
-
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/portainer/portainer/api/http/middlewares"
-)
-
-type updatePayload struct {
-	Name         string
-	GroupIDs     []portainer.EdgeGroupID
-	Environments map[portainer.EndpointID]string
-	Type         edgetypes.UpdateScheduleType
-	Time         int64
-}
-
-func (payload *updatePayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.Name) {
-		return errors.New("Invalid tag name")
-	}
-
-	if len(payload.GroupIDs) == 0 {
-		return errors.New("Required to choose at least one group")
-	}
-
-	if payload.Type != edgetypes.UpdateScheduleRollback && payload.Type != edgetypes.UpdateScheduleUpdate {
-		return errors.New("Invalid schedule type")
-	}
-
-	if len(payload.Environments) == 0 {
-		return errors.New("No Environment is scheduled for update")
-	}
-
-	return nil
-}
-
-// @id EdgeUpdateScheduleUpdate
-// @summary Updates an Edge Update Schedule
-// @description **Access policy**: administrator
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @accept json
-// @param body body updatePayload true "Schedule details"
-// @produce json
-// @success 204
-// @failure 500
-// @router /edge_update_schedules [post]
-func (handler *Handler) update(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	item, err := middlewares.FetchItem[edgetypes.UpdateSchedule](r, contextKey)
-	if err != nil {
-		return httperror.InternalServerError(err.Error(), err)
-	}
-
-	var payload updatePayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
-	}
-
-	if payload.Name != item.Name {
-		err = handler.validateUniqueName(payload.Name, item.ID)
-		if err != nil {
-			return httperror.NewError(http.StatusConflict, "Edge update schedule name already in use", err)
-		}
-
-		item.Name = payload.Name
-	}
-
-	// if scheduled time didn't passed, then can update the schedule
-	if item.Time > time.Now().Unix() {
-		item.GroupIDs = payload.GroupIDs
-		item.Time = payload.Time
-		item.Type = payload.Type
-
-		item.Status = map[portainer.EndpointID]edgetypes.UpdateScheduleStatus{}
-		for environmentID, version := range payload.Environments {
-			environment, err := handler.dataStore.Endpoint().Endpoint(environmentID)
-			if err != nil {
-				return httperror.InternalServerError("Unable to retrieve environment from the database", err)
-			}
-
-			if environment.Type != portainer.EdgeAgentOnDockerEnvironment {
-				return httperror.BadRequest("Only standalone docker Environments are supported for remote update", nil)
-			}
-
-			item.Status[environmentID] = edgetypes.UpdateScheduleStatus{
-				TargetVersion:  version,
-				CurrentVersion: environment.Agent.Version,
-			}
-		}
-	}
-
-	err = handler.dataStore.EdgeUpdateSchedule().Update(item.ID, item)
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist the edge update schedule", err)
-	}
-
-	return response.JSON(w, item)
-}

api/http/handler/edgeupdateschedules/handler.go

@@ -1,67 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-
-	"github.com/gorilla/mux"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/portainer/portainer/api/http/middlewares"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-const contextKey = "edgeUpdateSchedule_item"
-
-// Handler is the HTTP handler used to handle edge environment update operations.
-type Handler struct {
-	*mux.Router
-	requestBouncer *security.RequestBouncer
-	dataStore      dataservices.DataStore
-}
-
-// NewHandler creates a handler to manage environment update operations.
-func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore) *Handler {
-	h := &Handler{
-		Router:         mux.NewRouter(),
-		requestBouncer: bouncer,
-		dataStore:      dataStore,
-	}
-
-	router := h.PathPrefix("/edge_update_schedules").Subrouter()
-	router.Use(bouncer.AdminAccess)
-	router.Use(middlewares.FeatureFlag(dataStore.Settings(), portainer.FeatureFlagEdgeRemoteUpdate))
-
-	router.Handle("",
-		httperror.LoggerHandler(h.list)).Methods(http.MethodGet)
-
-	router.Handle("",
-		httperror.LoggerHandler(h.create)).Methods(http.MethodPost)
-
-	router.Handle("/active",
-		httperror.LoggerHandler(h.activeSchedules)).Methods(http.MethodPost)
-
-	router.Handle("/agent_versions",
-		httperror.LoggerHandler(h.agentVersions)).Methods(http.MethodGet)
-
-	router.Handle("/previous_versions",
-		httperror.LoggerHandler(h.previousVersions)).Methods(http.MethodGet)
-
-	itemRouter := router.PathPrefix("/{id}").Subrouter()
-	itemRouter.Use(middlewares.WithItem(func(id edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error) {
-		return dataStore.EdgeUpdateSchedule().Item(id)
-	}, "id", contextKey))
-
-	itemRouter.Handle("",
-		httperror.LoggerHandler(h.inspect)).Methods(http.MethodGet)
-
-	itemRouter.Handle("",
-		httperror.LoggerHandler(h.update)).Methods(http.MethodPut)
-
-	itemRouter.Handle("",
-		httperror.LoggerHandler(h.delete)).Methods(http.MethodDelete)
-
-	return h
-}

api/http/handler/edgeupdateschedules/previous_versions.go

@@ -1,86 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"net/http"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/response"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/edgetypes"
-	"golang.org/x/exp/slices"
-)
-
-// @id EdgeUpdatePreviousVersions
-// @summary Fetches the previous versions of updated agents
-// @description
-// @description **Access policy**: authenticated
-// @tags edge_update_schedules
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @success 200 {array} string
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @router /edge_update_schedules/agent_versions [get]
-func (handler *Handler) previousVersions(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	schedules, err := handler.dataStore.EdgeUpdateSchedule().List()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve the edge update schedules list", err)
-	}
-
-	versionMap := previousVersions(schedules)
-
-	return response.JSON(w, versionMap)
-}
-
-type EnvironmentVersionDetails struct {
-	version    string
-	skip       bool
-	skipReason string
-}
-
-func previousVersions(schedules []edgetypes.UpdateSchedule) map[portainer.EndpointID]string {
-
-	slices.SortFunc(schedules, func(a edgetypes.UpdateSchedule, b edgetypes.UpdateSchedule) bool {
-		return a.Created > b.Created
-	})
-
-	environmentMap := map[portainer.EndpointID]*EnvironmentVersionDetails{}
-	// to all schedules[:schedule index -1].Created > schedule.Created
-	for _, schedule := range schedules {
-		for environmentId, status := range schedule.Status {
-			props, ok := environmentMap[environmentId]
-			if !ok {
-				environmentMap[environmentId] = &EnvironmentVersionDetails{}
-				props = environmentMap[environmentId]
-			}
-
-			if props.version != "" || props.skip {
-				continue
-			}
-
-			if schedule.Type == edgetypes.UpdateScheduleRollback {
-				props.skip = true
-				props.skipReason = "has rollback"
-				continue
-			}
-
-			if status.Status == edgetypes.UpdateScheduleStatusPending || status.Status == edgetypes.UpdateScheduleStatusError {
-				props.skip = true
-				props.skipReason = "has active schedule"
-				continue
-			}
-
-			props.version = status.CurrentVersion
-		}
-	}
-
-	versionMap := map[portainer.EndpointID]string{}
-	for environmentId, props := range environmentMap {
-		if !props.skip {
-			versionMap[environmentId] = props.version
-		}
-	}
-
-	return versionMap
-}

api/http/handler/edgeupdateschedules/previous_versions_test.go

@@ -1,63 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestPreviousVersions(t *testing.T) {
-
-	schedules := []edgetypes.UpdateSchedule{
-		{
-			ID:   1,
-			Type: edgetypes.UpdateScheduleUpdate,
-			Status: map[portainer.EndpointID]edgetypes.UpdateScheduleStatus{
-				1: {
-					TargetVersion:  "2.14.0",
-					CurrentVersion: "2.11.0",
-					Status:         edgetypes.UpdateScheduleStatusSuccess,
-				},
-				2: {
-					TargetVersion:  "2.13.0",
-					CurrentVersion: "2.12.0",
-					Status:         edgetypes.UpdateScheduleStatusSuccess,
-				},
-			},
-			Created: 1500000000,
-		},
-		{
-			ID:   2,
-			Type: edgetypes.UpdateScheduleRollback,
-			Status: map[portainer.EndpointID]edgetypes.UpdateScheduleStatus{
-				1: {
-					TargetVersion:  "2.11.0",
-					CurrentVersion: "2.14.0",
-					Status:         edgetypes.UpdateScheduleStatusSuccess,
-				},
-			},
-			Created: 1500000001,
-		},
-		{
-			ID:   3,
-			Type: edgetypes.UpdateScheduleUpdate,
-			Status: map[portainer.EndpointID]edgetypes.UpdateScheduleStatus{
-				2: {
-					TargetVersion:  "2.14.0",
-					CurrentVersion: "2.13.0",
-					Status:         edgetypes.UpdateScheduleStatusSuccess,
-				},
-			},
-			Created: 1500000002,
-		},
-	}
-
-	actual := previousVersions(schedules)
-
-	assert.Equal(t, map[portainer.EndpointID]string{
-		2: "2.13.0",
-	}, actual)
-
-}

api/http/handler/edgeupdateschedules/utils.go

@@ -1,21 +0,0 @@
-package edgeupdateschedules
-
-import (
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/edgetypes"
-)
-
-func (handler *Handler) validateUniqueName(name string, id edgetypes.UpdateScheduleID) error {
-	list, err := handler.dataStore.EdgeUpdateSchedule().List()
-	if err != nil {
-		return errors.WithMessage(err, "Unable to list edge update schedules")
-	}
-
-	for _, schedule := range list {
-		if id != schedule.ID && schedule.Name == name {
-			return errors.New("Edge update schedule name already in use")
-		}
-	}
-
-	return nil
-}

api/http/handler/endpointedge/endpoint_edgejob_logs.go

@@ -65,10 +65,12 @@ func (handler *Handler) endpointEdgeJobsLogs(w http.ResponseWriter, r *http.Requ
 		return httperror.InternalServerError("Unable to save task log to the filesystem", err)
 	}
 
-	meta := edgeJob.Endpoints[endpoint.ID]
-	meta.CollectLogs = false
-	meta.LogsStatus = portainer.EdgeJobLogsStatusCollected
-	edgeJob.Endpoints[endpoint.ID] = meta
+	meta := portainer.EdgeJobEndpointMeta{CollectLogs: false, LogsStatus: portainer.EdgeJobLogsStatusCollected}
+	if _, ok := edgeJob.GroupLogsCollection[endpoint.ID]; ok {
+		edgeJob.GroupLogsCollection[endpoint.ID] = meta
+	} else {
+		edgeJob.Endpoints[endpoint.ID] = meta
+	}
 
 	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
 

api/http/handler/endpointedge/endpoint_edgestack_inspect.go

@@ -10,11 +10,14 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes"
 )
 
 type configResponse struct {
 	StackFileContent string
 	Name             string
+	// Namespace to use for Kubernetes manifests, leave empty to use the namespaces defined in the manifest
+	Namespace string
 }
 
 // @summary Inspect an Edge Stack for an Environment(Endpoint)
@@ -59,12 +62,18 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
 		}
 	}
 
+	namespace := ""
+	if !edgeStack.UseManifestNamespaces {
+		namespace = kubernetes.DefaultNamespace
+	}
+
 	if endpointutils.IsKubernetesEndpoint(endpoint) {
 		fileName = edgeStack.ManifestPath
 
 		if fileName == "" {
 			return httperror.BadRequest("Kubernetes is not supported by this stack", errors.New("Kubernetes is not supported by this stack"))
 		}
+
 	}
 
 	stackFileContent, err := handler.FileService.GetFileContent(edgeStack.ProjectPath, fileName)
@@ -75,5 +84,6 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
 	return response.JSON(w, configResponse{
 		StackFileContent: string(stackFileContent),
 		Name:             edgeStack.Name,
+		Namespace:        namespace,
 	})
 }

api/http/handler/endpointedge/endpoint_edgestatus_inspect.go

@@ -158,10 +158,17 @@ func parseAgentPlatform(r *http.Request) (portainer.EndpointType, error) {
 func (handler *Handler) buildSchedules(endpointID portainer.EndpointID, tunnel portainer.TunnelDetails) ([]edgeJobResponse, *httperror.HandlerError) {
 	schedules := []edgeJobResponse{}
 	for _, job := range tunnel.Jobs {
+		var collectLogs bool
+		if _, ok := job.GroupLogsCollection[endpointID]; ok {
+			collectLogs = job.GroupLogsCollection[endpointID].CollectLogs
+		} else {
+			collectLogs = job.Endpoints[endpointID].CollectLogs
+		}
+
 		schedule := edgeJobResponse{
 			ID:             job.ID,
 			CronExpression: job.CronExpression,
-			CollectLogs:    job.Endpoints[endpointID].CollectLogs,
+			CollectLogs:    collectLogs,
 			Version:        job.Version,
 		}
 

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -324,7 +324,7 @@ func TestEdgeStackStatus(t *testing.T) {
 		ID:   edgeStackID,
 		Name: "test-edge-stack-17",
 		Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
-			endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpoint.ID},
+			endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpoint.ID},
 		},
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{1, 2},

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -91,7 +91,7 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+		err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
 			tag.EndpointGroups[endpointGroup.ID] = true
 		})
 

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -66,7 +66,7 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+		err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
 			delete(tag.EndpointGroups, endpointGroup.ID)
 		})
 

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -81,7 +81,7 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 			removeTags := tag.Difference(endpointGroupTagSet, payloadTagSet)
 
 			for tagID := range removeTags {
-				handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+				err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
 					delete(tag.EndpointGroups, endpointGroup.ID)
 				})
 
@@ -94,7 +94,7 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 
 			endpointGroup.TagIDs = payload.TagIDs
 			for _, tagID := range payload.TagIDs {
-				handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+				err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
 					tag.EndpointGroups[endpointGroup.ID] = true
 				})
 

api/http/handler/endpoints/endpoint_create.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
 type endpointCreatePayload struct {
@@ -244,6 +245,22 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 		for _, stackID := range relatedEdgeStacks {
 			relationObject.EdgeStacks[stackID] = true
 		}
+	} else if endpointutils.IsKubernetesEndpoint(endpoint) {
+		endpointutils.InitialIngressClassDetection(
+			endpoint,
+			handler.DataStore.Endpoint(),
+			handler.K8sClientFactory,
+		)
+		endpointutils.InitialMetricsDetection(
+			endpoint,
+			handler.DataStore.Endpoint(),
+			handler.K8sClientFactory,
+		)
+		endpointutils.InitialStorageDetection(
+			endpoint,
+			handler.DataStore.Endpoint(),
+			handler.K8sClientFactory,
+		)
 	}
 
 	err = handler.DataStore.EndpointRelation().Create(relationObject)
@@ -385,7 +402,7 @@ func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload)
 
 	err = handler.saveEndpointAndUpdateAuthorizations(endpoint)
 	if err != nil {
-		return nil, httperror.InternalServerError("An error occured while trying to create the environment", err)
+		return nil, httperror.InternalServerError("An error occurred while trying to create the environment", err)
 	}
 
 	return endpoint, nil
@@ -515,7 +532,7 @@ func (handler *Handler) snapshotAndPersistEndpoint(endpoint *portainer.Endpoint)
 
 	err = handler.saveEndpointAndUpdateAuthorizations(endpoint)
 	if err != nil {
-		return httperror.InternalServerError("An error occured while trying to create the environment", err)
+		return httperror.InternalServerError("An error occurred while trying to create the environment", err)
 	}
 
 	return nil

api/http/handler/endpoints/endpoint_delete.go

@@ -9,6 +9,7 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
 // @id EndpointDelete
@@ -49,6 +50,11 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
+	err = handler.DataStore.Snapshot().DeleteSnapshot(portainer.EndpointID(endpointID))
+	if err != nil {
+		return httperror.InternalServerError("Unable to remove the snapshot from the database", err)
+	}
+
 	err = handler.DataStore.Endpoint().DeleteEndpoint(portainer.EndpointID(endpointID))
 	if err != nil {
 		return httperror.InternalServerError("Unable to remove environment from the database", err)
@@ -78,15 +84,12 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		return httperror.InternalServerError("Unable to retrieve edge groups from the database", err)
 	}
 
-	for idx := range edgeGroups {
-		edgeGroup := &edgeGroups[idx]
-		endpointIdx := findEndpointIndex(edgeGroup.Endpoints, endpoint.ID)
-		if endpointIdx != -1 {
-			edgeGroup.Endpoints = removeElement(edgeGroup.Endpoints, endpointIdx)
-			err = handler.DataStore.EdgeGroup().UpdateEdgeGroup(edgeGroup.ID, edgeGroup)
-			if err != nil {
-				return httperror.InternalServerError("Unable to update edge group", err)
-			}
+	for _, edgeGroup := range edgeGroups {
+		err = handler.DataStore.EdgeGroup().UpdateEdgeGroupFunc(edgeGroup.ID, func(g *portainer.EdgeGroup) {
+			g.Endpoints = removeElement(g.Endpoints, endpoint.ID)
+		})
+		if err != nil {
+			return httperror.InternalServerError("Unable to update edge group", err)
 		}
 	}
 
@@ -122,23 +125,39 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
+	if !endpointutils.IsEdgeEndpoint(endpoint) {
+		return response.Empty(w)
+	}
+
+	edgeJobs, err := handler.DataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve edge jobs from the database", err)
+	}
+
+	for idx := range edgeJobs {
+		edgeJob := &edgeJobs[idx]
+		if _, ok := edgeJob.Endpoints[endpoint.ID]; ok {
+			err = handler.DataStore.EdgeJob().UpdateEdgeJobFunc(edgeJob.ID, func(j *portainer.EdgeJob) {
+				delete(j.Endpoints, endpoint.ID)
+			})
+
+			if err != nil {
+				return httperror.InternalServerError("Unable to update edge job", err)
+			}
+		}
+	}
+
 	return response.Empty(w)
 }
 
-func findEndpointIndex(tags []portainer.EndpointID, searchEndpointID portainer.EndpointID) int {
-	for idx, tagID := range tags {
-		if searchEndpointID == tagID {
-			return idx
+func removeElement(slice []portainer.EndpointID, elem portainer.EndpointID) []portainer.EndpointID {
+	for i, id := range slice {
+		if id == elem {
+			slice[i] = slice[len(slice)-1]
+
+			return slice[:len(slice)-1]
 		}
 	}
-	return -1
-}
 
-func removeElement(arr []portainer.EndpointID, index int) []portainer.EndpointID {
-	if index < 0 {
-		return arr
-	}
-	lastTagIdx := len(arr) - 1
-	arr[index] = arr[lastTagIdx]
-	return arr[:lastTagIdx]
+	return slice
 }

api/http/handler/endpoints/endpoint_delete_test.go

@@ -0,0 +1,109 @@
+package endpoints
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/apikey"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/demo"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/jwt"
+)
+
+func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
+	const endpointsCount = 100
+
+	_, store, teardown := datastore.MustNewTestStore(t, true, false)
+	defer teardown()
+
+	user := &portainer.User{ID: 2, Username: "admin", Role: portainer.AdministratorRole}
+	err := store.User().Create(user)
+	if err != nil {
+		t.Fatal("could not create admin user:", err)
+	}
+
+	jwtService, err := jwt.NewService("1h", store)
+	if err != nil {
+		t.Fatal("could not initialize the JWT service:", err)
+	}
+
+	apiKeyService := apikey.NewAPIKeyService(store.APIKeyRepository(), store.User())
+	rawAPIKey, _, err := apiKeyService.GenerateApiKey(*user, "test")
+	if err != nil {
+		t.Fatal("could not generate API key:", err)
+	}
+
+	bouncer := security.NewRequestBouncer(store, jwtService, apiKeyService)
+
+	handler := NewHandler(bouncer, demo.NewService())
+	handler.DataStore = store
+	handler.ProxyManager = proxy.NewManager(nil, nil, nil, nil, nil, nil, nil)
+
+	// Create all the environments and add them to the same edge group
+
+	var endpointIDs []portainer.EndpointID
+
+	for i := 0; i < endpointsCount; i++ {
+		endpointID := portainer.EndpointID(i) + 1
+
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:   endpointID,
+			Name: "env-" + strconv.Itoa(int(endpointID)),
+			Type: portainer.EdgeAgentOnDockerEnvironment,
+		})
+		if err != nil {
+			t.Fatal("could not create endpoint:", err)
+		}
+
+		endpointIDs = append(endpointIDs, endpointID)
+	}
+
+	err = store.EdgeGroup().Create(&portainer.EdgeGroup{
+		ID:        1,
+		Name:      "edgegroup-1",
+		Endpoints: endpointIDs,
+	})
+	if err != nil {
+		t.Fatal("could not create edge group:", err)
+	}
+
+	// Remove the environments concurrently
+
+	var wg sync.WaitGroup
+	wg.Add(len(endpointIDs))
+
+	for _, endpointID := range endpointIDs {
+		go func(ID portainer.EndpointID) {
+			defer wg.Done()
+
+			req, err := http.NewRequest(http.MethodDelete, "/endpoints/"+strconv.Itoa(int(ID)), nil)
+			if err != nil {
+				t.Fail()
+				return
+			}
+			req.Header.Add("X-Api-Key", rawAPIKey)
+
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+		}(endpointID)
+	}
+
+	wg.Wait()
+
+	// Check that the edge group is consistent
+
+	edgeGroup, err := handler.DataStore.EdgeGroup().EdgeGroup(1)
+	if err != nil {
+		t.Fatal("could not retrieve the edge group:", err)
+	}
+
+	if len(edgeGroup.Endpoints) > 0 {
+		t.Fatal("the edge group is not consistent")
+	}
+}

api/http/handler/endpoints/endpoint_list.go

@@ -123,6 +123,10 @@ func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []porta
 
 	endpointCount := len(endpoints)
 
+	if start < 0 {
+		start = 0
+	}
+
 	if start > endpointCount {
 		start = endpointCount
 	}

api/http/handler/endpoints/filter.go

@@ -11,7 +11,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/endpointutils"
-	"golang.org/x/exp/slices"
+	"github.com/portainer/portainer/api/internal/slices"
 )
 
 type EnvironmentsQuery struct {

api/http/handler/gitops/git_repo_file_preview.go

@@ -1,87 +0,0 @@
-package gitops
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-
-	"github.com/asaskevich/govalidator"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/libhttp/response"
-	gittypes "github.com/portainer/portainer/api/git/types"
-)
-
-type fileResponse struct {
-	FileContent string
-}
-
-type repositoryFilePreviewPayload struct {
-	Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"`
-	Reference  string `json:"reference" example:"refs/heads/master"`
-	Username   string `json:"username" example:"myGitUsername"`
-	Password   string `json:"password" example:"myGitPassword"`
-	// Path to file whose content will be read
-	TargetFile string `json:"targetFile" example:"docker-compose.yml"`
-}
-
-func (payload *repositoryFilePreviewPayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.Repository) || !govalidator.IsURL(payload.Repository) {
-		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
-	}
-
-	if govalidator.IsNull(payload.Reference) {
-		payload.Reference = "refs/heads/main"
-	}
-
-	if govalidator.IsNull(payload.TargetFile) {
-		return errors.New("Invalid target filename.")
-	}
-
-	return nil
-}
-
-// @id GitOperationRepoFilePreview
-// @summary preview the content of target file in the git repository
-// @description Retrieve the compose file content based on git repository configuration
-// @description **Access policy**: authenticated
-// @tags gitops
-// @security ApiKeyAuth
-// @security jwt
-// @produce json
-// @param body body repositoryFilePreviewPayload true "Template details"
-// @success 200 {object} fileResponse "Success"
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @router /gitops/repo/file/preview [post]
-func (handler *Handler) gitOperationRepoFilePreview(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload repositoryFilePreviewPayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
-	}
-
-	projectPath, err := handler.fileService.GetTemporaryPath()
-	if err != nil {
-		return httperror.InternalServerError("Unable to create temporary folder", err)
-	}
-
-	err = handler.gitService.CloneRepository(projectPath, payload.Repository, payload.Reference, payload.Username, payload.Password)
-	if err != nil {
-		if err == gittypes.ErrAuthenticationFailure {
-			return httperror.BadRequest("Invalid git credential", err)
-		}
-
-		newErr := fmt.Errorf("unable to clone git repository: %w", err)
-		return httperror.InternalServerError(newErr.Error(), newErr)
-	}
-
-	defer handler.fileService.RemoveDirectory(projectPath)
-
-	fileContent, err := handler.fileService.GetFileContent(projectPath, payload.TargetFile)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
-	}
-
-	return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
-}

api/http/handler/gitops/handler.go

@@ -1,33 +0,0 @@
-package gitops
-
-import (
-	"net/http"
-
-	"github.com/gorilla/mux"
-	httperror "github.com/portainer/libhttp/error"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-// Handler is the HTTP handler used to handle git repo operation
-type Handler struct {
-	*mux.Router
-	dataStore   dataservices.DataStore
-	gitService  portainer.GitService
-	fileService portainer.FileService
-}
-
-func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService) *Handler {
-	h := &Handler{
-		Router:      mux.NewRouter(),
-		dataStore:   dataStore,
-		gitService:  gitService,
-		fileService: fileService,
-	}
-
-	h.Handle("/gitops/repo/file/preview",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.gitOperationRepoFilePreview))).Methods(http.MethodPost)
-
-	return h
-}

api/http/handler/handler.go

@@ -12,13 +12,11 @@ import (
 	"github.com/portainer/portainer/api/http/handler/edgejobs"
 	"github.com/portainer/portainer/api/http/handler/edgestacks"
 	"github.com/portainer/portainer/api/http/handler/edgetemplates"
-	"github.com/portainer/portainer/api/http/handler/edgeupdateschedules"
 	"github.com/portainer/portainer/api/http/handler/endpointedge"
 	"github.com/portainer/portainer/api/http/handler/endpointgroups"
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
-	"github.com/portainer/portainer/api/http/handler/gitops"
 	"github.com/portainer/portainer/api/http/handler/helm"
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/fdo"
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
@@ -31,8 +29,8 @@ import (
 	"github.com/portainer/portainer/api/http/handler/settings"
 	"github.com/portainer/portainer/api/http/handler/ssl"
 	"github.com/portainer/portainer/api/http/handler/stacks"
-	"github.com/portainer/portainer/api/http/handler/status"
 	"github.com/portainer/portainer/api/http/handler/storybook"
+	"github.com/portainer/portainer/api/http/handler/system"
 	"github.com/portainer/portainer/api/http/handler/tags"
 	"github.com/portainer/portainer/api/http/handler/teammemberships"
 	"github.com/portainer/portainer/api/http/handler/teams"
@@ -45,44 +43,42 @@ import (
 
 // Handler is a collection of all the service handlers.
 type Handler struct {
-	AuthHandler               *auth.Handler
-	BackupHandler             *backup.Handler
-	CustomTemplatesHandler    *customtemplates.Handler
-	DockerHandler             *docker.Handler
-	EdgeGroupsHandler         *edgegroups.Handler
-	EdgeJobsHandler           *edgejobs.Handler
-	EdgeUpdateScheduleHandler *edgeupdateschedules.Handler
-	EdgeStacksHandler         *edgestacks.Handler
-	EdgeTemplatesHandler      *edgetemplates.Handler
-	EndpointEdgeHandler       *endpointedge.Handler
-	EndpointGroupHandler      *endpointgroups.Handler
-	EndpointHandler           *endpoints.Handler
-	EndpointHelmHandler       *helm.Handler
-	EndpointProxyHandler      *endpointproxy.Handler
-	GitOperationHandler       *gitops.Handler
-	HelmTemplatesHandler      *helm.Handler
-	KubernetesHandler         *kubernetes.Handler
-	FileHandler               *file.Handler
-	LDAPHandler               *ldap.Handler
-	MOTDHandler               *motd.Handler
-	RegistryHandler           *registries.Handler
-	ResourceControlHandler    *resourcecontrols.Handler
-	RoleHandler               *roles.Handler
-	SettingsHandler           *settings.Handler
-	SSLHandler                *ssl.Handler
-	OpenAMTHandler            *openamt.Handler
-	FDOHandler                *fdo.Handler
-	StackHandler              *stacks.Handler
-	StatusHandler             *status.Handler
-	StorybookHandler          *storybook.Handler
-	TagHandler                *tags.Handler
-	TeamMembershipHandler     *teammemberships.Handler
-	TeamHandler               *teams.Handler
-	TemplatesHandler          *templates.Handler
-	UploadHandler             *upload.Handler
-	UserHandler               *users.Handler
-	WebSocketHandler          *websocket.Handler
-	WebhookHandler            *webhooks.Handler
+	AuthHandler            *auth.Handler
+	BackupHandler          *backup.Handler
+	CustomTemplatesHandler *customtemplates.Handler
+	DockerHandler          *docker.Handler
+	EdgeGroupsHandler      *edgegroups.Handler
+	EdgeJobsHandler        *edgejobs.Handler
+	EdgeStacksHandler      *edgestacks.Handler
+	EdgeTemplatesHandler   *edgetemplates.Handler
+	EndpointEdgeHandler    *endpointedge.Handler
+	EndpointGroupHandler   *endpointgroups.Handler
+	EndpointHandler        *endpoints.Handler
+	EndpointHelmHandler    *helm.Handler
+	EndpointProxyHandler   *endpointproxy.Handler
+	HelmTemplatesHandler   *helm.Handler
+	KubernetesHandler      *kubernetes.Handler
+	FileHandler            *file.Handler
+	LDAPHandler            *ldap.Handler
+	MOTDHandler            *motd.Handler
+	RegistryHandler        *registries.Handler
+	ResourceControlHandler *resourcecontrols.Handler
+	RoleHandler            *roles.Handler
+	SettingsHandler        *settings.Handler
+	SSLHandler             *ssl.Handler
+	OpenAMTHandler         *openamt.Handler
+	FDOHandler             *fdo.Handler
+	StackHandler           *stacks.Handler
+	StorybookHandler       *storybook.Handler
+	SystemHandler          *system.Handler
+	TagHandler             *tags.Handler
+	TeamMembershipHandler  *teammemberships.Handler
+	TeamHandler            *teams.Handler
+	TemplatesHandler       *templates.Handler
+	UploadHandler          *upload.Handler
+	UserHandler            *users.Handler
+	WebSocketHandler       *websocket.Handler
+	WebhookHandler         *webhooks.Handler
 }
 
 // @title PortainerCE API
@@ -125,8 +121,6 @@ type Handler struct {
 // @tag.description Manage Docker environments(endpoints)
 // @tag.name endpoint_groups
 // @tag.description Manage environment(endpoint) groups
-// @tag.name gitops
-// @tag.description Operate git repository
 // @tag.name kubernetes
 // @tag.description Manage Kubernetes cluster
 // @tag.name motd
@@ -139,8 +133,6 @@ type Handler struct {
 // @tag.description Manage roles
 // @tag.name settings
 // @tag.description Manage Portainer settings
-// @tag.name status
-// @tag.description Information about the Portainer instance
 // @tag.name users
 // @tag.description Manage users
 // @tag.name tags
@@ -161,6 +153,10 @@ type Handler struct {
 // @tag.description Manage webhooks
 // @tag.name websocket
 // @tag.description Create exec sessions using websockets
+// @tag.name status
+// @tag.description Information about the Portainer instance
+// @tag.name system
+// @tag.description Manage Portainer system
 
 // ServeHTTP delegates a request to the appropriate subhandler.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -173,8 +169,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/custom_templates"):
 		http.StripPrefix("/api", h.CustomTemplatesHandler).ServeHTTP(w, r)
-	case strings.HasPrefix(r.URL.Path, "/api/edge_update_schedules"):
-		http.StripPrefix("/api", h.EdgeUpdateScheduleHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/edge_stacks"):
 		http.StripPrefix("/api", h.EdgeStacksHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/edge_groups"):
@@ -211,8 +205,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		default:
 			http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r)
 		}
-	case strings.HasPrefix(r.URL.Path, "/api/gitops"):
-		http.StripPrefix("/api", h.GitOperationHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/ldap"):
 		http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/motd"):
@@ -228,7 +220,9 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case strings.HasPrefix(r.URL.Path, "/api/stacks"):
 		http.StripPrefix("/api", h.StackHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/status"):
-		http.StripPrefix("/api", h.StatusHandler).ServeHTTP(w, r)
+		http.StripPrefix("/api", h.SystemHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/system"):
+		http.StripPrefix("/api", h.SystemHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/tags"):
 		http.StripPrefix("/api", h.TagHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/templates/helm"):

api/http/handler/helm/handler.go

@@ -4,14 +4,14 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/libhelm"
-	"github.com/portainer/libhelm/options"
 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes"
+	"github.com/portainer/portainer/pkg/libhelm"
+	"github.com/portainer/portainer/pkg/libhelm/options"
 )
 
 type requestBouncer interface {

api/http/handler/helm/helm_delete.go

@@ -3,10 +3,10 @@ package helm
 import (
 	"net/http"
 
-	"github.com/portainer/libhelm/options"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/pkg/libhelm/options"
 )
 
 // @id HelmDelete

api/http/handler/helm/helm_delete_test.go

@@ -6,14 +6,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/portainer/libhelm/binary/test"
-	"github.com/portainer/libhelm/options"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
+	"github.com/portainer/portainer/pkg/libhelm/binary/test"
+	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/stretchr/testify/assert"
 
 	helper "github.com/portainer/portainer/api/internal/testhelpers"