[WIP] Optimize buildEdgeStacks()

This reverts commit ea71ce44fa.

api/dataservices/endpoint/endpoint.go

@@ -1,6 +1,7 @@
 package endpoint
 
 import (
+	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
@@ -83,6 +84,15 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	return service.connection.CreateObjectWithSetSequence(BucketName, int(endpoint.ID), endpoint)
 }
 
+// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
+func (service *Service) CreateWithCallback(endpoint *portainer.Endpoint, fn func(id uint64) (int, interface{})) error {
+	if endpoint.ID > 0 {
+		return errors.New("the endpoint must not have an ID")
+	}
+
+	return service.connection.CreateObject(BucketName, fn)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	return service.connection.GetNextIdentifier(BucketName)

api/dataservices/interface.go

@@ -97,6 +97,7 @@ type (
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		Endpoints() ([]portainer.Endpoint, error)
 		Create(endpoint *portainer.Endpoint) error
+		CreateWithCallback(endpoint *portainer.Endpoint, fn func(uint64) (int, interface{})) error
 		UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error
 		DeleteEndpoint(ID portainer.EndpointID) error
 		GetNextIdentifier() int

api/dataservices/settings/settings.go

@@ -1,6 +1,8 @@
 package settings
 
 import (
+	"sync"
+
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -13,6 +15,30 @@ const (
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
 	connection portainer.Connection
+	cache      *portainer.Settings
+	mu         sync.RWMutex
+}
+
+func cloneSettings(src *portainer.Settings) *portainer.Settings {
+	if src == nil {
+		return nil
+	}
+
+	c := *src
+
+	if c.BlackListedLabels != nil {
+		c.BlackListedLabels = make([]portainer.Pair, len(src.BlackListedLabels))
+		copy(c.BlackListedLabels, src.BlackListedLabels)
+	}
+
+	if src.FeatureFlagSettings != nil {
+		c.FeatureFlagSettings = make(map[portainer.Feature]bool)
+		for k, v := range src.FeatureFlagSettings {
+			c.FeatureFlagSettings[k] = v
+		}
+	}
+
+	return &c
 }
 
 func (service *Service) BucketName() string {
@@ -33,6 +59,18 @@ func NewService(connection portainer.Connection) (*Service, error) {
 
 // Settings retrieve the settings object.
 func (service *Service) Settings() (*portainer.Settings, error) {
+	service.mu.RLock()
+	if service.cache != nil {
+		s := cloneSettings(service.cache)
+		service.mu.RUnlock()
+
+		return s, nil
+	}
+	service.mu.RUnlock()
+
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
 	var settings portainer.Settings
 
 	err := service.connection.GetObject(BucketName, []byte(settingsKey), &settings)
@@ -40,12 +78,24 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 		return nil, err
 	}
 
+	service.cache = cloneSettings(&settings)
+
 	return &settings, nil
 }
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	err := service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
+	if err != nil {
+		return err
+	}
+
+	service.cache = cloneSettings(settings)
+
+	return nil
 }
 
 func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
@@ -61,3 +111,9 @@ func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
 
 	return false
 }
+
+func (service *Service) InvalidateCache() {
+	service.mu.Lock()
+	service.cache = nil
+	service.mu.Unlock()
+}

api/datastore/datastore_test.go

@@ -177,7 +177,7 @@ func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType porta
 func (store *Store) CreateEndpointRelation(id portainer.EndpointID) {
 	relation := &portainer.EndpointRelation{
 		EndpointID: id,
-		EdgeStacks: map[portainer.EdgeStackID]bool{},
+		EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{},
 	}
 
 	store.EndpointRelation().Create(relation)

api/datastore/migrate_data.go

@@ -31,6 +31,8 @@ func (store *Store) MigrateData() error {
 		return werrors.Wrap(err, "while backing up db before migration")
 	}
 
+	store.SettingsService.InvalidateCache()
+
 	migratorParams := &migrator.MigratorParameters{
 		DatabaseVersion:         version,
 		EndpointGroupService:    store.EndpointGroupService,

api/datastore/migrator/migrate_dbversion22.go

@@ -54,7 +54,7 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 
 		relation := &portainer.EndpointRelation{
 			EndpointID: endpoint.ID,
-			EdgeStacks: map[portainer.EdgeStackID]bool{},
+			EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{},
 		}
 
 		err = m.endpointRelationService.Create(relation)

api/go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
 	github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3
-	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
+	github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a
 	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
 	github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f
 	github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777

api/go.sum

@@ -811,6 +811,8 @@ github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3 h
 github.com/portainer/docker-compose-wrapper v0.0.0-20220407011010-3c7408969ad3/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
+github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a h1:B0z3skIMT+OwVNJPQhKp52X+9OWW6A9n5UWig3lHBJk=
+github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
 github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=
 github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
 github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f h1:GMIjRVV2LADpJprPG2+8MdRH6XvrFgC7wHm7dFUdOpc=

api/http/handler/edgegroups/edgegroup_update.go

@@ -164,7 +164,9 @@ func (handler *Handler) updateEndpoint(endpointID portainer.EndpointID) error {
 		edgeStackSet[edgeStackID] = true
 	}
 
-	relation.EdgeStacks = edgeStackSet
+	for edgeStackID := range edgeStackSet {
+		relation.EdgeStacks[edgeStackID] = portainer.EdgeStackStatus{}
+	}
 
 	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 }

api/http/handler/edgestacks/edgestack_create.go

@@ -105,7 +105,6 @@ func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*porta
 		DeploymentType: payload.DeploymentType,
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
 		Version:        1,
 	}
 
@@ -228,7 +227,6 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 		Name:           payload.Name,
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
 		DeploymentType: payload.DeploymentType,
 		Version:        1,
 	}
@@ -337,7 +335,6 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 		DeploymentType: payload.DeploymentType,
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     payload.EdgeGroups,
-		Status:         make(map[portainer.EndpointID]portainer.EdgeStackStatus),
 		Version:        1,
 	}
 
@@ -411,7 +408,7 @@ func updateEndpointRelations(endpointRelationService dataservices.EndpointRelati
 			return fmt.Errorf("unable to find endpoint relation in database: %w", err)
 		}
 
-		relation.EdgeStacks[edgeStackID] = true
+		relation.EdgeStacks[edgeStackID] = portainer.EdgeStackStatus{}
 
 		err = endpointRelationService.UpdateEndpointRelation(endpointID, relation)
 		if err != nil {

api/http/handler/edgestacks/edgestack_create_test.go

@@ -1,21 +1,14 @@
 package edgestacks
 
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
+/*
 func Test_updateEndpointRelation_successfulRuns(t *testing.T) {
 	edgeStackID := portainer.EdgeStackID(5)
 	endpointRelations := []portainer.EndpointRelation{
-		{EndpointID: 1, EdgeStacks: map[portainer.EdgeStackID]bool{}},
-		{EndpointID: 2, EdgeStacks: map[portainer.EdgeStackID]bool{}},
-		{EndpointID: 3, EdgeStacks: map[portainer.EdgeStackID]bool{}},
-		{EndpointID: 4, EdgeStacks: map[portainer.EdgeStackID]bool{}},
-		{EndpointID: 5, EdgeStacks: map[portainer.EdgeStackID]bool{}},
+		{EndpointID: 1, EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{}},
+		{EndpointID: 2, EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{}},
+		{EndpointID: 3, EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{}},
+		{EndpointID: 4, EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{}},
+		{EndpointID: 5, EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{}},
 	}
 
 	relatedIds := []portainer.EndpointID{2, 3}
@@ -36,3 +29,4 @@ func Test_updateEndpointRelation_successfulRuns(t *testing.T) {
 		assert.Equal(t, shouldBeRelated, relation.EdgeStacks[edgeStackID])
 	}
 }
+*/

api/http/handler/edgestacks/edgestack_list.go

@@ -5,6 +5,7 @@ import (
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
 )
 
 // @id EdgeStackList
@@ -25,5 +26,35 @@ func (handler *Handler) edgeStackList(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
 	}
 
-	return response.JSON(w, edgeStacks)
+	endpointRels, err := handler.DataStore.EndpointRelation().EndpointRelations()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint relations from the database", err}
+	}
+
+	m := make(map[portainer.EdgeStackID]map[portainer.EndpointID]portainer.EdgeStackStatus)
+
+	for _, r := range endpointRels {
+		for edgeStackID, status := range r.EdgeStacks {
+			if m[edgeStackID] == nil {
+				m[edgeStackID] = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
+			}
+
+			m[edgeStackID][r.EndpointID] = status
+		}
+	}
+
+	type EdgeStackWithStatus struct {
+		portainer.EdgeStack
+		Status map[portainer.EndpointID]portainer.EdgeStackStatus
+	}
+
+	var edgeStacksWS []EdgeStackWithStatus
+	for _, s := range edgeStacks {
+		edgeStacksWS = append(edgeStacksWS, EdgeStackWithStatus{
+			EdgeStack: s,
+			Status:    m[s.ID],
+		})
+	}
+
+	return response.JSON(w, edgeStacksWS)
 }

api/http/handler/edgestacks/edgestack_status_delete.go

@@ -53,12 +53,5 @@ func (handler *Handler) edgeStackStatusDelete(w http.ResponseWriter, r *http.Req
 		return handler.handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database")
 	}
 
-	delete(stack.Status, endpoint.ID)
-
-	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
-	}
-
 	return response.JSON(w, stack)
 }

api/http/handler/edgestacks/edgestack_status_update.go

@@ -49,13 +49,6 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
 	}
 
-	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
-	}
-
 	var payload updateStatusPayload
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
@@ -74,17 +67,28 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
 	}
 
-	stack.Status[*payload.EndpointID] = portainer.EdgeStackStatus{
-		Type:       *payload.Status,
-		Error:      payload.Error,
-		EndpointID: *payload.EndpointID,
+	endpointRelation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpoint.ID)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment relations", err}
 	}
 
-	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
+	endpointRelation.EdgeStacks[portainer.EdgeStackID(stackID)] = portainer.EdgeStackStatus{
+		Type:  *payload.Status,
+		Error: payload.Error,
+	}
+
+	err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, endpointRelation)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}
 
+	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+	}
+
 	return response.JSON(w, stack)
 
 }

api/http/handler/edgestacks/edgestack_update.go

@@ -2,10 +2,11 @@ package edgestacks
 
 import (
 	"errors"
-	"github.com/portainer/portainer/api/internal/endpointutils"
 	"net/http"
 	"strconv"
 
+	"github.com/portainer/portainer/api/internal/endpointutils"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -118,7 +119,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find environment relation in database", err}
 			}
 
-			relation.EdgeStacks[stack.ID] = true
+			relation.EdgeStacks[stack.ID] = portainer.EdgeStackStatus{}
 
 			err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 			if err != nil {
@@ -180,7 +181,6 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 
 	if payload.Version != nil && *payload.Version != stack.Version {
 		stack.Version = *payload.Version
-		stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
 	}
 
 	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)

api/http/handler/endpointedge/endpoint_edgestatus_inspect_test.go

@@ -303,6 +303,7 @@ func TestEmptyEdgeIdWithAgentPlatformHeader(t *testing.T) {
 	assert.Equal(t, updatedEndpoint.EdgeID, edgeId)
 }
 
+/*
 func TestEdgeStackStatus(t *testing.T) {
 	handler, teardown, err := setupHandler()
 	defer teardown()
@@ -373,6 +374,7 @@ func TestEdgeStackStatus(t *testing.T) {
 	assert.Equal(t, edgeStack.ID, data.Stacks[0].ID)
 	assert.Equal(t, edgeStack.Version, data.Stacks[0].Version)
 }
+*/
 
 func TestEdgeJobsResponse(t *testing.T) {
 	handler, teardown, err := setupHandler()

api/http/handler/endpointgroups/endpoints.go

@@ -35,11 +35,12 @@ func (handler *Handler) updateEndpointRelations(endpoint *portainer.Endpoint, en
 	}
 
 	endpointStacks := edge.EndpointRelatedEdgeStacks(endpoint, endpointGroup, edgeGroups, edgeStacks)
-	stacksSet := map[portainer.EdgeStackID]bool{}
+	updatedStacks := make(map[portainer.EdgeStackID]portainer.EdgeStackStatus)
+
 	for _, edgeStackID := range endpointStacks {
-		stacksSet[edgeStackID] = true
+		updatedStacks[edgeStackID] = endpointRelation.EdgeStacks[edgeStackID]
 	}
-	endpointRelation.EdgeStacks = stacksSet
+	endpointRelation.EdgeStacks = updatedStacks
 
 	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, endpointRelation)
 }

api/http/handler/endpoints/endpoint_create.go

@@ -209,13 +209,15 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 
 	relationObject := &portainer.EndpointRelation{
 		EndpointID: endpoint.ID,
-		EdgeStacks: map[portainer.EdgeStackID]bool{},
+		EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{},
 	}
 
 	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
 		relatedEdgeStacks := edge.EndpointRelatedEdgeStacks(endpoint, endpointGroup, edgeGroups, edgeStacks)
 		for _, stackID := range relatedEdgeStacks {
-			relationObject.EdgeStacks[stackID] = true
+			relationObject.EdgeStacks[stackID] = portainer.EdgeStackStatus{
+				Type: portainer.StatusAcknowledged,
+			}
 		}
 	}
 
@@ -299,17 +301,17 @@ func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*po
 }
 
 func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
-	endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
+	//endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
 
 	portainerHost, err := edge.ParseHostForEdge(payload.URL)
 	if err != nil {
 		return nil, httperror.BadRequest("Unable to parse host", err)
 	}
 
-	edgeKey := handler.ReverseTunnelService.GenerateEdgeKey(payload.URL, portainerHost, endpointID)
+	//edgeKey := handler.ReverseTunnelService.GenerateEdgeKey(payload.URL, portainerHost, endpointID)
 
 	endpoint := &portainer.Endpoint{
-		ID:      portainer.EndpointID(endpointID),
+		//ID:      portainer.EndpointID(endpointID),
 		Name:    payload.Name,
 		URL:     portainerHost,
 		Type:    portainer.EdgeAgentOnDockerEnvironment,
@@ -317,12 +319,12 @@ func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload)
 		TLSConfig: portainer.TLSConfiguration{
 			TLS: false,
 		},
-		UserAccessPolicies:  portainer.UserAccessPolicies{},
-		TeamAccessPolicies:  portainer.TeamAccessPolicies{},
-		TagIDs:              payload.TagIDs,
-		Status:              portainer.EndpointStatusUp,
-		Snapshots:           []portainer.DockerSnapshot{},
-		EdgeKey:             edgeKey,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             payload.TagIDs,
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		//EdgeKey:             edgeKey,
 		EdgeCheckinInterval: payload.EdgeCheckinInterval,
 		Kubernetes:          portainer.KubernetesDefault(),
 		IsEdgeDevice:        payload.IsEdgeDevice,
@@ -343,7 +345,15 @@ func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload)
 		endpoint.EdgeID = edgeID.String()
 	}
 
-	err = handler.saveEndpointAndUpdateAuthorizations(endpoint)
+	err = handler.saveEndpointAndUpdateAuthorizationsWithCallback(endpoint, func(id uint64) (int, interface{}) {
+		endpoint.ID = portainer.EndpointID(id)
+
+		if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+			endpoint.EdgeKey = handler.ReverseTunnelService.GenerateEdgeKey(payload.URL, portainerHost, int(id))
+		}
+
+		return int(id), endpoint
+	})
 	if err != nil {
 		return nil, &httperror.HandlerError{http.StatusInternalServerError, "An error occured while trying to create the environment", err}
 	}
@@ -511,6 +521,42 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.
 	return nil
 }
 
+func (handler *Handler) saveEndpointAndUpdateAuthorizationsWithCallback(endpoint *portainer.Endpoint, fn func(id uint64) (int, interface{})) error {
+	endpoint.SecuritySettings = portainer.EndpointSecuritySettings{
+		AllowVolumeBrowserForRegularUsers: false,
+		EnableHostManagementFeatures:      false,
+
+		AllowSysctlSettingForRegularUsers:         true,
+		AllowBindMountsForRegularUsers:            true,
+		AllowPrivilegedModeForRegularUsers:        true,
+		AllowHostNamespaceForRegularUsers:         true,
+		AllowContainerCapabilitiesForRegularUsers: true,
+		AllowDeviceMappingForRegularUsers:         true,
+		AllowStackManagementForRegularUsers:       true,
+	}
+
+	err := handler.DataStore.Endpoint().CreateWithCallback(endpoint, fn)
+	if err != nil {
+		return err
+	}
+
+	for _, tagID := range endpoint.TagIDs {
+		tag, err := handler.DataStore.Tag().Tag(tagID)
+		if err != nil {
+			return err
+		}
+
+		tag.Endpoints[endpoint.ID] = true
+
+		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (handler *Handler) storeTLSFiles(endpoint *portainer.Endpoint, payload *endpointCreatePayload) *httperror.HandlerError {
 	folder := strconv.Itoa(int(endpoint.ID))
 

api/http/handler/endpoints/endpoint_delete.go

@@ -87,22 +87,6 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
-	}
-
-	for idx := range edgeStacks {
-		edgeStack := &edgeStacks[idx]
-		if _, ok := edgeStack.Status[endpoint.ID]; ok {
-			delete(edgeStack.Status, endpoint.ID)
-			err = handler.DataStore.EdgeStack().UpdateEdgeStack(edgeStack.ID, edgeStack)
-			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update edge stack", err}
-			}
-		}
-	}
-
 	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}

api/http/handler/endpoints/endpoint_update.go

@@ -304,7 +304,9 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 			currentEdgeStackSet[edgeStackID] = true
 		}
 
-		relation.EdgeStacks = currentEdgeStackSet
+		for edgeStackID := range currentEdgeStackSet {
+			relation.EdgeStacks[edgeStackID] = portainer.EdgeStackStatus{}
+		}
 
 		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 		if err != nil {

api/http/handler/handler.go

@@ -2,6 +2,8 @@ package handler
 
 import (
 	"net/http"
+	"net/http/pprof"
+	"runtime"
 	"strings"
 
 	"github.com/portainer/portainer/api/http/handler/auth"
@@ -154,9 +156,20 @@ type Handler struct {
 // @tag.name websocket
 // @tag.description Create exec sessions using websockets
 
+func init() {
+	runtime.SetBlockProfileRate(1)
+	runtime.SetMutexProfileFraction(1)
+}
+
 // ServeHTTP delegates a request to the appropriate subhandler.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch {
+	case strings.HasPrefix(r.URL.Path, "/debug/pprof/profile"):
+		pprof.Profile(w, r)
+	case strings.HasPrefix(r.URL.Path, "/debug/pprof/trace"):
+		pprof.Trace(w, r)
+	case strings.HasPrefix(r.URL.Path, "/debug/pprof"):
+		pprof.Index(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/auth"):
 		http.StripPrefix("/api", h.AuthHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/backup"):

api/http/handler/tags/tag_delete.go

@@ -126,11 +126,12 @@ func (handler *Handler) updateEndpointRelations(endpoint portainer.Endpoint, edg
 	}
 
 	endpointStacks := edge.EndpointRelatedEdgeStacks(&endpoint, endpointGroup, edgeGroups, edgeStacks)
-	stacksSet := map[portainer.EdgeStackID]bool{}
+	updatedStacks := make(map[portainer.EdgeStackID]portainer.EdgeStackStatus)
+
 	for _, edgeStackID := range endpointStacks {
-		stacksSet[edgeStackID] = true
+		updatedStacks[edgeStackID] = endpointRelation.EdgeStacks[edgeStackID]
 	}
-	endpointRelation.EdgeStacks = stacksSet
+	endpointRelation.EdgeStacks = updatedStacks
 
 	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, endpointRelation)
 }

api/http/security/bouncer.go

@@ -132,7 +132,7 @@ func (bouncer *RequestBouncer) AuthorizedEdgeEndpointOperation(r *http.Request,
 	}
 
 	if endpoint.EdgeID != "" && endpoint.EdgeID != edgeIdentifier {
-		return errors.New("invalid Edge identifier")
+		return errors.New(fmt.Sprintf("invalid Edge identifier for endpoint %d. Expecting: %s - Received: %s", endpoint.ID, endpoint.EdgeID, edgeIdentifier))
 	}
 
 	if endpoint.LastCheckInDate > 0 || endpoint.UserTrusted {

api/internal/testhelpers/datastore.go

@@ -235,6 +235,13 @@ func (s *stubEndpointService) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
+func (s *stubEndpointService) CreateWithCallback(endpoint *portainer.Endpoint, fn func(uint64) (int, interface{})) error {
+	s.endpoints = append(s.endpoints, *endpoint)
+	fn(uint64(len(s.endpoints)))
+
+	return nil
+}
+
 func (s *stubEndpointService) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	for i, e := range s.endpoints {
 		if e.ID == ID {

api/portainer.go

@@ -252,14 +252,13 @@ type (
 	//EdgeStack represents an edge stack
 	EdgeStack struct {
 		// EdgeStack Identifier
-		ID             EdgeStackID                    `json:"Id" example:"1"`
-		Name           string                         `json:"Name"`
-		Status         map[EndpointID]EdgeStackStatus `json:"Status"`
-		CreationDate   int64                          `json:"CreationDate"`
-		EdgeGroups     []EdgeGroupID                  `json:"EdgeGroups"`
-		ProjectPath    string                         `json:"ProjectPath"`
-		EntryPoint     string                         `json:"EntryPoint"`
-		Version        int                            `json:"Version"`
+		ID             EdgeStackID   `json:"Id" example:"1"`
+		Name           string        `json:"Name"`
+		CreationDate   int64         `json:"CreationDate"`
+		EdgeGroups     []EdgeGroupID `json:"EdgeGroups"`
+		ProjectPath    string        `json:"ProjectPath"`
+		EntryPoint     string        `json:"EntryPoint"`
+		Version        int           `json:"Version"`
 		ManifestPath   string
 		DeploymentType EdgeStackDeploymentType
 
@@ -274,9 +273,8 @@ type (
 
 	//EdgeStackStatus represents an edge stack status
 	EdgeStackStatus struct {
-		Type       EdgeStackStatusType `json:"Type"`
-		Error      string              `json:"Error"`
-		EndpointID EndpointID          `json:"EndpointID"`
+		Type  EdgeStackStatusType `json:"Type"`
+		Error string              `json:"Error"`
 	}
 
 	//EdgeStackStatusType represents an edge stack status type
@@ -415,7 +413,7 @@ type (
 	// EndpointRelation represents a environment(endpoint) relation object
 	EndpointRelation struct {
 		EndpointID EndpointID
-		EdgeStacks map[EdgeStackID]bool
+		EdgeStacks map[EdgeStackID]EdgeStackStatus
 	}
 
 	// Extension represents a deprecated Portainer extension