fix(settings): return the right value for public

fix [EE-6923]

.eslintrc.yml

@@ -10,6 +10,7 @@ globals:
 extends:
   - 'eslint:recommended'
   - 'plugin:storybook/recommended'
+  - 'plugin:import/typescript'
   - prettier
 
 plugins:
@@ -29,6 +30,7 @@ rules:
   no-empty: warn
   no-empty-function: warn
   no-useless-escape: 'off'
+  import/named: error
   import/order:
     [
       'error',
@@ -43,6 +45,12 @@ rules:
         pathGroupsExcludedImportTypes: ['internal'],
       },
     ]
+  no-restricted-imports:
+    - error
+    - patterns:
+        - group:
+            - '@/react/test-utils/*'
+          message: 'These utils are just for test files'
 
 settings:
   'import/resolver':
@@ -51,6 +59,8 @@ settings:
         - ['@@', './app/react/components']
         - ['@', './app']
       extensions: ['.js', '.ts', '.tsx']
+    typescript: true
+    node: true
 
 overrides:
   - files:
@@ -75,7 +85,9 @@ overrides:
     settings:
       react:
         version: 'detect'
+
     rules:
+      no-console: error
       import/order:
         [
           'error',
@@ -108,6 +120,12 @@ overrides:
       'no-await-in-loop': 'off'
       'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
       'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
+      '@typescript-eslint/no-restricted-imports':
+        - error
+        - patterns:
+            - group:
+                - '@/react/test-utils/*'
+              message: 'These utils are just for test files'
     overrides: # allow props spreading for hoc files
       - files:
           - app/**/with*.ts{,x}
@@ -121,7 +139,13 @@ overrides:
       'vitest/env': true
     rules:
       'react/jsx-no-constructed-context-values': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
+      'react/jsx-props-no-spreading': off
   - files:
       - app/**/*.stories.*
     rules:
       'no-alert': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
+      'react/jsx-props-no-spreading': off

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -93,6 +93,11 @@ body:
       description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
       multiple: false
       options:
+        - '2.20.3'
+        - '2.20.2'
+        - '2.20.1'
+        - '2.20.0'
+        - '2.19.5'
         - '2.19.4'
         - '2.19.3'
         - '2.19.2'

.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ on:
 env:
   DOCKER_HUB_REPO: portainerci/portainer-ce
   EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/nightly-security-scan.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
 
 jobs:
   client-dependencies:

.github/workflows/pr-security.yml

@@ -14,7 +14,7 @@ on:
       - '.github/workflows/pr-security.yml'
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/test.yaml

@@ -1,17 +1,25 @@
 name: Test
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 on:
   pull_request:
+    branches:
+      - master
+      - develop
+      - release/*
     types:
       - opened
       - reopened
       - synchronize
       - ready_for_review
   push:
+    branches:
+      - master
+      - develop
+      - release/*
 
 jobs:
   test-client:

.github/workflows/validate-openapi-spec.yaml

@@ -13,7 +13,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.storybook/preview.tsx

@@ -3,12 +3,11 @@ import React from 'react';
 import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
 import { initialize as initMSW, mswLoader } from 'msw-storybook-addon';
 import { handlers } from '../app/setup-tests/server-handlers';
-import { QueryClient, QueryClientProvider } from 'react-query';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 
 initMSW(
   {
     onUnhandledRequest: ({ method, url }) => {
-      console.log(method, url);
       if (url.startsWith('/api')) {
         console.error(`Unhandled ${method} request to ${url}.
 

api/backup/backup.go

@@ -82,7 +82,8 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 }
 
 func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
-	_, err := datastore.Backup(filepath.Join(backupDirPath, "portainer.db"))
+	dbFileName := datastore.Connection().GetDatabaseFileName()
+	_, err := datastore.Backup(filepath.Join(backupDirPath, dbFileName))
 	return err
 }
 

api/backup/restore.go

@@ -26,7 +26,7 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
 	if password != "" {
 		archive, err = decrypt(archive, password)
 		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
+			return errors.Wrap(err, "failed to decrypt the archive. Please ensure the password is correct and try again")
 		}
 	}
 

api/chisel/schedules.go

@@ -5,6 +5,17 @@ import (
 	"github.com/portainer/portainer/api/internal/edge/cache"
 )
 
+// EdgeJobs retrieves the edge jobs for the given environment
+func (service *Service) EdgeJobs(endpointID portainer.EndpointID) []portainer.EdgeJob {
+	service.mu.RLock()
+	defer service.mu.RUnlock()
+
+	return append(
+		make([]portainer.EdgeJob, 0, len(service.edgeJobs[endpointID])),
+		service.edgeJobs[endpointID]...,
+	)
+}
+
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
 func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portainer.EdgeJob) {
 	if endpoint.Edge.AsyncMode {
@@ -12,10 +23,10 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	}
 
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpoint.ID)
+	defer service.mu.Unlock()
 
 	existingJobIndex := -1
-	for idx, existingJob := range tunnel.Jobs {
+	for idx, existingJob := range service.edgeJobs[endpoint.ID] {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
 
@@ -24,30 +35,28 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	}
 
 	if existingJobIndex == -1 {
-		tunnel.Jobs = append(tunnel.Jobs, *edgeJob)
+		service.edgeJobs[endpoint.ID] = append(service.edgeJobs[endpoint.ID], *edgeJob)
 	} else {
-		tunnel.Jobs[existingJobIndex] = *edgeJob
+		service.edgeJobs[endpoint.ID][existingJobIndex] = *edgeJob
 	}
 
 	cache.Del(endpoint.ID)
-
-	service.mu.Unlock()
 }
 
 // RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
 func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
 
-	for endpointID, tunnel := range service.tunnelDetailsMap {
+	for endpointID := range service.edgeJobs {
 		n := 0
-		for _, edgeJob := range tunnel.Jobs {
+		for _, edgeJob := range service.edgeJobs[endpointID] {
 			if edgeJob.ID != edgeJobID {
-				tunnel.Jobs[n] = edgeJob
+				service.edgeJobs[endpointID][n] = edgeJob
 				n++
 			}
 		}
 
-		tunnel.Jobs = tunnel.Jobs[:n]
+		service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 		cache.Del(endpointID)
 	}
@@ -57,19 +66,17 @@ func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 
 func (service *Service) RemoveEdgeJobFromEndpoint(endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
+	defer service.mu.Unlock()
 
 	n := 0
-	for _, edgeJob := range tunnel.Jobs {
+	for _, edgeJob := range service.edgeJobs[endpointID] {
 		if edgeJob.ID != edgeJobID {
-			tunnel.Jobs[n] = edgeJob
+			service.edgeJobs[endpointID][n] = edgeJob
 			n++
 		}
 	}
 
-	tunnel.Jobs = tunnel.Jobs[:n]
+	service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 	cache.Del(endpointID)
-
-	service.mu.Unlock()
 }

api/chisel/service.go

@@ -19,7 +19,6 @@ import (
 
 const (
 	tunnelCleanupInterval = 10 * time.Second
-	requiredTimeout       = 15 * time.Second
 	activeTimeout         = 4*time.Minute + 30*time.Second
 	pingTimeout           = 3 * time.Second
 )
@@ -28,32 +27,54 @@ const (
 // It is used to start a reverse tunnel server and to manage the connection status of each tunnel
 // connected to the tunnel server.
 type Service struct {
-	serverFingerprint string
-	serverPort        string
-	tunnelDetailsMap  map[portainer.EndpointID]*portainer.TunnelDetails
-	dataStore         dataservices.DataStore
-	snapshotService   portainer.SnapshotService
-	chiselServer      *chserver.Server
-	shutdownCtx       context.Context
-	ProxyManager      *proxy.Manager
-	mu                sync.Mutex
-	fileService       portainer.FileService
+	serverFingerprint      string
+	serverPort             string
+	activeTunnels          map[portainer.EndpointID]*portainer.TunnelDetails
+	edgeJobs               map[portainer.EndpointID][]portainer.EdgeJob
+	dataStore              dataservices.DataStore
+	snapshotService        portainer.SnapshotService
+	chiselServer           *chserver.Server
+	shutdownCtx            context.Context
+	ProxyManager           *proxy.Manager
+	mu                     sync.RWMutex
+	fileService            portainer.FileService
+	defaultCheckinInterval int
 }
 
 // NewService returns a pointer to a new instance of Service
 func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
+	defaultCheckinInterval := portainer.DefaultEdgeAgentCheckinIntervalInSeconds
+
+	settings, err := dataStore.Settings().Settings()
+	if err == nil {
+		defaultCheckinInterval = settings.EdgeAgentCheckinInterval
+	} else {
+		log.Error().Err(err).Msg("unable to retrieve the settings from the database")
+	}
+
 	return &Service{
-		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
-		dataStore:        dataStore,
-		shutdownCtx:      shutdownCtx,
-		fileService:      fileService,
+		activeTunnels:          make(map[portainer.EndpointID]*portainer.TunnelDetails),
+		edgeJobs:               make(map[portainer.EndpointID][]portainer.EdgeJob),
+		dataStore:              dataStore,
+		shutdownCtx:            shutdownCtx,
+		fileService:            fileService,
+		defaultCheckinInterval: defaultCheckinInterval,
 	}
 }
 
 // pingAgent ping the given agent so that the agent can keep the tunnel alive
 func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
-	tunnel := service.GetTunnelDetails(endpointID)
-	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
+	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
+	if err != nil {
+		return err
+	}
+
+	tunnelAddr, err := service.TunnelAddr(endpoint)
+	if err != nil {
+		return err
+	}
+
+	requestURL := fmt.Sprintf("http://%s/ping", tunnelAddr)
 	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
 	if err != nil {
 		return err
@@ -76,47 +97,49 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
 func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
-	go func() {
-		log.Debug().
-			Int("endpoint_id", int(endpointID)).
-			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("KeepTunnelAlive: start")
+	go service.keepTunnelAlive(endpointID, ctx, maxAlive)
+}
 
-		maxAliveTicker := time.NewTicker(maxAlive)
-		defer maxAliveTicker.Stop()
+func (service *Service) keepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
+	log.Debug().
+		Int("endpoint_id", int(endpointID)).
+		Float64("max_alive_minutes", maxAlive.Minutes()).
+		Msg("KeepTunnelAlive: start")
 
-		pingTicker := time.NewTicker(tunnelCleanupInterval)
-		defer pingTicker.Stop()
+	maxAliveTicker := time.NewTicker(maxAlive)
+	defer maxAliveTicker.Stop()
 
-		for {
-			select {
-			case <-pingTicker.C:
-				service.SetTunnelStatusToActive(endpointID)
-				err := service.pingAgent(endpointID)
-				if err != nil {
-					log.Debug().
-						Int("endpoint_id", int(endpointID)).
-						Err(err).
-						Msg("KeepTunnelAlive: ping agent")
-				}
-			case <-maxAliveTicker.C:
-				log.Debug().
-					Int("endpoint_id", int(endpointID)).
-					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("KeepTunnelAlive: tunnel keep alive timeout")
+	pingTicker := time.NewTicker(tunnelCleanupInterval)
+	defer pingTicker.Stop()
 
-				return
-			case <-ctx.Done():
-				err := ctx.Err()
+	for {
+		select {
+		case <-pingTicker.C:
+			service.UpdateLastActivity(endpointID)
+
+			if err := service.pingAgent(endpointID); err != nil {
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("KeepTunnelAlive: tunnel stop")
-
-				return
+					Msg("KeepTunnelAlive: ping agent")
 			}
+		case <-maxAliveTicker.C:
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Float64("timeout_minutes", maxAlive.Minutes()).
+				Msg("KeepTunnelAlive: tunnel keep alive timeout")
+
+			return
+		case <-ctx.Done():
+			err := ctx.Err()
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("KeepTunnelAlive: tunnel stop")
+
+			return
 		}
-	}()
+	}
 }
 
 // StartTunnelServer starts a tunnel server on the specified addr and port.
@@ -126,7 +149,6 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
 	privateKeyFile, err := service.retrievePrivateKeyFile()
-
 	if err != nil {
 		return err
 	}
@@ -144,21 +166,21 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	service.serverFingerprint = chiselServer.GetFingerprint()
 	service.serverPort = port
 
-	err = chiselServer.Start(addr, port)
-	if err != nil {
+	if err := chiselServer.Start(addr, port); err != nil {
 		return err
 	}
+
 	service.chiselServer = chiselServer
 
 	// TODO: work-around Chisel default behavior.
 	// By default, Chisel will allow anyone to connect if no user exists.
 	username, password := generateRandomCredentials()
-	err = service.chiselServer.AddUser(username, password, "127.0.0.1")
-	if err != nil {
+	if err = service.chiselServer.AddUser(username, password, "127.0.0.1"); err != nil {
 		return err
 	}
 
 	service.snapshotService = snapshotService
+
 	go service.startTunnelVerificationLoop()
 
 	return nil
@@ -172,37 +194,39 @@ func (service *Service) StopTunnelServer() error {
 func (service *Service) retrievePrivateKeyFile() (string, error) {
 	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
 
-	exist, _ := service.fileService.FileExists(privateKeyFile)
-	if !exist {
-		log.Debug().
-			Str("private-key", privateKeyFile).
-			Msg("Chisel private key file does not exist")
-
-		privateKey, err := ccrypto.GenerateKey("")
-		if err != nil {
-			log.Error().
-				Err(err).
-				Msg("Failed to generate chisel private key")
-			return "", err
-		}
-
-		err = service.fileService.StoreChiselPrivateKey(privateKey)
-		if err != nil {
-			log.Error().
-				Err(err).
-				Msg("Failed to save Chisel private key to disk")
-			return "", err
-		} else {
-			log.Info().
-				Str("private-key", privateKeyFile).
-				Msg("Generated a new Chisel private key file")
-		}
-	} else {
+	if exists, _ := service.fileService.FileExists(privateKeyFile); exists {
 		log.Info().
 			Str("private-key", privateKeyFile).
-			Msg("Found Chisel private key file on disk")
+			Msg("found Chisel private key file on disk")
+
+		return privateKeyFile, nil
 	}
 
+	log.Debug().
+		Str("private-key", privateKeyFile).
+		Msg("chisel private key file does not exist")
+
+	privateKey, err := ccrypto.GenerateKey("")
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to generate chisel private key")
+
+		return "", err
+	}
+
+	if err = service.fileService.StoreChiselPrivateKey(privateKey); err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to save Chisel private key to disk")
+
+		return "", err
+	}
+
+	log.Info().
+		Str("private-key", privateKeyFile).
+		Msg("generated a new Chisel private key file")
+
 	return privateKeyFile, nil
 }
 
@@ -230,63 +254,45 @@ func (service *Service) startTunnelVerificationLoop() {
 	}
 }
 
+// checkTunnels finds the first tunnel that has not had any activity recently
+// and attempts to take a snapshot, then closes it and returns
 func (service *Service) checkTunnels() {
-	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
+	service.mu.RLock()
 
-	service.mu.Lock()
-	for key, tunnel := range service.tunnelDetailsMap {
-		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
-			continue
-		}
-
-		tunnels[key] = *tunnel
-	}
-	service.mu.Unlock()
-
-	for endpointID, tunnel := range tunnels {
+	for endpointID, tunnel := range service.activeTunnels {
 		elapsed := time.Since(tunnel.LastActivity)
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
-			Str("status", tunnel.Status).
-			Float64("status_time_seconds", elapsed.Seconds()).
+			Float64("last_activity_seconds", elapsed.Seconds()).
 			Msg("environment tunnel monitoring")
 
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", requiredTimeout.Seconds()).
-				Msg("REQUIRED state timeout exceeded")
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed < activeTimeout {
+			continue
 		}
 
-		if tunnel.Status == portainer.EdgeAgentActive && elapsed > activeTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", activeTimeout.Seconds()).
-				Msg("ACTIVE state timeout exceeded")
+		tunnelPort := tunnel.Port
 
-			err := service.snapshotEnvironment(endpointID, tunnel.Port)
-			if err != nil {
-				log.Error().
-					Int("endpoint_id", int(endpointID)).
-					Err(err).
-					Msg("unable to snapshot Edge environment")
-			}
+		service.mu.RUnlock()
+
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Float64("last_activity_seconds", elapsed.Seconds()).
+			Float64("timeout_seconds", activeTimeout.Seconds()).
+			Msg("last activity timeout exceeded")
+
+		if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
+			log.Error().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("unable to snapshot Edge environment")
 		}
 
-		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
+		service.close(portainer.EndpointID(endpointID))
+
+		return
 	}
+
+	service.mu.RUnlock()
 }
 
 func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {

api/chisel/service_test.go

@@ -1,20 +1,27 @@
 package chisel
 
 import (
+	"context"
 	"net"
 	"net/http"
 	"testing"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestPingAgentPanic(t *testing.T) {
-	endpointID := portainer.EndpointID(1)
+	endpoint := &portainer.Endpoint{
+		ID:   1,
+		Type: portainer.EdgeAgentOnDockerEnvironment,
+	}
 
-	s := NewService(nil, nil, nil)
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	s := NewService(store, nil, nil)
 
 	defer func() {
 		require.Nil(t, recover())
@@ -28,12 +35,17 @@ func TestPingAgentPanic(t *testing.T) {
 	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
 	require.NoError(t, err)
 
+	srv := &http.Server{Handler: mux}
+
+	errCh := make(chan error)
 	go func() {
-		require.NoError(t, http.Serve(ln, mux))
+		errCh <- srv.Serve(ln)
 	}()
 
-	s.getTunnelDetails(endpointID)
-	s.tunnelDetailsMap[endpointID].Port = ln.Addr().(*net.TCPAddr).Port
+	s.Open(endpoint)
+	s.activeTunnels[endpoint.ID].Port = ln.Addr().(*net.TCPAddr).Port
 
-	require.Error(t, s.pingAgent(endpointID))
+	require.Error(t, s.pingAgent(endpoint.ID))
+	require.NoError(t, srv.Shutdown(context.Background()))
+	require.ErrorIs(t, <-errCh, http.ErrServerClosed)
 }

api/chisel/tunnel.go

@@ -5,14 +5,18 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"strings"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/pkg/libcrypto"
 
 	"github.com/dchest/uniuri"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -20,18 +24,181 @@ const (
 	maxAvailablePort = 65535
 )
 
+// Open will mark the tunnel as REQUIRED so the agent opens it
+func (s *Service) Open(endpoint *portainer.Endpoint) error {
+	if !endpointutils.IsEdgeEndpoint(endpoint) {
+		return errors.New("cannot open a tunnel for non-edge environments")
+	}
+
+	if endpoint.Edge.AsyncMode {
+		return errors.New("cannot open a tunnel for async edge environments")
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.activeTunnels[endpoint.ID]; ok {
+		return nil
+	}
+
+	defer cache.Del(endpoint.ID)
+
+	tun := &portainer.TunnelDetails{
+		Status:       portainer.EdgeAgentManagementRequired,
+		Port:         s.getUnusedPort(),
+		LastActivity: time.Now(),
+	}
+
+	username, password := generateRandomCredentials()
+
+	if s.chiselServer != nil {
+		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tun.Port)
+
+		if err := s.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
+			return err
+		}
+	}
+
+	credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
+	if err != nil {
+		return err
+	}
+
+	tun.Credentials = credentials
+
+	s.activeTunnels[endpoint.ID] = tun
+
+	return nil
+}
+
+// close removes the tunnel from the map so the agent will close it
+func (s *Service) close(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tun, ok := s.activeTunnels[endpointID]
+	if !ok {
+		return
+	}
+
+	if len(tun.Credentials) > 0 && s.chiselServer != nil {
+		user, _, _ := strings.Cut(tun.Credentials, ":")
+		s.chiselServer.DeleteUser(user)
+	}
+
+	if s.ProxyManager != nil {
+		s.ProxyManager.DeleteEndpointProxy(endpointID)
+	}
+
+	delete(s.activeTunnels, endpointID)
+
+	cache.Del(endpointID)
+}
+
+// Config returns the tunnel details needed for the agent to connect
+func (s *Service) Config(endpointID portainer.EndpointID) portainer.TunnelDetails {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		return *tun
+	}
+
+	return portainer.TunnelDetails{Status: portainer.EdgeAgentIdle}
+}
+
+// TunnelAddr returns the address of the local tunnel, including the port, it
+// will block until the tunnel is ready
+func (s *Service) TunnelAddr(endpoint *portainer.Endpoint) (string, error) {
+	if err := s.Open(endpoint); err != nil {
+		return "", err
+	}
+
+	tun := s.Config(endpoint.ID)
+	checkinInterval := time.Duration(s.tryEffectiveCheckinInterval(endpoint)) * time.Second
+
+	for t0 := time.Now(); ; {
+		if time.Since(t0) > 2*checkinInterval {
+			s.close(endpoint.ID)
+
+			return "", errors.New("unable to open the tunnel")
+		}
+
+		// Check if the tunnel is established
+		conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: tun.Port})
+		if err != nil {
+			time.Sleep(checkinInterval / 100)
+
+			continue
+		}
+
+		conn.Close()
+
+		break
+	}
+
+	s.UpdateLastActivity(endpoint.ID)
+
+	return fmt.Sprintf("127.0.0.1:%d", tun.Port), nil
+}
+
+// tryEffectiveCheckinInterval avoids a potential deadlock by returning a
+// previous known value after a timeout
+func (s *Service) tryEffectiveCheckinInterval(endpoint *portainer.Endpoint) int {
+	ch := make(chan int, 1)
+
+	go func() {
+		ch <- edge.EffectiveCheckinInterval(s.dataStore, endpoint)
+	}()
+
+	select {
+	case <-time.After(50 * time.Millisecond):
+		s.mu.RLock()
+		defer s.mu.RUnlock()
+
+		return s.defaultCheckinInterval
+	case i := <-ch:
+		s.mu.Lock()
+		s.defaultCheckinInterval = i
+		s.mu.Unlock()
+
+		return i
+	}
+}
+
+// UpdateLastActivity sets the current timestamp to avoid the tunnel timeout
+func (s *Service) UpdateLastActivity(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		tun.LastActivity = time.Now()
+	}
+}
+
 // NOTE: it needs to be called with the lock acquired
 // getUnusedPort is used to generate an unused random port in the dynamic port range.
 // Dynamic ports (also called private ports) are 49152 to 65535.
 func (service *Service) getUnusedPort() int {
 	port := randomInt(minAvailablePort, maxAvailablePort)
 
-	for _, tunnel := range service.tunnelDetailsMap {
+	for _, tunnel := range service.activeTunnels {
 		if tunnel.Port == port {
 			return service.getUnusedPort()
 		}
 	}
 
+	conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
+	if err == nil {
+		conn.Close()
+
+		log.Debug().
+			Int("port", port).
+			Msg("selected port is in use, trying a different one")
+
+		return service.getUnusedPort()
+	}
+
 	return port
 }
 
@@ -39,152 +206,10 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
-// NOTE: it needs to be called with the lock acquired
-func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
-
-	if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
-		return tunnel
-	}
-
-	tunnel := &portainer.TunnelDetails{
-		Status: portainer.EdgeAgentIdle,
-	}
-
-	service.tunnelDetailsMap[endpointID] = tunnel
-
-	cache.Del(endpointID)
-
-	return tunnel
-}
-
-// GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
-func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	return *service.getTunnelDetails(endpointID)
-}
-
-// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
-	if endpoint.Edge.AsyncMode {
-		return portainer.TunnelDetails{}, errors.New("cannot open tunnel on async endpoint")
-	}
-
-	tunnel := service.GetTunnelDetails(endpoint.ID)
-
-	if tunnel.Status == portainer.EdgeAgentActive {
-		// update the LastActivity
-		service.SetTunnelStatusToActive(endpoint.ID)
-	}
-
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := service.dataStore.Settings().Settings()
-			if err != nil {
-				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
-	}
-
-	return service.GetTunnelDetails(endpoint.ID), nil
-}
-
-// SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to ACTIVE.
-func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentActive
-	tunnel.Credentials = ""
-	tunnel.LastActivity = time.Now()
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to IDLE.
-// It removes any existing credentials associated to the tunnel.
-func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentIdle
-	tunnel.Port = 0
-	tunnel.LastActivity = time.Now()
-
-	credentials := tunnel.Credentials
-	if credentials != "" {
-		tunnel.Credentials = ""
-
-		if service.chiselServer != nil {
-			service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
-		}
-	}
-
-	service.ProxyManager.DeleteEndpointProxy(endpointID)
-
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to REQUIRED.
-// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
-// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
-// Credentials are encrypted using the Edge ID associated to the environment(endpoint).
-func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
-	defer cache.Del(endpointID)
-
-	tunnel := service.getTunnelDetails(endpointID)
-
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	if tunnel.Port == 0 {
-		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
-		if err != nil {
-			return err
-		}
-
-		tunnel.Status = portainer.EdgeAgentManagementRequired
-		tunnel.Port = service.getUnusedPort()
-		tunnel.LastActivity = time.Now()
-
-		username, password := generateRandomCredentials()
-		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-
-		if service.chiselServer != nil {
-			err = service.chiselServer.AddUser(username, password, authorizedRemote)
-			if err != nil {
-				return err
-			}
-		}
-
-		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
-		if err != nil {
-			return err
-		}
-		tunnel.Credentials = credentials
-	}
-
-	return nil
-}
-
 func generateRandomCredentials() (string, string) {
 	username := uniuri.NewLen(8)
 	password := uniuri.NewLen(8)
+
 	return username, password
 }
 

api/cli/cli.go

@@ -34,7 +34,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		DemoEnvironment:           kingpin.Flag("demo", "Demo environment").Bool(),
 		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
 		FeatureFlags:              kingpin.Flag("feat", "List of feature flags").Strings(),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
@@ -62,7 +61,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
-		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()

api/cmd/portainer/log.go

@@ -42,6 +42,13 @@ func setLoggingMode(mode string) {
 			TimeFormat:    "2006/01/02 03:04PM",
 			FormatMessage: formatMessage,
 		})
+	case "NOCOLOR":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage,
+			NoColor:       true,
+		})
 	case "JSON":
 		log.Logger = log.Output(os.Stderr)
 	}

api/cmd/portainer/main.go

@@ -19,7 +19,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/datastore/migrator"
-	"github.com/portainer/portainer/api/demo"
+	"github.com/portainer/portainer/api/datastore/postinit"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/exec"
@@ -42,6 +42,8 @@ import (
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/pendingactions"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/pkg/featureflags"
@@ -457,19 +459,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	authorizationService := authorization.NewService(dataStore)
 	authorizationService.K8sClientFactory = kubernetesClientFactory
 
-	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
-
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed initializing snapshot service")
-	}
-	snapshotService.Start()
-
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
 
-	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
+	proxyManager := proxy.NewManager(kubernetesClientFactory)
 
 	reverseTunnelService.ProxyManager = proxyManager
 
@@ -489,6 +483,19 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
+	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory)
+	pendingActionsService.RegisterHandler(actions.CleanNAPWithOverridePolicies, handlers.NewHandlerCleanNAPWithOverridePolicies(authorizationService, dataStore))
+	pendingActionsService.RegisterHandler(actions.DeletePortainerK8sRegistrySecrets, handlers.NewHandlerDeleteRegistrySecrets(authorizationService, dataStore, kubernetesClientFactory))
+	pendingActionsService.RegisterHandler(actions.PostInitMigrateEnvironment, handlers.NewHandlerPostInitMigrateEnvironment(authorizationService, dataStore, kubernetesClientFactory, dockerClientFactory, *flags.Assets, kubernetesDeployer))
+
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
+	}
+	snapshotService.Start()
+
+	proxyManager.NewProxyFactory(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService)
+
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing helm package manager")
@@ -501,14 +508,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	applicationStatus := initStatus(instanceID)
 
-	demoService := demo.NewService()
-	if *flags.DemoEnvironment {
-		err := demoService.Init(dataStore, cryptoService)
-		if err != nil {
-			log.Fatal().Err(err).Msg("failed initializing demo environment")
-		}
-	}
-
 	// channel to control when the admin user is created
 	adminCreationDone := make(chan struct{}, 1)
 
@@ -578,10 +577,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	// but some more complex migrations require access to a kubernetes or docker
 	// client. Therefore we run a separate migration process just before
 	// starting the server.
-	postInitMigrator := datastore.NewPostInitMigrator(
+	postInitMigrator := postinit.NewPostInitMigrator(
 		kubernetesClientFactory,
 		dockerClientFactory,
 		dataStore,
+		*flags.Assets,
+		kubernetesDeployer,
 	)
 	if err := postInitMigrator.PostInitMigrate(); err != nil {
 		log.Fatal().Err(err).Msg("failure during post init migrations")
@@ -621,7 +622,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
 		StackDeployer:               stackDeployer,
-		DemoService:                 demoService,
 		UpgradeService:              upgradeService,
 		AdminCreationDone:           adminCreationDone,
 		PendingActionsService:       pendingActionsService,
@@ -650,6 +650,7 @@ func main() {
 			Msg("starting Portainer")
 
 		err := server.Start()
+
 		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/crypto/aes.go

@@ -1,52 +1,216 @@
 package crypto
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
 	"io"
 
+	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/scrypt"
 )
 
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
+const (
+	// AES GCM settings
+	aesGcmHeader    = "AES256-GCM" // The encrypted file header
+	aesGcmBlockSize = 1024 * 1024  // 1MB block for aes gcm
 
-var emptySalt []byte = make([]byte, 0)
+	// Argon2 settings
+	// Recommded settings lower memory hardware according to current OWASP recommendations
+	// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
+	// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
+	argon2MemoryCost = 12 * 1024
+	argon2TimeCost   = 3
+	argon2Threads    = 1
+	argon2KeyLength  = 32
+)
 
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
+// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
 func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	err := aesEncryptGCM(input, output, passphrase)
 	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
+		return fmt.Errorf("error encrypting file: %w", err)
 	}
 
 	return nil
 }
 
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
 func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Read file header to determine how it was encrypted
+	inputReader := bufio.NewReader(input)
+	header, err := inputReader.Peek(len(aesGcmHeader))
+	if err != nil {
+		return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
+	}
+
+	if string(header) == aesGcmHeader {
+		reader, err := aesDecryptGCM(inputReader, passphrase)
+		if err != nil {
+			return nil, fmt.Errorf("error decrypting file: %w", err)
+		}
+
+		return reader, nil
+	}
+
+	// Use the previous decryption routine which has no header (to support older archives)
+	reader, err := aesDecryptOFB(inputReader, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
+	}
+
+	return reader, nil
+}
+
+// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
+func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
+	// Derive key using argon2 with a random salt
+	salt := make([]byte, 16) // 16 bytes salt
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		return err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return err
+	}
+
+	// Generate nonce
+	nonce, err := NewRandomNonce(aesgcm.NonceSize())
+	if err != nil {
+		return err
+	}
+
+	// write the header
+	if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
+		return err
+	}
+
+	// Write nonce and salt to the output file
+	if _, err := output.Write(salt); err != nil {
+		return err
+	}
+	if _, err := output.Write(nonce.Value()); err != nil {
+		return err
+	}
+
+	// Buffer for reading plaintext blocks
+	buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
+	ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
+
+	// Encrypt plaintext in blocks
+	for {
+		n, err := io.ReadFull(input, buf)
+		if n == 0 {
+			break // end of plaintext input
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return err
+		}
+
+		// Seal encrypts the plaintext using the nonce returning the updated slice.
+		ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
+
+		_, err = output.Write(ciphertext)
+		if err != nil {
+			return err
+		}
+
+		nonce.Increment()
+	}
+
+	return nil
+}
+
+// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
+func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Reader & verify header
+	header := make([]byte, len(aesGcmHeader))
+	if _, err := io.ReadFull(input, header); err != nil {
+		return nil, err
+	}
+
+	if string(header) != aesGcmHeader {
+		return nil, fmt.Errorf("invalid header")
+	}
+
+	// Read salt
+	salt := make([]byte, 16) // Salt size
+	if _, err := io.ReadFull(input, salt); err != nil {
+		return nil, err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+
+	// Initialize AES cipher block
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create GCM mode with the cipher block
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read nonce from the input reader
+	nonce := NewNonce(aesgcm.NonceSize())
+	if err := nonce.Read(input); err != nil {
+		return nil, err
+	}
+
+	// Initialize a buffer to store decrypted data
+	buf := bytes.Buffer{}
+	plaintext := make([]byte, aesGcmBlockSize)
+
+	// Decrypt the ciphertext in blocks
+	for {
+		// Read a block of ciphertext from the input reader
+		ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
+		n, err := io.ReadFull(input, ciphertextBlock)
+		if n == 0 {
+			break // end of ciphertext
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return nil, err
+		}
+
+		// Decrypt the block of ciphertext
+		plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = buf.Write(plaintext)
+		if err != nil {
+			return nil, err
+		}
+
+		nonce.Increment()
+	}
+
+	return &buf, nil
+}
+
+// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+// note: This function used to decrypt files that were encrypted without a header i.e. old archives
+func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
+	var emptySalt []byte = make([]byte, 0)
+
 	// making a 32 bytes key that would correspond to AES-256
 	// don't necessarily need a salt, so just kept in empty
 	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
@@ -59,11 +223,9 @@ func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
 		return nil, err
 	}
 
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
+	// If the key is unique for each ciphertext, then it's ok to use a zero IV.
 	var iv [aes.BlockSize]byte
 	stream := cipher.NewOFB(block, iv[:])
-
 	reader := &cipher.StreamReader{S: stream, R: input}
 
 	return reader, nil

api/crypto/aes_test.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -9,7 +10,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func randBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	const passphrase = "passphrase"
+
 	tmpdir := t.TempDir()
 
 	var (
@@ -18,17 +31,99 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024*1024*100 + 523)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withStrongPassphrase(t *testing.T) {
+	const passphrase = "A strong passphrase with special characters: !@#$%^&*()_+"
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withTheSamePasswordSmallFile(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
 
 	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
 	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
@@ -57,7 +152,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024 * 50)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -96,7 +191,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1034)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -117,11 +212,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	decryptedFileWriter, _ := os.Create(decryptedFilePath)
 	defer decryptedFileWriter.Close()
 
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := os.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+	_, err = AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.NotNil(t, err, "Should not allow decrypt with wrong passphrase")
 }

api/crypto/nonce.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+type Nonce struct {
+	val []byte
+}
+
+func NewNonce(size int) *Nonce {
+	return &Nonce{val: make([]byte, size)}
+}
+
+// NewRandomNonce generates a new initial nonce with the lower byte set to a random value
+// This ensures there are plenty of nonce values availble before rolling over
+// Based on ideas from the Secure Programming Cookbook for C and C++ by John Viega, Matt Messier
+// https://www.oreilly.com/library/view/secure-programming-cookbook/0596003943/ch04s09.html
+func NewRandomNonce(size int) (*Nonce, error) {
+	randomBytes := 1
+	if size <= randomBytes {
+		return nil, errors.New("nonce size must be greater than the number of random bytes")
+	}
+
+	randomPart := make([]byte, randomBytes)
+	if _, err := rand.Read(randomPart); err != nil {
+		return nil, err
+	}
+
+	zeroPart := make([]byte, size-randomBytes)
+	nonceVal := append(randomPart, zeroPart...)
+	return &Nonce{val: nonceVal}, nil
+}
+
+func (n *Nonce) Read(stream io.Reader) error {
+	_, err := io.ReadFull(stream, n.val)
+	return err
+}
+
+func (n *Nonce) Value() []byte {
+	return n.val
+}
+
+func (n *Nonce) Increment() error {
+	// Start incrementing from the least significant byte
+	for i := len(n.val) - 1; i >= 0; i-- {
+		// Increment the current byte
+		n.val[i]++
+
+		// Check for overflow
+		if n.val[i] != 0 {
+			// No overflow, nonce is successfully incremented
+			return nil
+		}
+	}
+
+	// If we reach here, it means the nonce has overflowed
+	return errors.New("nonce overflow")
+}

api/crypto/tls.go

@@ -22,6 +22,12 @@ func CreateTLSConfiguration() *tls.Config {
 			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
 		},
 	}
 }

api/dataservices/interface.go

@@ -36,6 +36,7 @@ type (
 	}
 
 	DataStore interface {
+		Connection() portainer.Connection
 		Open() (newStore bool, err error)
 		Init() error
 		Close() error
@@ -71,8 +72,9 @@ type (
 	}
 
 	PendingActionsService interface {
-		BaseCRUD[portainer.PendingActions, portainer.PendingActionsID]
+		BaseCRUD[portainer.PendingAction, portainer.PendingActionID]
 		GetNextIdentifier() int
+		DeleteByEndpointID(ID portainer.EndpointID) error
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks

api/dataservices/pendingactions/pendingactions.go

@@ -1,10 +1,12 @@
 package pendingactions
 
 import (
+	"fmt"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -12,11 +14,11 @@ const (
 )
 
 type Service struct {
-	dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]
+	dataservices.BaseDataService[portainer.PendingAction, portainer.PendingActionID]
 }
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]
+	dataservices.BaseDataServiceTx[portainer.PendingAction, portainer.PendingActionID]
 }
 
 func NewService(connection portainer.Connection) (*Service, error) {
@@ -26,28 +28,34 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]{
+		BaseDataService: dataservices.BaseDataService[portainer.PendingAction, portainer.PendingActionID]{
 			Bucket:     BucketName,
 			Connection: connection,
 		},
 	}, nil
 }
 
-func (s Service) Create(config *portainer.PendingActions) error {
+func (s Service) Create(config *portainer.PendingAction) error {
 	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
 		return s.Tx(tx).Create(config)
 	})
 }
 
-func (s Service) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
+func (s Service) Update(ID portainer.PendingActionID, config *portainer.PendingAction) error {
 	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
 		return s.Tx(tx).Update(ID, config)
 	})
 }
 
+func (s Service) DeleteByEndpointID(ID portainer.EndpointID) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).DeleteByEndpointID(ID)
+	})
+}
+
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingAction, portainer.PendingActionID]{
 			Bucket:     BucketName,
 			Connection: service.Connection,
 			Tx:         tx,
@@ -55,19 +63,42 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	}
 }
 
-func (s ServiceTx) Create(config *portainer.PendingActions) error {
+func (s ServiceTx) Create(config *portainer.PendingAction) error {
 	return s.Tx.CreateObject(BucketName, func(id uint64) (int, interface{}) {
-		config.ID = portainer.PendingActionsID(id)
+		config.ID = portainer.PendingActionID(id)
 		config.CreatedAt = time.Now().Unix()
 
 		return int(config.ID), config
 	})
 }
 
-func (s ServiceTx) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
+func (s ServiceTx) Update(ID portainer.PendingActionID, config *portainer.PendingAction) error {
 	return s.BaseDataServiceTx.Update(ID, config)
 }
 
+func (s ServiceTx) DeleteByEndpointID(ID portainer.EndpointID) error {
+	log.Debug().Int("endpointId", int(ID)).Msg("deleting pending actions for endpoint")
+	pendingActions, err := s.BaseDataServiceTx.ReadAll()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve pending-actions for endpoint (%d): %w", ID, err)
+	}
+
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == ID {
+			err := s.BaseDataServiceTx.Delete(pendingAction.ID)
+			if err != nil {
+				log.Debug().Int("endpointId", int(ID)).Msgf("failed to delete pending action: %v", err)
+			}
+		}
+	}
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
 	return service.Connection.GetNextIdentifier(BucketName)

api/datastore/migrate_data.go

@@ -86,6 +86,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		EdgeStackService:        store.EdgeStackService,
 		EdgeJobService:          store.EdgeJobService,
 		TunnelServerService:     store.TunnelServerService,
+		PendingActionsService:   store.PendingActionsService,
 	}
 }
 

api/datastore/migrate_post_init.go

@@ -1,117 +0,0 @@
-package datastore
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
-	"github.com/portainer/portainer/api/kubernetes/cli"
-
-	"github.com/rs/zerolog/log"
-)
-
-type PostInitMigrator struct {
-	kubeFactory   *cli.ClientFactory
-	dockerFactory *dockerclient.ClientFactory
-	dataStore     dataservices.DataStore
-}
-
-func NewPostInitMigrator(kubeFactory *cli.ClientFactory, dockerFactory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *PostInitMigrator {
-	return &PostInitMigrator{
-		kubeFactory:   kubeFactory,
-		dockerFactory: dockerFactory,
-		dataStore:     dataStore,
-	}
-}
-
-func (migrator *PostInitMigrator) PostInitMigrate() error {
-	if err := migrator.PostInitMigrateIngresses(); err != nil {
-		return err
-	}
-
-	migrator.PostInitMigrateGPUs()
-
-	return nil
-}
-
-func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
-	endpoints, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for i := range endpoints {
-		// Early exit if we do not need to migrate!
-		if !endpoints[i].PostInitMigrations.MigrateIngresses {
-			return nil
-		}
-
-		err := migrator.kubeFactory.MigrateEndpointIngresses(&endpoints[i])
-		if err != nil {
-			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
-		}
-	}
-
-	return nil
-}
-
-// PostInitMigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
-// If there's an error getting the containers, we'll log it and move on
-func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
-	environments, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		log.Err(err).Msg("failure getting endpoints")
-		return
-	}
-
-	for i := range environments {
-		if environments[i].Type == portainer.DockerEnvironment {
-			// // Early exit if we do not need to migrate!
-			if !environments[i].PostInitMigrations.MigrateGPUs {
-				return
-			}
-
-			// set the MigrateGPUs flag to false so we don't run this again
-			environments[i].PostInitMigrations.MigrateGPUs = false
-			migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-			// create a docker client
-			dockerClient, err := migrator.dockerFactory.CreateClient(&environments[i], "", nil)
-			if err != nil {
-				log.Err(err).Msg("failure creating docker client for environment: " + environments[i].Name)
-				return
-			}
-			defer dockerClient.Close()
-
-			// get all containers
-			containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
-			if err != nil {
-				log.Err(err).Msg("failed to list containers")
-				return
-			}
-
-			// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole endpoint
-		containersLoop:
-			for _, container := range containers {
-				// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
-				containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
-				if err != nil {
-					log.Err(err).Msg("failed to inspect container")
-					return
-				}
-
-				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
-				for _, deviceRequest := range deviceRequests {
-					if deviceRequest.Driver == "nvidia" {
-						environments[i].EnableGPUManagement = true
-						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-						break containersLoop
-					}
-				}
-			}
-		}
-	}
-}

api/datastore/migrator/migrate_dbversion110.go

@@ -23,3 +23,29 @@ func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
 
 	return migrator.settingsService.UpdateSettings(settings)
 }
+
+// In PortainerCE the resource overcommit option should always be true across all endpoints
+func (migrator *Migrator) updateResourceOverCommitToDB110() error {
+	log.Info().Msg("updating resource overcommit setting to true")
+
+	endpoints, err := migrator.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.Type == portainer.KubernetesLocalEnvironment ||
+			endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
+			endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+
+			endpoint.Kubernetes.Configuration.EnableResourceOverCommit = true
+
+			err = migrator.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion111.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
+)
+
+func (migrator *Migrator) cleanPendingActionsForDeletedEndpointsForDB111() error {
+	log.Info().Msg("cleaning up pending actions for deleted endpoints")
+
+	pendingActions, err := migrator.pendingActionsService.ReadAll()
+	if err != nil {
+		return err
+	}
+
+	endpoints := make(map[portainer.EndpointID]struct{})
+	for _, action := range pendingActions {
+		endpoints[action.EndpointID] = struct{}{}
+	}
+
+	for endpointId := range endpoints {
+		_, err := migrator.endpointService.Endpoint(endpointId)
+		if dataservices.IsErrObjectNotFound(err) {
+			err := migrator.pendingActionsService.DeleteByEndpointID(endpointId)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrate_dbversion130.go

@@ -0,0 +1,33 @@
+package migrator
+
+import (
+	"github.com/segmentio/encoding/json"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (migrator *Migrator) migratePendingActionsDataForDB130() error {
+	log.Info().Msg("Migrating pending actions data")
+
+	pendingActions, err := migrator.pendingActionsService.ReadAll()
+	if err != nil {
+		return err
+	}
+
+	for _, pa := range pendingActions {
+		actionData, err := json.Marshal(pa.ActionData)
+		if err != nil {
+			return err
+		}
+
+		pa.ActionData = string(actionData)
+
+		// Update the pending action
+		err = migrator.pendingActionsService.Update(pa.ID, &pa)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -14,6 +14,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
 	"github.com/portainer/portainer/api/dataservices/extension"
 	"github.com/portainer/portainer/api/dataservices/fdoprofile"
+	"github.com/portainer/portainer/api/dataservices/pendingactions"
 	"github.com/portainer/portainer/api/dataservices/registry"
 	"github.com/portainer/portainer/api/dataservices/resourcecontrol"
 	"github.com/portainer/portainer/api/dataservices/role"
@@ -58,6 +59,7 @@ type (
 		edgeStackService        *edgestack.Service
 		edgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		pendingActionsService   *pendingactions.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -85,6 +87,7 @@ type (
 		EdgeStackService        *edgestack.Service
 		EdgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		PendingActionsService   *pendingactions.Service
 	}
 )
 
@@ -114,6 +117,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		edgeStackService:        parameters.EdgeStackService,
 		edgeJobService:          parameters.EdgeJobService,
 		TunnelServerService:     parameters.TunnelServerService,
+		pendingActionsService:   parameters.PendingActionsService,
 	}
 
 	migrator.initMigrations()
@@ -230,9 +234,16 @@ func (m *Migrator) initMigrations() {
 	)
 	m.addMigrations("2.20",
 		m.updateAppTemplatesVersionForDB110,
+		m.updateResourceOverCommitToDB110,
+	)
+	m.addMigrations("2.20.2",
+		m.cleanPendingActionsForDeletedEndpointsForDB111,
+	)
+	m.addMigrations("2.22.0",
+		m.migratePendingActionsDataForDB130,
 	)
 
-	// Add new migrations below...
+	// Add new migrations above...
 	// One function per migration, each versions migration funcs in the same file.
 }
 

api/datastore/pendingactions_test.go

@@ -0,0 +1,98 @@
+package datastore
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
+)
+
+type cleanNAPWithOverridePolicies struct {
+	EndpointGroupID portainer.EndpointGroupID
+}
+
+func Test_ConvertCleanNAPWithOverridePoliciesPayload(t *testing.T) {
+	t.Run("test ConvertCleanNAPWithOverridePoliciesPayload", func(t *testing.T) {
+
+		_, store := MustNewTestStore(t, true, false)
+		defer store.Close()
+
+		gid := portainer.EndpointGroupID(1)
+
+		testData := []struct {
+			Name          string
+			PendingAction portainer.PendingAction
+			Expected      any
+			Err           bool
+		}{
+			{
+				Name: "test actiondata with EndpointGroupID 1",
+				PendingAction: handlers.NewCleanNAPWithOverridePolicies(
+					1,
+					&gid,
+				),
+				Expected: portainer.EndpointGroupID(1),
+			},
+			{
+				Name: "test actionData nil",
+				PendingAction: handlers.NewCleanNAPWithOverridePolicies(
+					2,
+					nil,
+				),
+				Expected: nil,
+			},
+			{
+				Name: "test actionData empty and expected error",
+				PendingAction: portainer.PendingAction{
+					EndpointID: 2,
+					Action:     actions.CleanNAPWithOverridePolicies,
+					ActionData: "",
+				},
+				Expected: nil,
+				Err:      true,
+			},
+		}
+
+		for _, d := range testData {
+			err := store.PendingActions().Create(&d.PendingAction)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			pendingActions, err := store.PendingActions().ReadAll()
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			for _, endpointPendingAction := range pendingActions {
+				t.Run(d.Name, func(t *testing.T) {
+					if endpointPendingAction.Action == actions.CleanNAPWithOverridePolicies {
+						var payload cleanNAPWithOverridePolicies
+
+						err := endpointPendingAction.UnmarshallActionData(&payload)
+
+						if d.Err && err == nil {
+							t.Error(err)
+						}
+
+						if d.Expected == nil && payload.EndpointGroupID != 0 {
+							t.Errorf("expected nil, got %d", payload.EndpointGroupID)
+						}
+
+						if d.Expected != nil {
+							expected := d.Expected.(portainer.EndpointGroupID)
+							if d.Expected != nil && expected != payload.EndpointGroupID {
+								t.Errorf("expected EndpointGroupID %d, got %d", expected, payload.EndpointGroupID)
+							}
+						}
+					}
+				})
+			}
+
+			store.PendingActions().Delete(d.PendingAction.ID)
+		}
+	})
+}

api/datastore/postinit/migrate_post_init.go

@@ -0,0 +1,184 @@
+package postinit
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerClient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+
+	"github.com/rs/zerolog/log"
+)
+
+type PostInitMigrator struct {
+	kubeFactory        *cli.ClientFactory
+	dockerFactory      *dockerClient.ClientFactory
+	dataStore          dataservices.DataStore
+	assetsPath         string
+	kubernetesDeployer portainer.KubernetesDeployer
+}
+
+func NewPostInitMigrator(
+	kubeFactory *cli.ClientFactory,
+	dockerFactory *dockerClient.ClientFactory,
+	dataStore dataservices.DataStore,
+	assetsPath string,
+	kubernetesDeployer portainer.KubernetesDeployer,
+) *PostInitMigrator {
+	return &PostInitMigrator{
+		kubeFactory:        kubeFactory,
+		dockerFactory:      dockerFactory,
+		dataStore:          dataStore,
+		assetsPath:         assetsPath,
+		kubernetesDeployer: kubernetesDeployer,
+	}
+}
+
+// PostInitMigrate will run all post-init migrations, which require docker/kube clients for all edge or non-edge environments
+func (postInitMigrator *PostInitMigrator) PostInitMigrate() error {
+	environments, err := postInitMigrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		log.Error().Err(err).Msg("Error getting environments")
+		return err
+	}
+
+	for _, environment := range environments {
+		// edge environments will run after the server starts, in pending actions
+		if endpointutils.IsEdgeEndpoint(&environment) {
+			log.Info().Msgf("Adding pending action 'PostInitMigrateEnvironment' for environment %d", environment.ID)
+			err = postInitMigrator.createPostInitMigrationPendingAction(environment.ID)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error creating pending action for environment %d", environment.ID)
+			}
+		} else {
+			// non-edge environments will run before the server starts.
+			err = postInitMigrator.MigrateEnvironment(&environment)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error running post-init migrations for non-edge environment %d", environment.ID)
+			}
+		}
+
+	}
+
+	return nil
+}
+
+// try to create a post init migration pending action. If it already exists, do nothing
+// this function exists for readability, not reusability
+// TODO: This should be moved into pending actions as part of the pending action migration
+func (postInitMigrator *PostInitMigrator) createPostInitMigrationPendingAction(environmentID portainer.EndpointID) error {
+	// If there are no pending actions for the given endpoint, create one
+	err := postInitMigrator.dataStore.PendingActions().Create(&portainer.PendingAction{
+		EndpointID: environmentID,
+		Action:     actions.PostInitMigrateEnvironment,
+	})
+	if err != nil {
+		log.Error().Err(err).Msgf("Error creating pending action for environment %d", environmentID)
+	}
+	return nil
+}
+
+// MigrateEnvironment runs migrations on a single environment
+func (migrator *PostInitMigrator) MigrateEnvironment(environment *portainer.Endpoint) error {
+	log.Info().Msgf("Executing post init migration for environment %d", environment.ID)
+
+	switch {
+	case endpointutils.IsKubernetesEndpoint(environment):
+		// get the kubeclient for the environment, and skip all kube migrations if there's an error
+		kubeclient, err := migrator.kubeFactory.GetKubeClient(environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating kubeclient for environment: %d", environment.ID)
+			return err
+		}
+		// if one environment fails, it is logged and the next migration runs. The error is returned at the end and handled by pending actions
+		err = migrator.MigrateIngresses(*environment, kubeclient)
+		if err != nil {
+			return err
+		}
+		return nil
+	case endpointutils.IsDockerEndpoint(environment):
+		// get the docker client for the environment, and skip all docker migrations if there's an error
+		dockerClient, err := migrator.dockerFactory.CreateClient(environment, "", nil)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating docker client for environment: %d", environment.ID)
+			return err
+		}
+		defer dockerClient.Close()
+		migrator.MigrateGPUs(*environment, dockerClient)
+	}
+
+	return nil
+}
+
+func (migrator *PostInitMigrator) MigrateIngresses(environment portainer.Endpoint, kubeclient *cli.KubeClient) error {
+	// Early exit if we do not need to migrate!
+	if !environment.PostInitMigrations.MigrateIngresses {
+		return nil
+	}
+	log.Debug().Msgf("Migrating ingresses for environment %d", environment.ID)
+
+	err := migrator.kubeFactory.MigrateEndpointIngresses(&environment, migrator.dataStore, kubeclient)
+	if err != nil {
+		log.Error().Err(err).Msgf("Error migrating ingresses for environment %d", environment.ID)
+		return err
+	}
+	return nil
+}
+
+// MigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
+// If there's an error getting the containers, we'll log it and move on
+func (migrator *PostInitMigrator) MigrateGPUs(e portainer.Endpoint, dockerClient *client.Client) error {
+	return migrator.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		environment, err := tx.Endpoint().Endpoint(e.ID)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting environment %d", environment.ID)
+			return err
+		}
+		// Early exit if we do not need to migrate!
+		if !environment.PostInitMigrations.MigrateGPUs {
+			return nil
+		}
+		log.Debug().Msgf("Migrating GPUs for environment %d", e.ID)
+
+		// get all containers
+		containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{All: true})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to list containers for environment %d", environment.ID)
+			return err
+		}
+
+		// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole environment
+	containersLoop:
+		for _, container := range containers {
+			// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
+			containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to inspect container")
+				continue
+			}
+
+			deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
+			for _, deviceRequest := range deviceRequests {
+				if deviceRequest.Driver == "nvidia" {
+					environment.EnableGPUManagement = true
+					break containersLoop
+				}
+			}
+		}
+
+		// set the MigrateGPUs flag to false so we don't run this again
+		environment.PostInitMigrations.MigrateGPUs = false
+		err = tx.Endpoint().UpdateEndpoint(environment.ID, environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error updating EnableGPUManagement flag for environment %d", environment.ID)
+			return err
+		}
+
+		return nil
+	})
+}

api/datastore/services_tx.go

@@ -16,7 +16,9 @@ func (tx *StoreTx) IsErrObjectNotFound(err error) bool {
 
 func (tx *StoreTx) CustomTemplate() dataservices.CustomTemplateService { return nil }
 
-func (tx *StoreTx) PendingActions() dataservices.PendingActionsService { return nil }
+func (tx *StoreTx) PendingActions() dataservices.PendingActionsService {
+	return tx.store.PendingActionsService.Tx(tx.tx)
+}
 
 func (tx *StoreTx) EdgeGroup() dataservices.EdgeGroupService {
 	return tx.store.EdgeGroupService.Tx(tx.tx)
@@ -68,7 +70,10 @@ func (tx *StoreTx) Snapshot() dataservices.SnapshotService {
 }
 
 func (tx *StoreTx) SSLSettings() dataservices.SSLSettingsService { return nil }
-func (tx *StoreTx) Stack() dataservices.StackService             { return nil }
+
+func (tx *StoreTx) Stack() dataservices.StackService {
+	return tx.store.StackService.Tx(tx.tx)
+}
 
 func (tx *StoreTx) Tag() dataservices.TagService {
 	return tx.store.TagService.Tx(tx.tx)
@@ -78,7 +83,8 @@ func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
 	return tx.store.TeamMembershipService.Tx(tx.tx)
 }
 
-func (tx *StoreTx) Team() dataservices.TeamService                 { return nil }
+func (tx *StoreTx) Team() dataservices.TeamService { return nil }
+
 func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }
 
 func (tx *StoreTx) User() dataservices.UserService {

api/datastore/test_data/output_24_to_latest.json

@@ -631,6 +631,7 @@
     "LogoURL": "",
     "OAuthSettings": {
       "AccessTokenURI": "",
+      "AuthStyle": 0,
       "AuthorizationURI": "",
       "ClientID": "",
       "DefaultTeamID": 0,
@@ -677,6 +678,7 @@
             "Architecture": "",
             "BridgeNfIp6tables": false,
             "BridgeNfIptables": false,
+            "CDISpecDirs": null,
             "CPUSet": false,
             "CPUShares": false,
             "CgroupDriver": "",
@@ -939,6 +941,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.21.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.22.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/demo/demo.go

@@ -1,118 +0,0 @@
-package demo
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
-)
-
-type EnvironmentDetails struct {
-	Enabled      bool                   `json:"enabled"`
-	Users        []portainer.UserID     `json:"users"`
-	Environments []portainer.EndpointID `json:"environments"`
-}
-
-type Service struct {
-	details EnvironmentDetails
-}
-
-func NewService() *Service {
-	return &Service{}
-}
-
-func (service *Service) Details() EnvironmentDetails {
-	return service.details
-}
-
-func (service *Service) Init(store dataservices.DataStore, cryptoService portainer.CryptoService) error {
-	log.Info().Msg("starting demo environment")
-
-	isClean, err := isCleanStore(store)
-	if err != nil {
-		return errors.WithMessage(err, "failed checking if store is clean")
-	}
-
-	if !isClean {
-		return errors.New(" Demo environment can only be initialized on a clean database")
-	}
-
-	id, err := initDemoUser(store, cryptoService)
-	if err != nil {
-		return errors.WithMessage(err, "failed creating demo user")
-	}
-
-	endpointIds, err := initDemoEndpoints(store)
-	if err != nil {
-		return errors.WithMessage(err, "failed creating demo endpoint")
-	}
-
-	err = initDemoSettings(store)
-	if err != nil {
-		return errors.WithMessage(err, "failed updating demo settings")
-	}
-
-	service.details = EnvironmentDetails{
-		Enabled: true,
-		Users:   []portainer.UserID{id},
-		// endpoints 2,3 are created after deployment of portainer
-		Environments: endpointIds,
-	}
-
-	return nil
-}
-
-func isCleanStore(store dataservices.DataStore) (bool, error) {
-	endpoints, err := store.Endpoint().Endpoints()
-	if err != nil {
-		return false, err
-	}
-
-	if len(endpoints) > 0 {
-		return false, nil
-	}
-
-	users, err := store.User().ReadAll()
-	if err != nil {
-		return false, err
-	}
-
-	if len(users) > 0 {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func (service *Service) IsDemo() bool {
-	return service.details.Enabled
-}
-
-func (service *Service) IsDemoEnvironment(environmentID portainer.EndpointID) bool {
-	if !service.IsDemo() {
-		return false
-	}
-
-	for _, demoEndpointID := range service.details.Environments {
-		if environmentID == demoEndpointID {
-			return true
-		}
-	}
-
-	return false
-}
-
-func (service *Service) IsDemoUser(userID portainer.UserID) bool {
-	if !service.IsDemo() {
-		return false
-	}
-
-	for _, demoUserID := range service.details.Users {
-		if userID == demoUserID {
-			return true
-		}
-	}
-
-	return false
-}

api/demo/init.go

@@ -1,88 +0,0 @@
-package demo
-
-import (
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-)
-
-func initDemoUser(
-	store dataservices.DataStore,
-	cryptoService portainer.CryptoService,
-) (portainer.UserID, error) {
-
-	password, err := cryptoService.Hash("tryportainer")
-	if err != nil {
-		return 0, errors.WithMessage(err, "failed creating password hash")
-	}
-
-	admin := &portainer.User{
-		Username: "admin",
-		Password: password,
-		Role:     portainer.AdministratorRole,
-	}
-
-	err = store.User().Create(admin)
-	return admin.ID, errors.WithMessage(err, "failed creating user")
-}
-
-func initDemoEndpoints(store dataservices.DataStore) ([]portainer.EndpointID, error) {
-	localEndpointId, err := initDemoLocalEndpoint(store)
-	if err != nil {
-		return nil, err
-	}
-
-	// second and third endpoints are going to be created with docker-compose as a part of the demo environment set up.
-	// ref: https://github.com/portainer/portainer-demo/blob/master/docker-compose.yml
-	return []portainer.EndpointID{localEndpointId, localEndpointId + 1, localEndpointId + 2}, nil
-}
-
-func initDemoLocalEndpoint(store dataservices.DataStore) (portainer.EndpointID, error) {
-	id := portainer.EndpointID(store.Endpoint().GetNextIdentifier())
-	localEndpoint := &portainer.Endpoint{
-		ID:        id,
-		Name:      "local",
-		URL:       "unix:///var/run/docker.sock",
-		PublicURL: "demo.portainer.io",
-		Type:      portainer.DockerEnvironment,
-		GroupID:   portainer.EndpointGroupID(1),
-		TLSConfig: portainer.TLSConfiguration{
-			TLS: false,
-		},
-		AuthorizedUsers:    []portainer.UserID{},
-		AuthorizedTeams:    []portainer.TeamID{},
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         portainer.KubernetesDefault(),
-	}
-
-	err := store.Endpoint().Create(localEndpoint)
-	if err != nil {
-		return id, errors.WithMessage(err, "failed creating local endpoint")
-	}
-
-	err = store.Snapshot().Create(&portainer.Snapshot{EndpointID: id})
-	if err != nil {
-		return id, errors.WithMessage(err, "failed creating snapshot")
-	}
-
-	return id, errors.WithMessage(err, "failed creating local endpoint")
-}
-
-func initDemoSettings(
-	store dataservices.DataStore,
-) error {
-	settings, err := store.Settings().Settings()
-	if err != nil {
-		return errors.WithMessage(err, "failed fetching settings")
-	}
-
-	settings.EnableTelemetry = false
-	settings.LogoURL = ""
-
-	err = store.Settings().UpdateSettings(settings)
-	return errors.WithMessage(err, "failed updating settings")
-}

api/docker/client/client.go

@@ -3,7 +3,6 @@ package client
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"io"
 	"maps"
 	"net/http"
@@ -13,7 +12,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	"github.com/segmentio/encoding/json"
 )
@@ -50,12 +49,12 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
 	case portainer.AgentOnDockerEnvironment:
 		return createAgentClient(endpoint, endpoint.URL, factory.signatureService, nodeName, timeout)
 	case portainer.EdgeAgentOnDockerEnvironment:
-		tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint)
+		tunnelAddr, err := factory.reverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return nil, err
 		}
 
-		endpointURL := fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+		endpointURL := "http://" + tunnelAddr
 
 		return createAgentClient(endpoint, endpointURL, factory.signatureService, nodeName, timeout)
 	}
@@ -93,11 +92,17 @@ func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*cli
 		return nil, err
 	}
 
-	return client.NewClientWithOpts(
+	opts := []client.Opt{
 		client.WithHost(endpoint.URL),
 		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
-	)
+	}
+
+	if nnTransport, ok := httpCli.Transport.(*NodeNameTransport); ok && nnTransport.TLSClientConfig != nil {
+		opts = append(opts, client.WithScheme("https"))
+	}
+
+	return client.NewClientWithOpts(opts...)
 }
 
 func createAgentClient(endpoint *portainer.Endpoint, endpointURL string, signatureService portainer.DigitalSignatureService, nodeName string, timeout *time.Duration) (*client.Client, error) {
@@ -159,7 +164,7 @@ func (t *NodeNameTransport) RoundTrip(req *http.Request) (*http.Response, error)
 	resp.Body = io.NopCloser(bytes.NewReader(body))
 
 	var rs []struct {
-		types.ImageSummary
+		image.Summary
 		Portainer struct {
 			Agent struct {
 				NodeName string

api/docker/consts/labels.go

@@ -5,4 +5,5 @@ const (
 	SwarmStackNameLabel   = "com.docker.stack.namespace"
 	SwarmServiceIdLabel   = "com.docker.swarm.service.id"
 	SwarmNodeIdLabel      = "com.docker.swarm.node.id"
+	HideStackLabel        = "io.portainer.hideStack"
 )

api/docker/container.go

@@ -119,7 +119,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 		for _, network := range container.NetworkSettings.Networks {
 			cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
 		}
-		cli.ContainerStart(ctx, containerId, types.ContainerStartOptions{})
+		cli.ContainerStart(ctx, containerId, dockercontainer.StartOptions{})
 	})
 
 	log.Debug().Str("container", strings.Split(container.Name, "/")[1]).Msg("starting to create a new container")
@@ -135,7 +135,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 	c.sr.push(func() {
 		log.Debug().Str("container_id", create.ID).Msg("removing the new container")
 		cli.ContainerStop(ctx, create.ID, dockercontainer.StopOptions{})
-		cli.ContainerRemove(ctx, create.ID, types.ContainerRemoveOptions{})
+		cli.ContainerRemove(ctx, create.ID, dockercontainer.RemoveOptions{})
 	})
 
 	if err != nil {
@@ -164,14 +164,14 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 
 	// 8. start the new container
 	log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
-	err = cli.ContainerStart(ctx, newContainerId, types.ContainerStartOptions{})
+	err = cli.ContainerStart(ctx, newContainerId, dockercontainer.StartOptions{})
 	if err != nil {
 		return nil, errors.Wrap(err, "start container error")
 	}
 
 	// 9. delete the old container
 	log.Debug().Str("container_id", containerId).Msg("starting to remove the old container")
-	_ = cli.ContainerRemove(ctx, containerId, types.ContainerRemoveOptions{})
+	_ = cli.ContainerRemove(ctx, containerId, dockercontainer.RemoveOptions{})
 
 	c.sr.disable()
 

api/docker/container_stats.go

@@ -0,0 +1,37 @@
+package docker
+
+import "github.com/docker/docker/api/types"
+
+type ContainerStats struct {
+	Running   int `json:"running"`
+	Stopped   int `json:"stopped"`
+	Healthy   int `json:"healthy"`
+	Unhealthy int `json:"unhealthy"`
+	Total     int `json:"total"`
+}
+
+func CalculateContainerStats(containers []types.Container) ContainerStats {
+	var running, stopped, healthy, unhealthy int
+	for _, container := range containers {
+		switch container.State {
+		case "running":
+			running++
+		case "healthy":
+			running++
+			healthy++
+		case "unhealthy":
+			running++
+			unhealthy++
+		case "exited", "stopped":
+			stopped++
+		}
+	}
+
+	return ContainerStats{
+		Running:   running,
+		Stopped:   stopped,
+		Healthy:   healthy,
+		Unhealthy: unhealthy,
+		Total:     len(containers),
+	}
+}

api/docker/container_stats_test.go

@@ -0,0 +1,27 @@
+package docker
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCalculateContainerStats(t *testing.T) {
+	containers := []types.Container{
+		{State: "running"},
+		{State: "running"},
+		{State: "exited"},
+		{State: "stopped"},
+		{State: "healthy"},
+		{State: "unhealthy"},
+	}
+
+	stats := CalculateContainerStats(containers)
+
+	assert.Equal(t, 4, stats.Running)
+	assert.Equal(t, 2, stats.Stopped)
+	assert.Equal(t, 1, stats.Healthy)
+	assert.Equal(t, 1, stats.Unhealthy)
+	assert.Equal(t, 6, stats.Total)
+}

api/docker/images/status.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	portainer "github.com/portainer/portainer/api"
 	consts "github.com/portainer/portainer/api/docker/consts"
@@ -157,7 +158,7 @@ func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string,
 		return Error, nil
 	}
 
-	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
+	containers, err := cli.ContainerList(ctx, container.ListOptions{
 		All:     true,
 		Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+serviceID)),
 	})

api/docker/snapshot.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	_container "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
@@ -147,24 +148,16 @@ func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Clien
 }
 
 func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
-	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
 	if err != nil {
 		return err
 	}
 
-	runningContainers := 0
-	stoppedContainers := 0
-	healthyContainers := 0
-	unhealthyContainers := 0
 	stacks := make(map[string]struct{})
 	gpuUseSet := make(map[string]struct{})
 	gpuUseAll := false
 	for _, container := range containers {
-		if container.State == "exited" || container.State == "stopped" {
-			stoppedContainers++
-		} else if container.State == "running" {
-			runningContainers++
-
+		if container.State == "running" {
 			// snapshot GPUs
 			response, err := cli.ContainerInspect(context.Background(), container.ID)
 			if err != nil {
@@ -201,15 +194,6 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 			}
 		}
 
-		if container.State == "healthy" {
-			runningContainers++
-			healthyContainers++
-		}
-
-		if container.State == "unhealthy" {
-			unhealthyContainers++
-		}
-
 		for k, v := range container.Labels {
 			if k == consts.ComposeStackNameLabel {
 				stacks[v] = struct{}{}
@@ -225,11 +209,13 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 	snapshot.GpuUseAll = gpuUseAll
 	snapshot.GpuUseList = gpuUseList
 
-	snapshot.ContainerCount = len(containers)
-	snapshot.RunningContainerCount = runningContainers
-	snapshot.StoppedContainerCount = stoppedContainers
-	snapshot.HealthyContainerCount = healthyContainers
-	snapshot.UnhealthyContainerCount = unhealthyContainers
+	stats := CalculateContainerStats(containers)
+
+	snapshot.ContainerCount = stats.Total
+	snapshot.RunningContainerCount = stats.Running
+	snapshot.StoppedContainerCount = stats.Stopped
+	snapshot.HealthyContainerCount = stats.Healthy
+	snapshot.UnhealthyContainerCount = stats.Unhealthy
 	snapshot.StackCount += len(stacks)
 	for _, container := range containers {
 		snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})

api/exec/compose_stack.go

@@ -76,15 +76,9 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		defer proxy.Close()
 	}
 
-	envFilePath, err := createEnvFile(stack)
-	if err != nil {
-		return errors.Wrap(err, "failed to create env file")
-	}
-
 	err = manager.deployer.Remove(ctx, stack.Name, nil, libstack.Options{
-		WorkingDir:  stack.ProjectPath,
-		EnvFilePath: envFilePath,
-		Host:        url,
+		WorkingDir: "",
+		Host:       url,
 	})
 
 	return errors.Wrap(err, "failed to remove a stack")
@@ -148,28 +142,46 @@ func createEnvFile(stack *portainer.Stack) (string, error) {
 	}
 	defer envfile.Close()
 
-	copyDefaultEnvFile(stack, envfile)
+	// Copy from default .env file
+	defaultEnvPath := path.Join(stack.ProjectPath, path.Dir(stack.EntryPoint), ".env")
+	if err = copyDefaultEnvFile(envfile, defaultEnvPath); err != nil {
+		return "", err
+	}
 
-	for _, v := range stack.Env {
-		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
+	// Copy from stack env vars
+	if err = copyConfigEnvVars(envfile, stack.Env); err != nil {
+		return "", err
 	}
 
 	return "stack.env", nil
 }
 
 // copyDefaultEnvFile copies the default .env file if it exists to the provided writer
-func copyDefaultEnvFile(stack *portainer.Stack, w io.Writer) {
-	defaultEnvFile, err := os.Open(path.Join(path.Join(stack.ProjectPath, path.Dir(stack.EntryPoint)), ".env"))
+func copyDefaultEnvFile(w io.Writer, defaultEnvFilePath string) error {
+	defaultEnvFile, err := os.Open(defaultEnvFilePath)
 	if err != nil {
 		// If cannot open a default file, then don't need to copy it.
 		// We could as well stat it and check if it exists, but this is more efficient.
-		return
+		return nil
 	}
 
 	defer defaultEnvFile.Close()
 
 	if _, err = io.Copy(w, defaultEnvFile); err == nil {
-		io.WriteString(w, "\n")
+		if _, err = fmt.Fprintf(w, "\n"); err != nil {
+			return fmt.Errorf("failed to copy default env file: %w", err)
+		}
 	}
+	return nil
 	// If couldn't copy the .env file, then ignore the error and try to continue
 }
+
+// copyConfigEnvVars write the environment variables from stack configuration to the writer
+func copyConfigEnvVars(w io.Writer, envs []portainer.Pair) error {
+	for _, v := range envs {
+		if _, err := fmt.Fprintf(w, "%s=%s\n", v.Name, v.Value); err != nil {
+			return fmt.Errorf("failed to copy config env vars: %w", err)
+		}
+	}
+	return nil
+}

api/exec/compose_stack_test.go

@@ -23,6 +23,7 @@ func Test_createEnvFile(t *testing.T) {
 			name: "should not add env file option if stack doesn't have env variables",
 			stack: &portainer.Stack{
 				ProjectPath: dir,
+				Env:         nil,
 			},
 			expected: "",
 		},

api/exec/swarm_stack.go

@@ -3,7 +3,6 @@ package exec
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"os"
 	"os/exec"
 	"path"
@@ -158,7 +157,10 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor
 	var stderr bytes.Buffer
 	cmd := exec.Command(command, args...)
 	cmd.Stderr = &stderr
-	cmd.Dir = workingDir
+
+	if workingDir != "" {
+		cmd.Dir = workingDir
+	}
 
 	if env != nil {
 		cmd.Env = os.Environ()
@@ -186,11 +188,11 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, config
 
 	endpointURL := endpoint.URL
 	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		tunnel, err := manager.reverseTunnelService.GetActiveTunnel(endpoint)
+		tunnelAddr, err := manager.reverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return "", nil, err
 		}
-		endpointURL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnel.Port)
+		endpointURL = "tcp://" + tunnelAddr
 	}
 
 	args = append(args, "-H", endpointURL)

api/filesystem/filesystem.go

@@ -934,7 +934,7 @@ func FileExists(filePath string) (bool, error) {
 func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
 	// 1. Backup the source directory to a different folder
 	backupDir := fmt.Sprintf("%s-%s", filepath.Dir(originalPath), "backup")
-	err := MoveDirectory(originalPath, backupDir)
+	err := MoveDirectory(originalPath, backupDir, false)
 	if err != nil {
 		return fmt.Errorf("failed to backup source directory: %w", err)
 	}
@@ -973,14 +973,14 @@ func restoreBackup(src, backupDir string) error {
 		return fmt.Errorf("failed to delete destination directory: %w", err)
 	}
 
-	err = MoveDirectory(backupDir, src)
+	err = MoveDirectory(backupDir, src, false)
 	if err != nil {
 		return fmt.Errorf("failed to restore backup directory: %w", err)
 	}
 	return nil
 }
 
-func MoveDirectory(originalPath, newPath string) error {
+func MoveDirectory(originalPath, newPath string, overwriteTargetPath bool) error {
 	if _, err := os.Stat(originalPath); err != nil {
 		return err
 	}
@@ -991,7 +991,13 @@ func MoveDirectory(originalPath, newPath string) error {
 	}
 
 	if alreadyExists {
-		return errors.New("Target path already exists")
+		if !overwriteTargetPath {
+			return fmt.Errorf("Target path already exists")
+		}
+
+		if err = os.RemoveAll(newPath); err != nil {
+			return fmt.Errorf("failed to overwrite path %s: %s", newPath, err.Error())
+		}
 	}
 
 	return os.Rename(originalPath, newPath)

api/filesystem/filesystem_move_test.go

@@ -16,7 +16,7 @@ func Test_movePath_shouldFailIfSourceDirDoesNotExist(t *testing.T) {
 	file1 := addFile(destinationDir, "dir", "file")
 	file2 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when source path is missing")
 	assert.FileExists(t, file1, "destination dir contents should remain")
 	assert.FileExists(t, file2, "destination dir contents should remain")
@@ -30,7 +30,7 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	file3 := addFile(destinationDir, "dir", "file")
 	file4 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when destination directory already exists")
 	assert.FileExists(t, file1, "source dir contents should remain")
 	assert.FileExists(t, file2, "source dir contents should remain")
@@ -38,6 +38,22 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	assert.FileExists(t, file4, "destination dir contents should remain")
 }
 
+func Test_movePath_succesIfOverwriteSetWhenDestinationDirExists(t *testing.T) {
+	sourceDir := t.TempDir()
+	file1 := addFile(sourceDir, "dir", "file")
+	file2 := addFile(sourceDir, "file")
+	destinationDir := t.TempDir()
+	file3 := addFile(destinationDir, "dir", "file")
+	file4 := addFile(destinationDir, "file")
+
+	err := MoveDirectory(sourceDir, destinationDir, true)
+	assert.NoError(t, err)
+	assert.NoFileExists(t, file1, "source dir contents should be moved")
+	assert.NoFileExists(t, file2, "source dir contents should be moved")
+	assert.FileExists(t, file3, "destination dir contents should remain")
+	assert.FileExists(t, file4, "destination dir contents should remain")
+}
+
 func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T) {
 	tmp := t.TempDir()
 	sourceDir := path.Join(tmp, "source")
@@ -46,7 +62,7 @@ func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T)
 	file2 := addFile(sourceDir, "file")
 	destinationDir := path.Join(tmp, "destination")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.NoError(t, err)
 	assert.NoFileExists(t, file1, "source dir contents should be moved")
 	assert.NoFileExists(t, file2, "source dir contents should be moved")

api/git/backup.go

@@ -38,7 +38,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 		}
 	}
 
-	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath)
+	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath, true)
 	if err != nil {
 		return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
 	}
@@ -48,7 +48,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 	err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password, options.TLSSkipVerify)
 	if err != nil {
 		cleanUp = false
-		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath)
+		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath, false)
 		if restoreError != nil {
 			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
 		}

api/http/csrf/csrf.go

@@ -21,7 +21,11 @@ func WithProtect(handler http.Handler) (http.Handler, error) {
 		return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
 	}
 
-	handler = gorillacsrf.Protect([]byte(token), gorillacsrf.Path("/"))(handler)
+	handler = gorillacsrf.Protect(
+		[]byte(token),
+		gorillacsrf.Path("/"),
+		gorillacsrf.Secure(false),
+	)(handler)
 
 	return withSkipCSRF(handler), nil
 }

api/http/errors/errors.go

@@ -9,6 +9,4 @@ var (
 	ErrUnauthorized = errors.New("Unauthorized")
 	// ErrResourceAccessDenied Access denied to resource error
 	ErrResourceAccessDenied = errors.New("Access denied to resource")
-	// ErrNotAvailableInDemo feature is not allowed in demo
-	ErrNotAvailableInDemo = errors.New("This feature is not available in the demo version of Portainer")
 )

api/http/handler/auth/authenticate.go

@@ -75,7 +75,12 @@ func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *h
 		if settings.AuthenticationMethod == portainer.AuthenticationInternal ||
 			settings.AuthenticationMethod == portainer.AuthenticationOAuth ||
 			(settings.AuthenticationMethod == portainer.AuthenticationLDAP && !settings.LDAPSettings.AutoCreateUsers) {
-			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+			// avoid username enumeration timing attack by creating a fake user
+			// https://en.wikipedia.org/wiki/Timing_attack
+			user = &portainer.User{
+				Username: "portainer-fake-username",
+				Password: "$2a$10$abcdefghijklmnopqrstuvwx..ABCDEFGHIJKLMNOPQRSTUVWXYZ12", // fake but valid format bcrypt hash
+			}
 		}
 	}
 
@@ -112,7 +117,11 @@ func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portai
 func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
 	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
 	if err != nil {
-		return httperror.Forbidden("Only initial admin is allowed to login without oauth", err)
+		if errors.Is(err, httperrors.ErrUnauthorized) {
+			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+		}
+
+		return httperror.InternalServerError("Unable to authenticate user against LDAP", err)
 	}
 
 	if user == nil {

api/http/handler/backup/backup_test.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/portainer/portainer/api/adminmonitor"
 	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/http/offlinegate"
 	"github.com/portainer/portainer/api/internal/testhelpers"
 
@@ -55,8 +54,7 @@ func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T)
 		gate,
 		"./test_assets/handler_test",
 		func() {},
-		adminMonitor,
-		&demo.Service{}).backup(w, r)
+		adminMonitor).backup(w, r)
 	assert.Nil(t, handlerErr, "Handler should not fail")
 
 	response := w.Result()
@@ -99,8 +97,7 @@ func Test_backupHandlerWithPassword_shouldCreateEncryptedATarballArchive(t *test
 		gate,
 		"./test_assets/handler_test",
 		func() {},
-		adminMonitor,
-		&demo.Service{}).backup(w, r)
+		adminMonitor).backup(w, r)
 	assert.Nil(t, handlerErr, "Handler should not fail")
 
 	response := w.Result()

api/http/handler/backup/handler.go

@@ -6,8 +6,6 @@ import (
 
 	"github.com/portainer/portainer/api/adminmonitor"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/demo"
-	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/offlinegate"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -34,10 +32,7 @@ func NewHandler(
 	filestorePath string,
 	shutdownTrigger context.CancelFunc,
 	adminMonitor *adminmonitor.Monitor,
-	demoService *demo.Service,
-
 ) *Handler {
-
 	h := &Handler{
 		Router:          mux.NewRouter(),
 		bouncer:         bouncer,
@@ -48,11 +43,8 @@ func NewHandler(
 		adminMonitor:    adminMonitor,
 	}
 
-	demoRestrictedRouter := h.NewRoute().Subrouter()
-	demoRestrictedRouter.Use(middlewares.RestrictDemoEnv(demoService.IsDemo))
-
-	demoRestrictedRouter.Handle("/backup", bouncer.RestrictedAccess(adminAccess(httperror.LoggerHandler(h.backup)))).Methods(http.MethodPost)
-	demoRestrictedRouter.Handle("/restore", bouncer.PublicAccess(httperror.LoggerHandler(h.restore))).Methods(http.MethodPost)
+	h.Handle("/backup", bouncer.RestrictedAccess(adminAccess(httperror.LoggerHandler(h.backup)))).Methods(http.MethodPost)
+	h.Handle("/restore", bouncer.PublicAccess(httperror.LoggerHandler(h.restore))).Methods(http.MethodPost)
 
 	return h
 }

api/http/handler/backup/restore_test.go

@@ -14,7 +14,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/adminmonitor"
-	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/http/offlinegate"
 	"github.com/portainer/portainer/api/internal/testhelpers"
 
@@ -63,7 +62,6 @@ func Test_restoreArchive_usingCombinationOfPasswords(t *testing.T) {
 				"./test_assets/handler_test",
 				func() {},
 				adminMonitor,
-				&demo.Service{},
 			)
 
 			//backup
@@ -96,7 +94,6 @@ func Test_restoreArchive_shouldFailIfSystemWasAlreadyInitialized(t *testing.T) {
 		"./test_assets/handler_test",
 		func() {},
 		adminMonitor,
-		&demo.Service{},
 	)
 
 	//backup

api/http/handler/docker/containers/handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 )
@@ -30,7 +31,7 @@ func NewHandler(routePrefix string, bouncer security.BouncerService, dataStore d
 	}
 
 	router := h.PathPrefix(routePrefix).Subrouter()
-	router.Use(bouncer.AuthenticatedAccess)
+	router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
 
 	router.Handle("/{containerId}/gpus", httperror.LoggerHandler(h.containerGpusInspect)).Methods(http.MethodGet)
 	router.Handle("/{containerId}/recreate", httperror.LoggerHandler(h.recreate)).Methods(http.MethodPost)

api/http/handler/docker/dashboard.go

@@ -0,0 +1,164 @@
+package docker
+
+import (
+	"net/http"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/volume"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/docker"
+	"github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/handler/docker/utils"
+	"github.com/portainer/portainer/api/http/middlewares"
+	"github.com/portainer/portainer/api/http/security"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+type imagesCounters struct {
+	Total int   `json:"total"`
+	Size  int64 `json:"size"`
+}
+
+type dashboardResponse struct {
+	Containers docker.ContainerStats `json:"containers"`
+	Services   int                   `json:"services"`
+	Images     imagesCounters        `json:"images"`
+	Volumes    int                   `json:"volumes"`
+	Networks   int                   `json:"networks"`
+	Stacks     int                   `json:"stacks"`
+}
+
+// @id dockerDashboard
+// @summary Get counters for the dashboard
+// @description **Access policy**: restricted
+// @tags docker
+// @security jwt
+// @param environmentId path int true "Environment identifier"
+// @accept json
+// @produce json
+// @success 200 {object} dashboardResponse "Success"
+// @failure 400 "Bad request"
+// @failure 500 "Internal server error"
+// @router /docker/{environmentId}/dashboard [post]
+func (h *Handler) dashboard(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var resp dashboardResponse
+	err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
+		cli, httpErr := utils.GetClient(r, h.dockerClientFactory)
+		if httpErr != nil {
+			return httpErr
+		}
+
+		context, err := security.RetrieveRestrictedRequestContext(r)
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve user details from request context", err)
+		}
+
+		containers, err := cli.ContainerList(r.Context(), container.ListOptions{All: true})
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker containers", err)
+		}
+
+		containers, err = utils.FilterByResourceControl(tx, containers, portainer.ContainerResourceControl, context, func(c types.Container) string {
+			return c.ID
+		})
+		if err != nil {
+			return err
+		}
+
+		images, err := cli.ImageList(r.Context(), image.ListOptions{})
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker images", err)
+		}
+
+		var totalSize int64
+		for _, image := range images {
+			totalSize += image.Size
+		}
+
+		info, err := cli.Info(r.Context())
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker info", err)
+		}
+
+		isSwarmManager := info.Swarm.ControlAvailable && info.Swarm.NodeID != ""
+
+		var services []swarm.Service
+		if isSwarmManager {
+			servicesRes, err := cli.ServiceList(r.Context(), types.ServiceListOptions{})
+			if err != nil {
+				return httperror.InternalServerError("Unable to retrieve Docker services", err)
+			}
+
+			filteredServices, err := utils.FilterByResourceControl(tx, servicesRes, portainer.ServiceResourceControl, context, func(c swarm.Service) string {
+				return c.ID
+			})
+			if err != nil {
+				return err
+			}
+
+			services = filteredServices
+		}
+
+		volumesRes, err := cli.VolumeList(r.Context(), volume.ListOptions{})
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker volumes", err)
+		}
+
+		volumes, err := utils.FilterByResourceControl(tx, volumesRes.Volumes, portainer.NetworkResourceControl, context, func(c *volume.Volume) string {
+			return c.Name
+		})
+		if err != nil {
+			return err
+		}
+
+		networks, err := cli.NetworkList(r.Context(), types.NetworkListOptions{})
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker networks", err)
+		}
+
+		networks, err = utils.FilterByResourceControl(tx, networks, portainer.NetworkResourceControl, context, func(c types.NetworkResource) string {
+			return c.Name
+		})
+		if err != nil {
+			return err
+		}
+
+		environment, err := middlewares.FetchEndpoint(r)
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve environment", err)
+		}
+
+		stackCount := 0
+		if environment.SecuritySettings.AllowStackManagementForRegularUsers || context.IsAdmin {
+			stacks, err := utils.GetDockerStacks(tx, context, environment.ID, containers, services)
+			if err != nil {
+				return httperror.InternalServerError("Unable to retrieve stacks", err)
+			}
+
+			stackCount = len(stacks)
+		}
+
+		resp = dashboardResponse{
+			Images: imagesCounters{
+				Total: len(images),
+				Size:  totalSize,
+			},
+			Services:   len(services),
+			Containers: docker.CalculateContainerStats(containers),
+			Networks:   len(networks),
+			Volumes:    len(volumes),
+			Stacks:     stackCount,
+		}
+
+		return nil
+	})
+
+	return errors.TxResponse(err, func() *httperror.HandlerError {
+		return response.JSON(w, resp)
+	})
+}

api/http/handler/docker/handler.go

@@ -40,14 +40,16 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	}
 
 	// endpoints
-	endpointRouter := h.PathPrefix("/{id}").Subrouter()
-	endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
-	endpointRouter.Use(dockerOnlyMiddleware)
+	endpointRouter := h.PathPrefix("/docker/{id}").Subrouter()
+	endpointRouter.Use(bouncer.AuthenticatedAccess)
+	endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"), dockerOnlyMiddleware)
 
-	containersHandler := containers.NewHandler("/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
+	endpointRouter.Handle("/dashboard", httperror.LoggerHandler(h.dashboard)).Methods(http.MethodGet)
+
+	containersHandler := containers.NewHandler("/docker/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
 	endpointRouter.PathPrefix("/containers").Handler(containersHandler)
 
-	imagesHandler := images.NewHandler("/{id}/images", bouncer, dockerClientFactory)
+	imagesHandler := images.NewHandler("/docker/{id}/images", bouncer, dockerClientFactory)
 	endpointRouter.PathPrefix("/images").Handler(imagesHandler)
 	return h
 }

api/http/handler/docker/images/handler.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
@@ -25,7 +26,7 @@ func NewHandler(routePrefix string, bouncer security.BouncerService, dockerClien
 	}
 
 	router := h.PathPrefix(routePrefix).Subrouter()
-	router.Use(bouncer.AuthenticatedAccess)
+	router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
 
 	router.Handle("", httperror.LoggerHandler(h.imagesList)).Methods(http.MethodGet)
 	return h

api/http/handler/docker/images/images_list.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 )
 
 type ImageResponse struct {
@@ -63,7 +64,9 @@ func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *http
 
 	imageUsageSet := set.Set[string]{}
 	if withUsage {
-		containers, err := cli.ContainerList(r.Context(), types.ContainerListOptions{})
+		containers, err := cli.ContainerList(r.Context(), container.ListOptions{
+			All: true,
+		})
 		if err != nil {
 			return httperror.InternalServerError("Unable to retrieve Docker containers", err)
 		}
@@ -75,7 +78,7 @@ func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *http
 
 	imagesList := make([]ImageResponse, len(images))
 	for i, image := range images {
-		if (image.RepoTags == nil || len(image.RepoTags) == 0) && (image.RepoDigests != nil && len(image.RepoDigests) > 0) {
+		if len(image.RepoTags) == 0 && len(image.RepoDigests) > 0 {
 			for _, repoDigest := range image.RepoDigests {
 				image.RepoTags = append(image.RepoTags, repoDigest[0:strings.Index(repoDigest, "@")]+":<none>")
 			}

api/http/handler/docker/utils/filter_by_uac.go

@@ -0,0 +1,36 @@
+package utils
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/slices"
+)
+
+// filterByResourceControl filters a list of items based on the user's role and the resource control associated to the item.
+func FilterByResourceControl[T any](tx dataservices.DataStoreTx, items []T, rcType portainer.ResourceControlType, securityContext *security.RestrictedRequestContext, idGetter func(T) string) ([]T, error) {
+	if securityContext.IsAdmin {
+		return items, nil
+	}
+
+	userTeamIDs := slices.Map(securityContext.UserMemberships, func(membership portainer.TeamMembership) portainer.TeamID {
+		return membership.TeamID
+	})
+
+	filteredItems := make([]T, 0)
+	for _, item := range items {
+		resourceControl, err := tx.ResourceControl().ResourceControlByResourceIDAndType(idGetter(item), portainer.ContainerResourceControl)
+		if err != nil {
+			return nil, fmt.Errorf("Unable to retrieve resource control: %w", err)
+		}
+
+		if resourceControl == nil || authorization.UserCanAccessResource(securityContext.UserID, userTeamIDs, resourceControl) {
+			filteredItems = append(filteredItems, item)
+		}
+
+	}
+	return filteredItems, nil
+}

api/http/handler/docker/utils/get_stacks.go

@@ -0,0 +1,83 @@
+package utils
+
+import (
+	"fmt"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	portainer "github.com/portainer/portainer/api"
+	portaineree "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerconsts "github.com/portainer/portainer/api/docker/consts"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+type StackViewModel struct {
+	InternalStack *portaineree.Stack
+
+	ID         portainer.StackID
+	Name       string
+	IsExternal bool
+	Type       portainer.StackType
+}
+
+// GetDockerStacks retrieves all the stacks associated to a specific environment filtered by the user's access.
+func GetDockerStacks(tx dataservices.DataStoreTx, securityContext *security.RestrictedRequestContext, environmentID portainer.EndpointID, containers []types.Container, services []swarm.Service) ([]StackViewModel, error) {
+
+	stacks, err := tx.Stack().ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("Unable to retrieve stacks: %w", err)
+	}
+
+	stacksNameSet := map[string]*StackViewModel{}
+
+	for i := range stacks {
+		stack := stacks[i]
+		if stack.EndpointID == environmentID {
+			stacksNameSet[stack.Name] = &StackViewModel{
+				InternalStack: &stack,
+				ID:            stack.ID,
+				Name:          stack.Name,
+				IsExternal:    false,
+				Type:          stack.Type,
+			}
+		}
+	}
+
+	for _, container := range containers {
+		name := container.Labels[dockerconsts.ComposeStackNameLabel]
+
+		if name != "" && stacksNameSet[name] == nil && !isHiddenStack(container.Labels) {
+			stacksNameSet[name] = &StackViewModel{
+				Name:       name,
+				IsExternal: true,
+				Type:       portainer.DockerComposeStack,
+			}
+		}
+	}
+
+	for _, service := range services {
+		name := service.Spec.Labels[dockerconsts.SwarmStackNameLabel]
+
+		if name != "" && stacksNameSet[name] == nil && !isHiddenStack(service.Spec.Labels) {
+			stacksNameSet[name] = &StackViewModel{
+				Name:       name,
+				IsExternal: true,
+				Type:       portainer.DockerSwarmStack,
+			}
+		}
+	}
+
+	stacksList := make([]StackViewModel, 0)
+	for _, stack := range stacksNameSet {
+		stacksList = append(stacksList, *stack)
+	}
+
+	return FilterByResourceControl(tx, stacksList, portainer.StackResourceControl, securityContext, func(c StackViewModel) string {
+		return c.Name
+	})
+}
+
+func isHiddenStack(labels map[string]string) bool {
+	return labels[dockerconsts.HideStackLabel] != ""
+}

api/http/handler/docker/utils/get_stacks_test.go

@@ -0,0 +1,96 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	portainer "github.com/portainer/portainer/api"
+	portaineree "github.com/portainer/portainer/api"
+	dockerconsts "github.com/portainer/portainer/api/docker/consts"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandler_getDockerStacks(t *testing.T) {
+	environment := &portaineree.Endpoint{
+		ID: 1,
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowStackManagementForRegularUsers: true,
+		},
+	}
+
+	containers := []types.Container{
+		{
+			Labels: map[string]string{
+				dockerconsts.ComposeStackNameLabel: "stack1",
+			},
+		},
+		{
+			Labels: map[string]string{
+				dockerconsts.ComposeStackNameLabel: "stack2",
+			},
+		},
+	}
+
+	services := []swarm.Service{
+		{
+			Spec: swarm.ServiceSpec{
+				Annotations: swarm.Annotations{
+					Labels: map[string]string{
+						dockerconsts.SwarmStackNameLabel: "stack3",
+					},
+				},
+			},
+		},
+	}
+
+	stack1 := portaineree.Stack{
+		ID:         1,
+		Name:       "stack1",
+		EndpointID: 1,
+		Type:       portainer.DockerComposeStack,
+	}
+
+	datastore := testhelpers.NewDatastore(
+		testhelpers.WithEndpoints([]portaineree.Endpoint{*environment}),
+		testhelpers.WithStacks([]portaineree.Stack{
+			stack1,
+			{
+				ID:         2,
+				Name:       "stack2",
+				EndpointID: 2,
+				Type:       portainer.DockerSwarmStack,
+			},
+		}),
+	)
+
+	stacksList, err := GetDockerStacks(datastore, &security.RestrictedRequestContext{
+		IsAdmin: true,
+	}, environment.ID, containers, services)
+	assert.NoError(t, err)
+	assert.Len(t, stacksList, 3)
+
+	expectedStacks := []StackViewModel{
+		{
+			InternalStack: &stack1,
+			ID:            1,
+			Name:          "stack1",
+			IsExternal:    false,
+			Type:          portainer.DockerComposeStack,
+		},
+		{
+			Name:       "stack2",
+			IsExternal: true,
+			Type:       portainer.DockerComposeStack,
+		},
+		{
+			Name:       "stack3",
+			IsExternal: true,
+			Type:       portainer.DockerSwarmStack,
+		},
+	}
+
+	assert.ElementsMatch(t, expectedStacks, stacksList)
+}

api/http/handler/edgegroups/edgegroup_delete.go

@@ -19,8 +19,9 @@ import (
 // @security jwt
 // @param id path int true "EdgeGroup Id"
 // @success 204
+// @failure 409 "Edge group is in use by an Edge stack or Edge job"
 // @failure 503 "Edge compute features are disabled"
-// @failure 500
+// @failure 500 "Server error"
 // @router /edge_groups/{id} [delete]
 func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgejobs/edgejob_update.go

@@ -48,7 +48,7 @@ func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
 // @failure 500
 // @failure 400
 // @failure 503 "Edge compute features are disabled"
-// @router /edge_jobs/{id} [post]
+// @router /edge_jobs/{id} [put]
 func (handler *Handler) edgeJobUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/edgestacks/edgestack_status_update.go

@@ -135,6 +135,11 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht
 }
 
 func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeStack, deploymentStatus portainer.EdgeStackDeploymentStatus) {
+	if deploymentStatus.Type == portainer.EdgeStackStatusRemoved {
+		delete(stack.Status, environmentId)
+		return
+	}
+
 	environmentStatus, ok := stack.Status[environmentId]
 	if !ok {
 		environmentStatus = portainer.EdgeStackStatus{

api/http/handler/endpointedge/endpointedge_status_inspect.go

@@ -15,6 +15,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -134,7 +135,7 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
 
 	// Take an initial snapshot
 	if endpoint.LastCheckInDate == 0 {
-		handler.ReverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+		handler.ReverseTunnelService.Open(endpoint)
 	}
 
 	agentPlatform, agentPlatformErr := parseAgentPlatform(r)
@@ -153,34 +154,21 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
 		return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err)
 	}
 
-	checkinInterval := endpoint.EdgeCheckinInterval
-	if endpoint.EdgeCheckinInterval == 0 {
-		settings, err := tx.Settings().Settings()
-		if err != nil {
-			return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
-		}
-		checkinInterval = settings.EdgeAgentCheckinInterval
-	}
-
-	tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)
+	tunnel := handler.ReverseTunnelService.Config(endpoint.ID)
 
 	statusResponse := endpointEdgeStatusInspectResponse{
 		Status:          tunnel.Status,
 		Port:            tunnel.Port,
-		CheckinInterval: checkinInterval,
+		CheckinInterval: edge.EffectiveCheckinInterval(tx, endpoint),
 		Credentials:     tunnel.Credentials,
 	}
 
-	schedules, handlerErr := handler.buildSchedules(endpoint.ID, tunnel)
+	schedules, handlerErr := handler.buildSchedules(endpoint.ID)
 	if handlerErr != nil {
 		return nil, handlerErr
 	}
 	statusResponse.Schedules = schedules
 
-	if tunnel.Status == portainer.EdgeAgentManagementRequired {
-		handler.ReverseTunnelService.SetTunnelStatusToActive(endpoint.ID)
-	}
-
 	edgeStacksStatus, handlerErr := handler.buildEdgeStacks(tx, endpoint.ID)
 	if handlerErr != nil {
 		return nil, handlerErr
@@ -213,9 +201,9 @@ func parseAgentPlatform(r *http.Request) (portainer.EndpointType, error) {
 	}
 }
 
-func (handler *Handler) buildSchedules(endpointID portainer.EndpointID, tunnel portainer.TunnelDetails) ([]edgeJobResponse, *httperror.HandlerError) {
+func (handler *Handler) buildSchedules(endpointID portainer.EndpointID) ([]edgeJobResponse, *httperror.HandlerError) {
 	schedules := []edgeJobResponse{}
-	for _, job := range tunnel.Jobs {
+	for _, job := range handler.ReverseTunnelService.EdgeJobs(endpointID) {
 		var collectLogs bool
 		if _, ok := job.GroupLogsCollection[endpointID]; ok {
 			collectLogs = job.GroupLogsCollection[endpointID].CollectLogs

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -8,6 +8,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/tag"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -156,11 +157,7 @@ func (handler *Handler) updateEndpointGroup(tx dataservices.DataStoreTx, endpoin
 					if err != nil {
 						// Update flag with endpoint and continue
 						go func(endpointID portainer.EndpointID, endpointGroupID portainer.EndpointGroupID) {
-							err := handler.PendingActionsService.Create(portainer.PendingActions{
-								EndpointID: endpointID,
-								Action:     "CleanNAPWithOverridePolicies",
-								ActionData: endpointGroupID,
-							})
+							err := handler.PendingActionsService.Create(handlers.NewCleanNAPWithOverridePolicies(endpointID, &endpointGroupID))
 							if err != nil {
 								log.Error().Err(err).Msgf("Unable to create pending action to clean NAP with override policies for endpoint (%d) and endpoint group (%d).", endpointID, endpointGroupID)
 							}

api/http/handler/endpointproxy/proxy_docker.go

@@ -34,7 +34,7 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 			return httperror.InternalServerError("No Edge agent registered with the environment", errors.New("No agent available"))
 		}
 
-		_, err := handler.ReverseTunnelService.GetActiveTunnel(endpoint)
+		_, err := handler.ReverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return httperror.InternalServerError("Unable to get the active tunnel", err)
 		}

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -34,7 +34,7 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 			return httperror.InternalServerError("No Edge agent registered with the environment", errors.New("No agent available"))
 		}
 
-		_, err := handler.ReverseTunnelService.GetActiveTunnel(endpoint)
+		_, err := handler.ReverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return httperror.InternalServerError("Unable to get the active tunnel", err)
 		}

api/http/handler/endpoints/endpoint_association_delete.go

@@ -59,8 +59,6 @@ func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http
 		return httperror.InternalServerError("Failed persisting environment in database", err)
 	}
 
-	handler.ReverseTunnelService.SetTunnelStatusToIdle(endpoint.ID)
-
 	return response.Empty(w)
 }
 

api/http/handler/endpoints/endpoint_create.go

@@ -201,6 +201,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @param Gpus formData string false "List of GPUs - json stringified array of {name, value} structs"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
+// @failure 409 "Name is not unique"
 // @failure 500 "Server error"
 // @router /endpoints [post]
 func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/endpoints/endpoint_create_global_key_test.go

@@ -10,10 +10,7 @@ import (
 )
 
 func TestEmptyGlobalKey(t *testing.T) {
-	handler := NewHandler(
-		helper.NewTestRequestBouncer(),
-		nil,
-	)
+	handler := NewHandler(helper.NewTestRequestBouncer())
 
 	req, err := http.NewRequest(http.MethodPost, "https://portainer.io:9443/endpoints/global-key", nil)
 	if err != nil {

api/http/handler/endpoints/endpoint_delete.go

@@ -2,13 +2,13 @@ package endpoints
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"slices"
 	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -17,19 +17,40 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+type endpointDeleteRequest struct {
+	ID            int  `json:"id"`
+	DeleteCluster bool `json:"deleteCluster"`
+}
+
+type endpointDeleteBatchPayload struct {
+	Endpoints []endpointDeleteRequest `json:"endpoints"`
+}
+
+type endpointDeleteBatchPartialResponse struct {
+	Deleted []int `json:"deleted"`
+	Errors  []int `json:"errors"`
+}
+
+func (payload *endpointDeleteBatchPayload) Validate(r *http.Request) error {
+	if payload == nil || len(payload.Endpoints) == 0 {
+		return fmt.Errorf("invalid request payload. You must provide a list of environments to delete")
+	}
+
+	return nil
+}
+
 // @id EndpointDelete
-// @summary Remove an environment(endpoint)
-// @description Remove an environment(endpoint).
-// @description **Access policy**: administrator
+// @summary Remove an environment
+// @description Remove the environment associated to the specified identifier and optionally clean-up associated resources.
+// @description **Access policy**: Administrator only.
 // @tags endpoints
-// @security ApiKeyAuth
-// @security jwt
+// @security ApiKeyAuth || jwt
 // @param id path int true "Environment(Endpoint) identifier"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "Environment(Endpoint) not found"
-// @failure 500 "Server error"
+// @success 204 "Environment successfully deleted."
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 403 "Unauthorized access or operation not allowed."
+// @failure 404 "Unable to find the environment with the specified identifier inside the database."
+// @failure 500 "Server error occurred while attempting to delete the environment."
 // @router /endpoints/{id} [delete]
 func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
@@ -43,10 +64,6 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		return httperror.BadRequest("Invalid boolean query parameter", err)
 	}
 
-	if handler.demoService.IsDemoEnvironment(portainer.EndpointID(endpointID)) {
-		return httperror.Forbidden(httperrors.ErrNotAvailableInDemo.Error(), httperrors.ErrNotAvailableInDemo)
-	}
-
 	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
 		return handler.deleteEndpoint(tx, portainer.EndpointID(endpointID), deleteCluster)
 	})
@@ -62,6 +79,56 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 	return response.Empty(w)
 }
 
+// @id EndpointDeleteBatch
+// @summary Remove multiple environments
+// @description Remove multiple environments and optionally clean-up associated resources.
+// @description **Access policy**: Administrator only.
+// @tags endpoints
+// @security ApiKeyAuth || jwt
+// @accept json
+// @produce json
+// @param body body endpointDeleteBatchPayload true "List of environments to delete, with optional deleteCluster flag to clean-up assocaited resources (cloud environments only)"
+// @success 204 "Environment(s) successfully deleted."
+// @failure 207 {object} endpointDeleteBatchPartialResponse "Partial success. Some environments were deleted successfully, while others failed."
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 403 "Unauthorized access or operation not allowed."
+// @failure 500 "Server error occurred while attempting to delete the specified environments."
+// @router /endpoints [delete]
+func (handler *Handler) endpointDeleteBatch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var p endpointDeleteBatchPayload
+	if err := request.DecodeAndValidateJSONPayload(r, &p); err != nil {
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	resp := endpointDeleteBatchPartialResponse{
+		Deleted: []int{},
+		Errors:  []int{},
+	}
+
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		for _, e := range p.Endpoints {
+			if err := handler.deleteEndpoint(tx, portainer.EndpointID(e.ID), e.DeleteCluster); err != nil {
+				resp.Errors = append(resp.Errors, e.ID)
+				log.Warn().Err(err).Int("environment_id", e.ID).Msg("Unable to remove environment")
+
+				continue
+			}
+
+			resp.Deleted = append(resp.Deleted, e.ID)
+		}
+
+		return nil
+	}); err != nil {
+		return httperror.InternalServerError("Unable to delete environments", err)
+	}
+
+	if len(resp.Errors) > 0 {
+		return response.JSONWithStatus(w, resp, http.StatusPartialContent)
+	}
+
+	return response.Empty(w)
+}
+
 func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID portainer.EndpointID, deleteCluster bool) error {
 	endpoint, err := tx.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if tx.IsErrObjectNotFound(err) {
@@ -179,6 +246,12 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 		}
 	}
 
+	// delete the pending actions
+	err = tx.PendingActions().DeleteByEndpointID(endpoint.ID)
+	if err != nil {
+		log.Warn().Err(err).Int("endpointId", int(endpoint.ID)).Msgf("Unable to delete pending actions")
+	}
+
 	err = tx.Endpoint().DeleteEndpoint(portainer.EndpointID(endpointID))
 	if err != nil {
 		return httperror.InternalServerError("Unable to delete the environment from the database", err)

api/http/handler/endpoints/endpoint_delete_test.go

@@ -9,7 +9,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
-	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/internal/testhelpers"
 )
@@ -19,9 +18,10 @@ func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
 
 	_, store := datastore.MustNewTestStore(t, true, false)
 
-	handler := NewHandler(testhelpers.NewTestRequestBouncer(), demo.NewService())
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
 	handler.DataStore = store
-	handler.ProxyManager = proxy.NewManager(nil, nil, nil, nil, nil, nil, nil)
+	handler.ProxyManager = proxy.NewManager(nil)
+	handler.ProxyManager.NewProxyFactory(nil, nil, nil, nil, nil, nil, nil, nil)
 
 	// Create all the environments and add them to the same edge group
 

api/http/handler/endpoints/endpoint_force_update_service.go

@@ -11,9 +11,10 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/rs/zerolog/log"
 
-	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 )
 
@@ -39,7 +40,7 @@ func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
 // @produce json
 // @param id path int true "endpoint identifier"
 // @param body body forceUpdateServicePayload true "details"
-// @success 200 {object} dockertypes.ServiceUpdateResponse "Success"
+// @success 200 {object} swarm.ServiceUpdateResponse "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 404 "endpoint not found"
@@ -94,10 +95,14 @@ func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *htt
 	go func() {
 		images.EvictImageStatus(payload.ServiceID)
 		images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
-		containers, _ := dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{
+		// ignore errors from this cleanup function, log them instead
+		containers, err := dockerClient.ContainerList(context.TODO(), container.ListOptions{
 			All:     true,
 			Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+payload.ServiceID)),
 		})
+		if err != nil {
+			log.Warn().Err(err).Str("Environment", endpoint.Name).Msg("Error listing containers")
+		}
 
 		for _, container := range containers {
 			images.EvictImageStatus(container.ID)

api/http/handler/endpoints/endpoint_list_test.go

@@ -194,7 +194,7 @@ func setupEndpointListHandler(t *testing.T, endpoints []portainer.Endpoint) *Han
 
 	bouncer := testhelpers.NewTestRequestBouncer()
 
-	handler := NewHandler(bouncer, nil)
+	handler := NewHandler(bouncer)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()
 	handler.SnapshotService, _ = snapshot.NewService("1s", store, nil, nil, nil, nil)

api/http/handler/endpoints/endpoint_update.go

@@ -9,6 +9,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -69,6 +70,7 @@ func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
+// @failure 409 "Name is not unique"
 // @failure 500 "Server error"
 // @router /endpoints/{id} [put]
 func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -265,11 +267,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
 			err = handler.AuthorizationService.CleanNAPWithOverridePolicies(handler.DataStore, endpoint, nil)
 			if err != nil {
-				handler.PendingActionsService.Create(portainer.PendingActions{
-					EndpointID: endpoint.ID,
-					Action:     "CleanNAPWithOverridePolicies",
-					ActionData: nil,
-				})
+				handler.PendingActionsService.Create(handlers.NewCleanNAPWithOverridePolicies(endpoint.ID, nil))
 				log.Warn().Err(err).Msgf("Unable to clean NAP with override policies for endpoint (%d). Will try to update when endpoint is online.", endpoint.ID)
 			}
 		}

api/http/handler/endpoints/filter.go

@@ -334,11 +334,16 @@ func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portai
 		status := endpoint.Status
 		if endpointutils.IsEdgeEndpoint(&endpoint) {
 			isCheckValid := false
+
 			edgeCheckinInterval := endpoint.EdgeCheckinInterval
-			if endpoint.EdgeCheckinInterval == 0 {
+			if edgeCheckinInterval == 0 {
 				edgeCheckinInterval = settings.EdgeAgentCheckinInterval
 			}
 
+			if endpoint.Edge.AsyncMode {
+				edgeCheckinInterval = getShortestAsyncInterval(&endpoint, settings)
+			}
+
 			if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 {
 				isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd)
 			}
@@ -622,9 +627,36 @@ func getEdgeStackStatusParam(r *http.Request) (*portainer.EdgeStackStatusType, e
 		portainer.EdgeStackStatusRunning,
 		portainer.EdgeStackStatusDeploying,
 		portainer.EdgeStackStatusRemoving,
+		portainer.EdgeStackStatusCompleted,
 	}, edgeStackStatus) {
 		return nil, errors.New("invalid edgeStackStatus parameter")
 	}
 
 	return &edgeStackStatus, nil
 }
+
+func getShortestAsyncInterval(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
+	var edgeIntervalUseDefault int = -1
+	pingInterval := endpoint.Edge.PingInterval
+	if pingInterval == edgeIntervalUseDefault {
+		pingInterval = settings.Edge.PingInterval
+	}
+	shortestAsyncInterval := pingInterval
+
+	snapshotInterval := endpoint.Edge.SnapshotInterval
+	if snapshotInterval == edgeIntervalUseDefault {
+		snapshotInterval = settings.Edge.SnapshotInterval
+	}
+	if shortestAsyncInterval > snapshotInterval {
+		shortestAsyncInterval = snapshotInterval
+	}
+
+	commandInterval := endpoint.Edge.CommandInterval
+	if commandInterval == edgeIntervalUseDefault {
+		commandInterval = settings.Edge.CommandInterval
+	}
+	if shortestAsyncInterval > commandInterval {
+		shortestAsyncInterval = commandInterval
+	}
+	return shortestAsyncInterval
+}

api/http/handler/endpoints/filter_test.go

@@ -194,7 +194,7 @@ func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) *Handler {
 	is.NoError(err, "error creating a user")
 
 	bouncer := testhelpers.NewTestRequestBouncer()
-	handler := NewHandler(bouncer, nil)
+	handler := NewHandler(bouncer)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()
 

api/http/handler/endpoints/handler.go

@@ -5,7 +5,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/demo"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
@@ -28,7 +27,6 @@ func hideFields(endpoint *portainer.Endpoint) {
 type Handler struct {
 	*mux.Router
 	requestBouncer        security.BouncerService
-	demoService           *demo.Service
 	DataStore             dataservices.DataStore
 	FileService           portainer.FileService
 	ProxyManager          *proxy.Manager
@@ -44,11 +42,10 @@ type Handler struct {
 }
 
 // NewHandler creates a handler to manage environment(endpoint) operations.
-func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Handler {
+func NewHandler(bouncer security.BouncerService) *Handler {
 	h := &Handler{
 		Router:         mux.NewRouter(),
 		requestBouncer: bouncer,
-		demoService:    demoService,
 	}
 
 	h.Handle("/endpoints",
@@ -71,6 +68,8 @@ func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Han
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoints/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoints",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDeleteBatch))).Methods(http.MethodDelete)
 	h.Handle("/endpoints/{id}/dockerhub/{registryId}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
 	h.Handle("/endpoints/{id}/snapshot",

api/http/handler/handler.go

@@ -85,7 +85,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.21.0
+// @version 2.22.0
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -199,7 +199,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case strings.HasPrefix(r.URL.Path, "/api/kubernetes"):
 		http.StripPrefix("/api", h.KubernetesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/docker"):
-		http.StripPrefix("/api/docker", h.DockerHandler).ServeHTTP(w, r)
+		http.StripPrefix("/api", h.DockerHandler).ServeHTTP(w, r)
 
 	// Helm subpath under kubernetes -> /api/endpoints/{id}/kubernetes/helm
 	case strings.HasPrefix(r.URL.Path, "/api/endpoints/") && strings.Contains(r.URL.Path, "/kubernetes/helm"):

api/http/handler/helm/handler.go

@@ -38,19 +38,20 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 		kubeClusterAccessService: kubeClusterAccessService,
 	}
 
-	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
+	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"),
+		bouncer.AuthenticatedAccess)
 
 	// `helm list -o json`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmList))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmList)).Methods(http.MethodGet)
 
 	// `helm delete RELEASE_NAME`
 	h.Handle("/{id}/kubernetes/helm/{release}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmDelete))).Methods(http.MethodDelete)
+		httperror.LoggerHandler(h.helmDelete)).Methods(http.MethodDelete)
 
 	// `helm install [NAME] [CHART] flags`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmInstall))).Methods(http.MethodPost)
+		httperror.LoggerHandler(h.helmInstall)).Methods(http.MethodPost)
 
 	// Deprecated
 	h.Handle("/{id}/kubernetes/helm/repositories",
@@ -69,12 +70,14 @@ func NewTemplateHandler(bouncer security.BouncerService, helmPackageManager libh
 		requestBouncer:     bouncer,
 	}
 
+	h.Use(bouncer.AuthenticatedAccess)
+
 	h.Handle("/templates/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmRepoSearch))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmRepoSearch)).Methods(http.MethodGet)
 
 	// helm show [COMMAND] [CHART] [REPO] flags
 	h.Handle("/templates/helm/{command:chart|values|readme}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmShow))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmShow)).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/helm/helm_install.go

@@ -61,8 +61,7 @@ func (handler *Handler) helmInstall(w http.ResponseWriter, r *http.Request) *htt
 		return httperror.InternalServerError("Unable to install a chart", err)
 	}
 
-	w.WriteHeader(http.StatusCreated)
-	return response.JSON(w, release)
+	return response.JSONWithStatus(w, release, http.StatusCreated)
 }
 
 func (p *installChartPayload) Validate(_ *http.Request) error {

api/http/handler/hostmanagement/openamt/amtrpc.go

@@ -155,7 +155,7 @@ func pullImage(ctx context.Context, docker *client.Client, imageName string) err
 // runContainer should be used to run a short command that returns information to stdout
 // TODO: add k8s support
 func runContainer(ctx context.Context, docker *client.Client, imageName, containerName string, cmdLine []string) (output string, err error) {
-	opts := types.ContainerListOptions{All: true}
+	opts := container.ListOptions{All: true}
 	opts.Filters = filters.NewArgs()
 	opts.Filters.Add("name", containerName)
 	existingContainers, err := docker.ContainerList(ctx, opts)
@@ -170,7 +170,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 	}
 
 	if len(existingContainers) > 0 {
-		err = docker.ContainerRemove(ctx, existingContainers[0].ID, types.ContainerRemoveOptions{Force: true})
+		err = docker.ContainerRemove(ctx, existingContainers[0].ID, container.RemoveOptions{Force: true})
 		if err != nil {
 			log.Error().
 				Str("image_name", imageName).
@@ -211,7 +211,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 		return "", err
 	}
 
-	err = docker.ContainerStart(ctx, created.ID, types.ContainerStartOptions{})
+	err = docker.ContainerStart(ctx, created.ID, container.StartOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).
@@ -243,14 +243,14 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 
 	log.Debug().Int64("status", statusCode).Msg("container wait status")
 
-	out, err := docker.ContainerLogs(ctx, created.ID, types.ContainerLogsOptions{ShowStdout: true})
+	out, err := docker.ContainerLogs(ctx, created.ID, container.LogsOptions{ShowStdout: true})
 	if err != nil {
 		log.Error().Err(err).Str("image_name", imageName).Str("container_name", containerName).Msg("getting container log")
 
 		return "", err
 	}
 
-	err = docker.ContainerRemove(ctx, created.ID, types.ContainerRemoveOptions{})
+	err = docker.ContainerRemove(ctx, created.ID, container.RemoveOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).

api/http/handler/kubernetes/dashboard.go

@@ -0,0 +1,37 @@
+package kubernetes
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+// @id GetKubernetesDashboard
+// @summary Get the dashboard summary data
+// @description Get the dashboard summary data which is simply a count of a range of different commonly used kubernetes resources
+// @description **Access policy**: authenticated
+// @tags kubernetes
+// @security ApiKeyAuth
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "Environment (Endpoint) identifier"
+// @success 200 {array} kubernetes.K8sDashboard "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /kubernetes/{id}/dashboard [get]
+func (handler *Handler) getKubernetesDashboard(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+
+	cli, httpErr := handler.getProxyKubeClient(r)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	dashboard, err := cli.GetDashboard()
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve dashboard data", err)
+	}
+
+	return response.JSON(w, dashboard)
+}

api/http/handler/kubernetes/handler.go

@@ -52,6 +52,7 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	endpointRouter.Use(kubeOnlyMiddleware)
 	endpointRouter.Use(h.kubeClientMiddleware)
 
+	endpointRouter.Handle("/dashboard", httperror.LoggerHandler(h.getKubernetesDashboard)).Methods(http.MethodGet)
 	endpointRouter.Handle("/nodes_limits", httperror.LoggerHandler(h.getKubernetesNodesLimits)).Methods(http.MethodGet)
 	endpointRouter.Handle("/max_resource_limits", httperror.LoggerHandler(h.getKubernetesMaxResourceLimits)).Methods(http.MethodGet)
 	endpointRouter.Handle("/metrics/nodes", httperror.LoggerHandler(h.getKubernetesMetricsForAllNodes)).Methods(http.MethodGet)

api/http/handler/registries/registry_create.go

@@ -89,6 +89,7 @@ func (payload *registryCreatePayload) Validate(_ *http.Request) error {
 // @param body body registryCreatePayload true "Registry details"
 // @success 200 {object} portainer.Registry "Success"
 // @failure 400 "Invalid request"
+// @failure 409 "Another registry with the same name or same URL & credentials already exists"
 // @failure 500 "Server error"
 // @router /registries [post]
 func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/registries/registry_delete.go

@@ -7,7 +7,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/pendingactions"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -89,17 +89,9 @@ func (handler *Handler) deleteKubernetesSecrets(registry *portainer.Registry) er
 			}
 
 			if len(failedNamespaces) > 0 {
-				handler.PendingActionsService.Create(portainer.PendingActions{
-					EndpointID: endpointId,
-					Action:     pendingactions.DeletePortainerK8sRegistrySecrets,
-
-					// When extracting the data, this is the type we need to pull out
-					// i.e. pendingactions.DeletePortainerK8sRegistrySecretsData
-					ActionData: pendingactions.DeletePortainerK8sRegistrySecretsData{
-						RegistryID: registry.ID,
-						Namespaces: failedNamespaces,
-					},
-				})
+				handler.PendingActionsService.Create(
+					handlers.NewDeleteK8sRegistrySecrets(portainer.EndpointID(endpointId), registry.ID, failedNamespaces),
+				)
 			}
 		}
 	}

api/http/handler/registries/registry_update.go

@@ -52,7 +52,7 @@ func (payload *registryUpdatePayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.Registry "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Registry not found"
-// @failure 409 "Another registry with the same URL already exists"
+// @failure 409 "Another registry with the same name or same URL & credentials already exists"
 // @failure 500 "Server error"
 // @router /registries/{id} [put]
 func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/resourcecontrols/resourcecontrol_create.go

@@ -63,7 +63,7 @@ func (payload *resourceControlCreatePayload) Validate(r *http.Request) error {
 // @param body body resourceControlCreatePayload true "Resource control details"
 // @success 200 {object} portainer.ResourceControl "Success"
 // @failure 400 "Invalid request"
-// @failure 409 "Resource control already exists"
+// @failure 409 "A resource control is already associated to this resource"
 // @failure 500 "Server error"
 // @router /resource_controls [post]
 func (handler *Handler) resourceControlCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/settings/handler.go

@@ -5,19 +5,12 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/gorilla/mux"
 )
 
-func hideFields(settings *portainer.Settings) {
-	settings.LDAPSettings.Password = ""
-	settings.OAuthSettings.ClientSecret = ""
-	settings.OAuthSettings.KubeSecretKey = nil
-}
-
 // Handler is the HTTP handler used to handle settings operations.
 type Handler struct {
 	*mux.Router
@@ -26,21 +19,25 @@ type Handler struct {
 	JWTService      portainer.JWTService
 	LDAPService     portainer.LDAPService
 	SnapshotService portainer.SnapshotService
-	demoService     *demo.Service
 }
 
 // NewHandler creates a handler to manage settings operations.
-func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Handler {
+func NewHandler(bouncer security.BouncerService) *Handler {
 	h := &Handler{
-		Router:      mux.NewRouter(),
-		demoService: demoService,
+		Router: mux.NewRouter(),
 	}
-	h.Handle("/settings",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsInspect))).Methods(http.MethodGet)
-	h.Handle("/settings",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsUpdate))).Methods(http.MethodPut)
-	h.Handle("/settings/public",
-		bouncer.PublicAccess(httperror.LoggerHandler(h.settingsPublic))).Methods(http.MethodGet)
+
+	adminRouter := h.NewRoute().Subrouter()
+	adminRouter.Use(bouncer.AdminAccess)
+	adminRouter.Handle("/settings", httperror.LoggerHandler(h.settingsUpdate)).Methods(http.MethodPut)
+
+	authenticatedRouter := h.NewRoute().Subrouter()
+	authenticatedRouter.Use(bouncer.AuthenticatedAccess)
+	authenticatedRouter.Handle("/settings", httperror.LoggerHandler(h.settingsInspect)).Methods(http.MethodGet)
+
+	publicRouter := h.NewRoute().Subrouter()
+	publicRouter.Use(bouncer.PublicAccess)
+	publicRouter.Handle("/settings/public", httperror.LoggerHandler(h.settingsPublic)).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/settings/settings_inspect.go

@@ -3,27 +3,45 @@ package settings
 import (
 	"net/http"
 
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/rbacutils"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/http/utils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
-	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
 // @id SettingsInspect
-// @summary Retrieve Portainer settings
-// @description Retrieve Portainer settings.
-// @description **Access policy**: administrator
+// @summary Retrieve the settings of the Portainer instance
+// @description Get the settings of the Portainer instance. Will return either all the settings or a subset of settings based on the user role.
+// @description **Access policy**: Authenticated user.
 // @tags settings
-// @security ApiKeyAuth
-// @security jwt
+// @security ApiKeyAuth || jwt
 // @produce json
-// @success 200 {object} portainer.Settings "Success"
-// @failure 500 "Server error"
+// @success 200 {object} settingsInspectResponse "The settings object"
+// @failure 401 "Unauthorized access or operation not allowed."
+// @failure 500 "Server error occurred while attempting to retrieve the settings."
 // @router /settings [get]
 func (handler *Handler) settingsInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	settings, err := handler.DataStore.Settings().Settings()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve the settings from the database", err)
-	}
+	var roleBasedResponse interface{}
+	err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
+		settings, err := tx.Settings().Settings()
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve the settings from the database", err)
+		}
 
-	hideFields(settings)
-	return response.JSON(w, settings)
+		user, err := security.RetrieveUserFromRequest(r, tx)
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve user details from request", err)
+		}
+
+		response := buildResponse(settings)
+
+		role := rbacutils.RoleFromUser(user)
+
+		roleBasedResponse = response.ForRole(role)
+
+		return nil
+	})
+
+	return utils.TxResponse(w, roleBasedResponse, err)
 }

api/http/handler/settings/settings_inspect_response_helper.go

@@ -0,0 +1,300 @@
+package settings
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/pkg/featureflags"
+	"golang.org/x/oauth2"
+)
+
+type settingsInspectResponse struct {
+	adminResponse
+}
+
+type authenticatedResponse struct {
+	publicSettingsResponse
+
+	// Deployment options for encouraging git ops workflows
+	GlobalDeploymentOptions portainer.GlobalDeploymentOptions `json:"GlobalDeploymentOptions"`
+	// Whether edge compute features are enabled
+	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures"`
+	// The expiry of a Kubeconfig
+	KubeconfigExpiry string `json:"KubeconfigExpiry" example:"24h"`
+
+	// Helm repository URL, defaults to "https://charts.bitnami.com/bitnami"
+	HelmRepositoryURL string `json:"HelmRepositoryURL" example:"https://charts.bitnami.com/bitnami"`
+
+	IsAMTEnabled bool `json:"isAMTEnabled"`
+	IsFDOEnabled bool `json:"isFDOEnabled"`
+}
+
+type edgeSettings struct {
+	// The command list interval for edge agent - used in edge async mode (in seconds)
+	CommandInterval int `json:"CommandInterval" example:"5"`
+	// The ping interval for edge agent - used in edge async mode (in seconds)
+	PingInterval int `json:"PingInterval" example:"5"`
+	// The snapshot interval for edge agent - used in edge async mode (in seconds)
+	SnapshotInterval int `json:"SnapshotInterval" example:"5"`
+}
+
+type edgeAdminResponse struct {
+	authenticatedResponse
+
+	Edge edgeSettings
+
+	// TrustOnFirstConnect makes Portainer accepting edge agent connection by default
+	TrustOnFirstConnect bool `json:"TrustOnFirstConnect" example:"false"`
+	// EnforceEdgeID makes Portainer store the Edge ID instead of accepting anyone
+	EnforceEdgeID bool `json:"EnforceEdgeID" example:"false"`
+
+	// EdgePortainerURL is the URL that is exposed to edge agents
+	EdgePortainerURL string `json:"EdgePortainerUrl"`
+
+	// The default check in interval for edge agent (in seconds)
+	EdgeAgentCheckinInterval int `json:"EdgeAgentCheckinInterval" example:"5"`
+}
+
+type oauthSettings struct {
+	ClientID             string           `json:"ClientID"`
+	AccessTokenURI       string           `json:"AccessTokenURI"`
+	AuthorizationURI     string           `json:"AuthorizationURI"`
+	ResourceURI          string           `json:"ResourceURI"`
+	RedirectURI          string           `json:"RedirectURI"`
+	UserIdentifier       string           `json:"UserIdentifier"`
+	Scopes               string           `json:"Scopes"`
+	OAuthAutoCreateUsers bool             `json:"OAuthAutoCreateUsers"`
+	DefaultTeamID        portainer.TeamID `json:"DefaultTeamID"`
+	SSO                  bool             `json:"SSO"`
+	LogoutURI            string           `json:"LogoutURI"`
+	AuthStyle            oauth2.AuthStyle `json:"AuthStyle"`
+}
+
+type ldapSettings struct {
+	// Enable this option if the server is configured for Anonymous access. When enabled, ReaderDN and Password will not be used
+	AnonymousMode bool `json:"AnonymousMode" example:"true" validate:"validate_bool"`
+	// Account that will be used to search for users
+	ReaderDN string `json:"ReaderDN" example:"cn=readonly-account,dc=ldap,dc=domain,dc=tld" validate:"required_if=AnonymousMode false"`
+	// URL or IP address of the LDAP server
+	URL       string                     `json:"URL" example:"myldap.domain.tld:389" validate:"hostname_port"`
+	TLSConfig portainer.TLSConfiguration `json:"TLSConfig"`
+	// Whether LDAP connection should use StartTLS
+	StartTLS            bool                                `json:"StartTLS" example:"true"`
+	SearchSettings      []portainer.LDAPSearchSettings      `json:"SearchSettings"`
+	GroupSearchSettings []portainer.LDAPGroupSearchSettings `json:"GroupSearchSettings"`
+	// Automatically provision users and assign them to matching LDAP group names
+	AutoCreateUsers bool `json:"AutoCreateUsers" example:"true"`
+}
+
+type adminResponse struct {
+	edgeAdminResponse
+	// A list of label name & value that will be used to hide containers when querying containers
+	BlackListedLabels []portainer.Pair `json:"BlackListedLabels"`
+
+	LDAPSettings         ldapSettings                   `json:"LDAPSettings"`
+	OAuthSettings        oauthSettings                  `json:"OAuthSettings"`
+	InternalAuthSettings portainer.InternalAuthSettings `json:"InternalAuthSettings"`
+	OpenAMTConfiguration portainer.OpenAMTConfiguration `json:"openAMTConfiguration"`
+	FDOConfiguration     portainer.FDOConfiguration     `json:"fdoConfiguration"`
+	// The interval in which environment(endpoint) snapshots are created
+	SnapshotInterval string `json:"SnapshotInterval" example:"5m"`
+	// URL to the templates that will be displayed in the UI when navigating to App Templates
+	TemplatesURL string `json:"TemplatesURL" example:"https://raw.githubusercontent.com/portainer/templates/v3/templates.json"`
+	// The duration of a user session
+	UserSessionTimeout string `json:"UserSessionTimeout" example:"5m"`
+	// KubectlImage, defaults to portainer/kubectl-shell
+	KubectlShellImage string `json:"KubectlShellImage" example:"portainer/kubectl-shell"`
+	// Container environment parameter AGENT_SECRET
+	AgentSecret string `json:"AgentSecret"`
+}
+
+type publicSettingsResponse struct {
+	// global settings
+
+	// URL to a logo that will be displayed on the login page as well as on top of the sidebar. Will use default Portainer logo when value is empty string
+	LogoURL string `json:"LogoURL" example:"https://mycompany.mydomain.tld/logo.png"`
+	// Whether telemetry is enabled
+	EnableTelemetry bool `json:"EnableTelemetry" example:"true"`
+
+	// login settings:
+
+	// Active authentication method for the Portainer instance. Valid values are: 1 for internal, 2 for LDAP, or 3 for oauth
+	AuthenticationMethod portainer.AuthenticationMethod `json:"AuthenticationMethod" example:"1"`
+	// The URL used for oauth login
+	OAuthLoginURI string `json:"OAuthLoginURI" example:"https://gitlab.com/oauth"`
+	// The minimum required length for a password of any user when using internal auth mode
+	RequiredPasswordLength int `json:"RequiredPasswordLength" example:"1"`
+	// The URL used for oauth logout
+	OAuthLogoutURI string `json:"OAuthLogoutURI" example:"https://gitlab.com/oauth/logout"`
+	// Whether team sync is enabled
+	TeamSync bool `json:"TeamSync" example:"true"`
+	// Supported feature flags
+	Features map[featureflags.Feature]bool `json:"Features"`
+
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	GlobalDeploymentOptions portainer.GlobalDeploymentOptions `json:"GlobalDeploymentOptions"`
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	ShowKomposeBuildOption bool `json:"ShowKomposeBuildOption" example:"false"`
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures" example:"true"`
+
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	KubeconfigExpiry string `example:"24h" default:"0"`
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	IsFDOEnabled bool
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	IsAMTEnabled bool
+
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	DefaultRegistry struct {
+		Hide bool `json:"Hide" example:"false"`
+	}
+
+	// Deprecated v2.22
+	// please use `GET /api/settings`
+	Edge struct {
+		// Deprecated v2.22
+		// please use `GET /api/settings`
+		PingInterval int `json:"PingInterval" example:"60"`
+		// Deprecated v2.22
+		// please use `GET /api/settings`
+		SnapshotInterval int `json:"SnapshotInterval" example:"60"`
+		// Deprecated v2.22
+		// please use `GET /api/settings`
+		CommandInterval int `json:"CommandInterval" example:"60"`
+		// Deprecated v2.22
+		// please use `GET /api/settings`
+		CheckinInterval int `example:"60"`
+	}
+}
+
+func (res *settingsInspectResponse) ForRole(role portainer.UserRole) interface{} {
+	switch role {
+	case portainer.AdministratorRole:
+		return res.adminResponse
+	case portainer.EdgeAdminRole:
+		return res.edgeAdminResponse
+	case portainer.StandardUserRole:
+		return res.authenticatedResponse
+	default:
+		return res.publicSettingsResponse
+	}
+}
+
+func buildResponse(settings *portainer.Settings) settingsInspectResponse {
+	hideFields(settings)
+
+	return settingsInspectResponse{
+		adminResponse: adminResponse{
+			edgeAdminResponse: edgeAdminResponse{
+				authenticatedResponse: authenticatedResponse{
+					publicSettingsResponse: generatePublicSettings(settings),
+
+					GlobalDeploymentOptions:   settings.GlobalDeploymentOptions,
+					EnableEdgeComputeFeatures: settings.EnableEdgeComputeFeatures,
+					KubeconfigExpiry:          settings.KubeconfigExpiry,
+					HelmRepositoryURL:         settings.HelmRepositoryURL,
+					IsAMTEnabled:              settings.EnableEdgeComputeFeatures && settings.OpenAMTConfiguration.Enabled,
+					IsFDOEnabled:              settings.EnableEdgeComputeFeatures && settings.FDOConfiguration.Enabled,
+				},
+
+				Edge: edgeSettings{
+					CommandInterval:  settings.Edge.CommandInterval,
+					PingInterval:     settings.Edge.PingInterval,
+					SnapshotInterval: settings.Edge.SnapshotInterval,
+				},
+				TrustOnFirstConnect:      settings.TrustOnFirstConnect,
+				EnforceEdgeID:            settings.EnforceEdgeID,
+				EdgePortainerURL:         settings.EdgePortainerURL,
+				EdgeAgentCheckinInterval: settings.EdgeAgentCheckinInterval,
+			},
+			BlackListedLabels: settings.BlackListedLabels,
+			LDAPSettings: ldapSettings{
+				AnonymousMode:       settings.LDAPSettings.AnonymousMode,
+				ReaderDN:            settings.LDAPSettings.ReaderDN,
+				TLSConfig:           settings.LDAPSettings.TLSConfig,
+				StartTLS:            settings.LDAPSettings.StartTLS,
+				SearchSettings:      settings.LDAPSettings.SearchSettings,
+				GroupSearchSettings: settings.LDAPSettings.GroupSearchSettings,
+				AutoCreateUsers:     settings.LDAPSettings.AutoCreateUsers,
+				URL:                 settings.LDAPSettings.URL,
+			},
+			OAuthSettings: oauthSettings{
+				ClientID:             settings.OAuthSettings.ClientID,
+				AccessTokenURI:       settings.OAuthSettings.AccessTokenURI,
+				AuthorizationURI:     settings.OAuthSettings.AuthorizationURI,
+				ResourceURI:          settings.OAuthSettings.ResourceURI,
+				RedirectURI:          settings.OAuthSettings.RedirectURI,
+				UserIdentifier:       settings.OAuthSettings.UserIdentifier,
+				Scopes:               settings.OAuthSettings.Scopes,
+				OAuthAutoCreateUsers: settings.OAuthSettings.OAuthAutoCreateUsers,
+				DefaultTeamID:        settings.OAuthSettings.DefaultTeamID,
+				SSO:                  settings.OAuthSettings.SSO,
+				LogoutURI:            settings.OAuthSettings.LogoutURI,
+				AuthStyle:            settings.OAuthSettings.AuthStyle,
+			},
+			InternalAuthSettings: settings.InternalAuthSettings,
+			OpenAMTConfiguration: settings.OpenAMTConfiguration,
+			FDOConfiguration:     settings.FDOConfiguration,
+			SnapshotInterval:     settings.SnapshotInterval,
+			TemplatesURL:         settings.TemplatesURL,
+			UserSessionTimeout:   settings.UserSessionTimeout,
+			KubectlShellImage:    settings.KubectlShellImage,
+			AgentSecret:          settings.AgentSecret,
+		},
+	}
+}
+
+func getTeamSync(settings *portainer.Settings) bool {
+	if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
+		return settings.LDAPSettings.GroupSearchSettings != nil && len(settings.LDAPSettings.GroupSearchSettings) > 0 && len(settings.LDAPSettings.GroupSearchSettings[0].GroupBaseDN) > 0
+	}
+
+	return false
+}
+
+func generatePublicSettings(appSettings *portainer.Settings) publicSettingsResponse {
+	publicSettings := publicSettingsResponse{
+		LogoURL:                   appSettings.LogoURL,
+		AuthenticationMethod:      appSettings.AuthenticationMethod,
+		RequiredPasswordLength:    appSettings.InternalAuthSettings.RequiredPasswordLength,
+		EnableEdgeComputeFeatures: appSettings.EnableEdgeComputeFeatures,
+		GlobalDeploymentOptions:   appSettings.GlobalDeploymentOptions,
+		ShowKomposeBuildOption:    appSettings.ShowKomposeBuildOption,
+		EnableTelemetry:           appSettings.EnableTelemetry,
+		KubeconfigExpiry:          appSettings.KubeconfigExpiry,
+		Features:                  featureflags.FeatureFlags(),
+		IsFDOEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.FDOConfiguration.Enabled,
+		IsAMTEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.OpenAMTConfiguration.Enabled,
+		TeamSync:                  getTeamSync(appSettings),
+	}
+
+	publicSettings.Edge.PingInterval = appSettings.Edge.PingInterval
+	publicSettings.Edge.SnapshotInterval = appSettings.Edge.SnapshotInterval
+	publicSettings.Edge.CommandInterval = appSettings.Edge.CommandInterval
+	publicSettings.Edge.CheckinInterval = appSettings.EdgeAgentCheckinInterval
+
+	//if OAuth authentication is on, compose the related fields from application settings
+	if publicSettings.AuthenticationMethod == portainer.AuthenticationOAuth {
+		publicSettings.OAuthLogoutURI = appSettings.OAuthSettings.LogoutURI
+		publicSettings.OAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
+			appSettings.OAuthSettings.AuthorizationURI,
+			appSettings.OAuthSettings.ClientID,
+			appSettings.OAuthSettings.RedirectURI,
+			appSettings.OAuthSettings.Scopes)
+
+		//control prompt=login param according to the SSO setting
+		if !appSettings.OAuthSettings.SSO {
+			publicSettings.OAuthLoginURI += "&prompt=login"
+		}
+	}
+
+	return publicSettings
+}

api/http/handler/settings/settings_inspect_test.go

@@ -0,0 +1,168 @@
+package settings
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandler_settingsInspect(t *testing.T) {
+	t.Run("check that /api/settings returns the right value for admin", func(t *testing.T) {
+
+		user := portainer.User{
+			ID:       1,
+			Username: "admin",
+			Role:     portainer.AdministratorRole,
+		}
+
+		settings := &portainer.Settings{
+			LogoURL:              "https://nondefault.com/logo.png",
+			BlackListedLabels:    []portainer.Pair{{Name: "customlabel1", Value: "customvalue1"}},
+			AuthenticationMethod: 2,
+			InternalAuthSettings: portainer.InternalAuthSettings{
+				RequiredPasswordLength: 10,
+			},
+			LDAPSettings: portainer.LDAPSettings{
+				AnonymousMode: true,
+				ReaderDN:      "readerDN",
+				Password:      "password",
+				TLSConfig: portainer.TLSConfiguration{
+					TLS:           true,
+					TLSSkipVerify: true,
+					TLSCACertPath: "/path/to/ca-cert",
+					TLSCertPath:   "/path/to/cert",
+					TLSKeyPath:    "/path/to/key",
+				},
+				StartTLS: true,
+				SearchSettings: []portainer.LDAPSearchSettings{{
+					BaseDN:            "baseDN",
+					Filter:            "filter",
+					UserNameAttribute: "username",
+				}},
+				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{{
+					GroupBaseDN:    "groupBaseDN",
+					GroupFilter:    "groupFilter",
+					GroupAttribute: "groupAttribute",
+				}},
+				AutoCreateUsers: true,
+				URL:             "ldap://admin.example.com",
+			},
+			OAuthSettings: portainer.OAuthSettings{
+				ClientID:             "clientID",
+				ClientSecret:         "clientSecret",
+				AccessTokenURI:       "https://access-token-uri",
+				AuthorizationURI:     "https://authorization-uri",
+				ResourceURI:          "https://resource-uri",
+				RedirectURI:          "https://redirect-uri",
+				UserIdentifier:       "userIdentifier",
+				Scopes:               "scope1 scope2",
+				OAuthAutoCreateUsers: true,
+				DefaultTeamID:        1,
+				SSO:                  true,
+				LogoutURI:            "https://logout-uri",
+				KubeSecretKey:        []byte("secretKey"),
+				AuthStyle:            1,
+			},
+			OpenAMTConfiguration: portainer.OpenAMTConfiguration{
+				Enabled:          true,
+				MPSServer:        "mps-server",
+				MPSUser:          "mps-user",
+				MPSPassword:      "mps-password",
+				MPSToken:         "mps-token",
+				CertFileName:     "cert-filename",
+				CertFileContent:  "cert-file-content",
+				CertFilePassword: "cert-file-password",
+				DomainName:       "domain-name",
+			},
+			FDOConfiguration: portainer.FDOConfiguration{
+				Enabled:       true,
+				OwnerURL:      "https://owner-url",
+				OwnerUsername: "owner-username",
+				OwnerPassword: "owner-password",
+			},
+			SnapshotInterval: "30m",
+			TemplatesURL:     "https://nondefault.com/templates",
+			GlobalDeploymentOptions: portainer.GlobalDeploymentOptions{
+				HideStacksFunctionality: true,
+			},
+			EnableEdgeComputeFeatures: true,
+			UserSessionTimeout:        "1h",
+			KubeconfigExpiry:          "48h",
+			EnableTelemetry:           true,
+			HelmRepositoryURL:         "https://nondefault.com/helm",
+			KubectlShellImage:         "portainer/kubectl-shell:v2.0.0",
+			TrustOnFirstConnect:       true,
+			EnforceEdgeID:             true,
+			AgentSecret:               "nondefaultsecret",
+			EdgePortainerURL:          "https://edge.nondefault.com",
+			EdgeAgentCheckinInterval:  20,
+			Edge: portainer.EdgeSettings{
+				CommandInterval:  10,
+				PingInterval:     10,
+				SnapshotInterval: 10,
+			},
+		}
+
+		// copy settings so we can compare later (since we will change the settings struct in the handler)
+		dbSettings, err := cloneMyStruct(settings)
+		assert.NoError(t, err)
+
+		dataStore := testhelpers.NewDatastore(
+			testhelpers.WithSettingsService(dbSettings),
+			testhelpers.WithUsers([]portainer.User{user}),
+		)
+
+		handler := &Handler{
+			DataStore: dataStore,
+		}
+
+		// Create a mock request
+		req, err := http.NewRequest("GET", "/settings", nil)
+		assert.NoError(t, err)
+
+		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: user.ID, Username: user.Username, Role: user.Role})
+		req = req.WithContext(ctx)
+
+		restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: user.ID, IsAdmin: user.Role == portainer.AdministratorRole})
+		req = req.WithContext(restrictedCtx)
+
+		// Create a mock response recorder
+		rr := httptest.NewRecorder()
+		// Call the handler function
+		err = handler.settingsInspect(rr, req)
+
+		// Check for any handler errors
+		assert.Nil(t, err)
+
+		// Check the response status code
+		assert.Equal(t, http.StatusOK, rr.Code)
+
+		hideFields(settings)
+
+		actualSettings := &portainer.Settings{}
+		err = json.Unmarshal(rr.Body.Bytes(), actualSettings)
+		assert.NoError(t, err)
+
+		assert.EqualExportedValues(t, settings, actualSettings)
+	})
+}
+
+func cloneMyStruct[T any](orig *T) (*T, error) {
+	origJSON, err := json.Marshal(orig)
+	if err != nil {
+		return nil, err
+	}
+
+	clone := new(T)
+	if err = json.Unmarshal(origJSON, clone); err != nil {
+		return nil, err
+	}
+
+	return clone, nil
+}

api/http/handler/settings/settings_public.go

@@ -1,68 +1,19 @@
 package settings
 
 import (
-	"fmt"
 	"net/http"
 
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/pkg/featureflags"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
-type publicSettingsResponse struct {
-	// URL to a logo that will be displayed on the login page as well as on top of the sidebar. Will use default Portainer logo when value is empty string
-	LogoURL string `json:"LogoURL" example:"https://mycompany.mydomain.tld/logo.png"`
-	// Active authentication method for the Portainer instance. Valid values are: 1 for internal, 2 for LDAP, or 3 for oauth
-	AuthenticationMethod portainer.AuthenticationMethod `json:"AuthenticationMethod" example:"1"`
-	// The minimum required length for a password of any user when using internal auth mode
-	RequiredPasswordLength int `json:"RequiredPasswordLength" example:"1"`
-	// Deployment options for encouraging deployment as code
-	GlobalDeploymentOptions portainer.GlobalDeploymentOptions `json:"GlobalDeploymentOptions"`
-	// Show the Kompose build option (discontinued in 2.18)
-	ShowKomposeBuildOption bool `json:"ShowKomposeBuildOption" example:"false"`
-	// Whether edge compute features are enabled
-	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures" example:"true"`
-	// Supported feature flags
-	Features map[featureflags.Feature]bool `json:"Features"`
-	// The URL used for oauth login
-	OAuthLoginURI string `json:"OAuthLoginURI" example:"https://gitlab.com/oauth"`
-	// The URL used for oauth logout
-	OAuthLogoutURI string `json:"OAuthLogoutURI" example:"https://gitlab.com/oauth/logout"`
-	// Whether telemetry is enabled
-	EnableTelemetry bool `json:"EnableTelemetry" example:"true"`
-	// The expiry of a Kubeconfig
-	KubeconfigExpiry string `example:"24h" default:"0"`
-	// Whether team sync is enabled
-	TeamSync bool `json:"TeamSync" example:"true"`
-
-	// Whether FDO is enabled
-	IsFDOEnabled bool
-	// Whether AMT is enabled
-	IsAMTEnabled bool
-
-	Edge struct {
-		// The ping interval for edge agent - used in edge async mode [seconds]
-		PingInterval int `json:"PingInterval" example:"60"`
-		// The snapshot interval for edge agent - used in edge async mode [seconds]
-		SnapshotInterval int `json:"SnapshotInterval" example:"60"`
-		// The command list interval for edge agent - used in edge async mode [seconds]
-		CommandInterval int `json:"CommandInterval" example:"60"`
-		// The check in interval for edge agent (in seconds) - used in non async mode [seconds]
-		CheckinInterval int `example:"60"`
-	}
-
-	IsDockerDesktopExtension bool `json:"IsDockerDesktopExtension" example:"false"`
-}
-
 // @id SettingsPublic
-// @summary Retrieve Portainer public settings
-// @description Retrieve public settings. Returns a small set of settings that are not reserved to administrators only.
-// @description **Access policy**: public
+// @summary Retrieve the public settings of the Portainer instance
+// @description Get the settings of the Portainer instance. Will return only a subset of settings.
 // @tags settings
 // @produce json
-// @success 200 {object} publicSettingsResponse "Success"
-// @failure 500 "Server error"
+// @success 200 {object} publicSettingsResponse "The settings object"
+// @failure 500 "Server error occurred while attempting to retrieve the settings."
 // @router /settings/public [get]
 func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	settings, err := handler.DataStore.Settings().Settings()
@@ -73,47 +24,3 @@ func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *
 	publicSettings := generatePublicSettings(settings)
 	return response.JSON(w, publicSettings)
 }
-
-func generatePublicSettings(appSettings *portainer.Settings) *publicSettingsResponse {
-	publicSettings := &publicSettingsResponse{
-		LogoURL:                   appSettings.LogoURL,
-		AuthenticationMethod:      appSettings.AuthenticationMethod,
-		RequiredPasswordLength:    appSettings.InternalAuthSettings.RequiredPasswordLength,
-		EnableEdgeComputeFeatures: appSettings.EnableEdgeComputeFeatures,
-		GlobalDeploymentOptions:   appSettings.GlobalDeploymentOptions,
-		ShowKomposeBuildOption:    appSettings.ShowKomposeBuildOption,
-		EnableTelemetry:           appSettings.EnableTelemetry,
-		KubeconfigExpiry:          appSettings.KubeconfigExpiry,
-		Features:                  featureflags.FeatureFlags(),
-		IsFDOEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.FDOConfiguration.Enabled,
-		IsAMTEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.OpenAMTConfiguration.Enabled,
-	}
-
-	publicSettings.Edge.PingInterval = appSettings.Edge.PingInterval
-	publicSettings.Edge.SnapshotInterval = appSettings.Edge.SnapshotInterval
-	publicSettings.Edge.CommandInterval = appSettings.Edge.CommandInterval
-	publicSettings.Edge.CheckinInterval = appSettings.EdgeAgentCheckinInterval
-
-	publicSettings.IsDockerDesktopExtension = appSettings.IsDockerDesktopExtension
-
-	//if OAuth authentication is on, compose the related fields from application settings
-	if publicSettings.AuthenticationMethod == portainer.AuthenticationOAuth {
-		publicSettings.OAuthLogoutURI = appSettings.OAuthSettings.LogoutURI
-		publicSettings.OAuthLoginURI = fmt.Sprintf("%s?response_type=code&client_id=%s&redirect_uri=%s&scope=%s",
-			appSettings.OAuthSettings.AuthorizationURI,
-			appSettings.OAuthSettings.ClientID,
-			appSettings.OAuthSettings.RedirectURI,
-			appSettings.OAuthSettings.Scopes)
-		//control prompt=login param according to the SSO setting
-		if !appSettings.OAuthSettings.SSO {
-			publicSettings.OAuthLoginURI += "&prompt=login"
-		}
-	}
-	//if LDAP authentication is on, compose the related fields from application settings
-	if publicSettings.AuthenticationMethod == portainer.AuthenticationLDAP && appSettings.LDAPSettings.GroupSearchSettings != nil {
-		if len(appSettings.LDAPSettings.GroupSearchSettings) > 0 {
-			publicSettings.TeamSync = len(appSettings.LDAPSettings.GroupSearchSettings[0].GroupBaseDN) > 0
-		}
-	}
-	return publicSettings
-}

api/http/handler/settings/settings_update.go

@@ -13,6 +13,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"golang.org/x/oauth2"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
@@ -95,6 +96,11 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 		}
 	}
 
+	if payload.OAuthSettings != nil {
+		if payload.OAuthSettings.AuthStyle < oauth2.AuthStyleAutoDetect || payload.OAuthSettings.AuthStyle > oauth2.AuthStyleInHeader {
+			return errors.New("Invalid OAuth AuthStyle")
+		}
+	}
 	return nil
 }
 
@@ -137,17 +143,18 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 	return response.JSON(w, settings)
 }
 
+func hideFields(settings *portainer.Settings) {
+	settings.LDAPSettings.Password = ""
+	settings.OAuthSettings.ClientSecret = ""
+	settings.OAuthSettings.KubeSecretKey = nil
+}
+
 func (handler *Handler) updateSettings(tx dataservices.DataStoreTx, payload settingsUpdatePayload) (*portainer.Settings, error) {
 	settings, err := tx.Settings().Settings()
 	if err != nil {
 		return nil, httperror.InternalServerError("Unable to retrieve the settings from the database", err)
 	}
 
-	if handler.demoService.IsDemo() {
-		payload.EnableTelemetry = nil
-		payload.LogoURL = nil
-	}
-
 	if payload.AuthenticationMethod != nil {
 		settings.AuthenticationMethod = portainer.AuthenticationMethod(*payload.AuthenticationMethod)
 	}
@@ -225,6 +232,7 @@ func (handler *Handler) updateSettings(tx dataservices.DataStoreTx, payload sett
 		settings.OAuthSettings = *payload.OAuthSettings
 		settings.OAuthSettings.ClientSecret = clientSecret
 		settings.OAuthSettings.KubeSecretKey = kubeSecret
+		settings.OAuthSettings.AuthStyle = payload.OAuthSettings.AuthStyle
 	}
 
 	if payload.EnableEdgeComputeFeatures != nil {

api/http/handler/stacks/create_compose_stack.go

@@ -3,6 +3,7 @@ package stacks
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
@@ -229,6 +230,7 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 // @param body body composeStackFromGitRepositoryPayload true "stack config"
 // @success 200 {object} portainer.Stack
 // @failure 400 "Invalid request"
+// @failure 409 "Stack name or webhook ID already exists"
 // @failure 500 "Server error"
 // @router /stacks/create/standalone/repository [post]
 func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {
@@ -282,7 +284,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 	}
 
 	stackPayload := createStackPayloadFromComposeGitPayload(payload.Name,
-		payload.RepositoryURL,
+		strings.TrimSuffix(payload.RepositoryURL, "/"),
 		payload.RepositoryReferenceName,
 		payload.RepositoryUsername,
 		payload.RepositoryPassword,

api/http/handler/stacks/create_kubernetes_stack.go

@@ -195,6 +195,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 // @param endpointId query int true "Identifier of the environment that will be used to deploy the stack"
 // @success 200 {object} portainer.Stack
 // @failure 400 "Invalid request"
+// @failure 409 "Stack name or webhook ID already exists"
 // @failure 500 "Server error"
 // @router /stacks/create/kubernetes/repository [post]
 func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {