chore(version): bump version to 2.21.1 (#12203)

Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>
Co-authored-by: Yajith Dayarathna <yajith.dayarathna@portainer.io>

.eslintrc.yml

@@ -10,6 +10,7 @@ globals:
 extends:
   - 'eslint:recommended'
   - 'plugin:storybook/recommended'
+  - 'plugin:import/typescript'
   - prettier
 
 plugins:
@@ -29,6 +30,7 @@ rules:
   no-empty: warn
   no-empty-function: warn
   no-useless-escape: 'off'
+  import/named: error
   import/order:
     [
       'error',
@@ -43,6 +45,12 @@ rules:
         pathGroupsExcludedImportTypes: ['internal'],
       },
     ]
+  no-restricted-imports:
+    - error
+    - patterns:
+        - group:
+            - '@/react/test-utils/*'
+          message: 'These utils are just for test files'
 
 settings:
   'import/resolver':
@@ -51,6 +59,8 @@ settings:
         - ['@@', './app/react/components']
         - ['@', './app']
       extensions: ['.js', '.ts', '.tsx']
+    typescript: true
+    node: true
 
 overrides:
   - files:
@@ -75,6 +85,7 @@ overrides:
     settings:
       react:
         version: 'detect'
+
     rules:
       import/order:
         [
@@ -108,6 +119,12 @@ overrides:
       'no-await-in-loop': 'off'
       'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
       'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
+      '@typescript-eslint/no-restricted-imports':
+        - error
+        - patterns:
+            - group:
+                - '@/react/test-utils/*'
+              message: 'These utils are just for test files'
     overrides: # allow props spreading for hoc files
       - files:
           - app/**/with*.ts{,x}
@@ -121,7 +138,11 @@ overrides:
       'vitest/env': true
     rules:
       'react/jsx-no-constructed-context-values': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
   - files:
       - app/**/*.stories.*
     rules:
       'no-alert': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off

.github/workflows/ci.yaml

@@ -5,7 +5,7 @@ on:
   push:
     branches:
       - 'develop'
-      - '!release/*'
+      - 'release/*'
   pull_request:
     branches:
       - 'develop'
@@ -20,9 +20,9 @@ on:
       - ready_for_review
 
 env:
-  DOCKER_HUB_REPO: portainerci/portainer
-  NODE_ENV: testing
-  GO_VERSION: 1.21.6
+  DOCKER_HUB_REPO: portainerci/portainer-ce
+  EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
+  GO_VERSION: 1.21.11
   NODE_VERSION: 18.x
 
 jobs:
@@ -30,81 +30,58 @@ jobs:
     strategy:
       matrix:
         config:
-          - { platform: linux, arch: amd64 }
-          - { platform: linux, arch: arm64 }
+          - { platform: linux, arch: amd64, version: "" }
+          - { platform: linux, arch: arm64, version: "" }
+          - { platform: linux, arch: arm, version: "" }
+          - { platform: linux, arch: ppc64le, version: "" }
           - { platform: windows, arch: amd64, version: 1809 }
           - { platform: windows, arch: amd64, version: ltsc2022 }
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
     steps:
       - name: '[preparation] checkout the current branch'
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4.1.1
         with:
           ref: ${{ github.event.inputs.branch }}
       - name: '[preparation] set up golang'
-        uses: actions/setup-go@v4.0.1
+        uses: actions/setup-go@v5.0.0
         with:
           go-version: ${{ env.GO_VERSION }}
-          cache: false
-      - name: '[preparation] cache paths'
-        id: cache-dir-path
-        run: |
-          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
-          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
-      - name: '[preparation] cache go'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache-dir-path.outputs.go-build-dir }}
-            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
-          enableCrossOsArchive: true
       - name: '[preparation] set up node.js'
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4.0.1
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: ''
-      - name: '[preparation] cache yarn'
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
-          enableCrossOsArchive: true
+          cache: 'yarn'
       - name: '[preparation] set up qemu'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3.0.0
       - name: '[preparation] set up docker context for buildx'
         run: docker context create builders
       - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3.0.0
         with:
           endpoint: builders
       - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: '[preparation] set the container image tag'
         run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
             CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
           else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
             CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
           fi
-
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
-          else 
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
-          fi
-
-          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
+          
+          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}" >> $GITHUB_ENV
       - name: '[execution] build linux & windows portainer binaries'
         run: |
           export YARN_VERSION=$(yarn --version)
@@ -112,6 +89,12 @@ jobs:
           export BUILDNUMBER=${GITHUB_RUN_NUMBER}
           GIT_COMMIT_HASH_LONG=${{ github.sha }}
           export GIT_COMMIT_HASH_SHORT={GIT_COMMIT_HASH_LONG:0:7}
+          
+          NODE_ENV="testing"
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            NODE_ENV="production"
+          fi
+          
           make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
         env:
           CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
@@ -123,35 +106,61 @@ jobs:
           else 
             docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
             docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            
+            if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            fi
           fi
         env:
           CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
   build_manifests:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
     needs: [build_images]
     steps:
       - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: '[preparation] set up docker context for buildx'
         run: docker version && docker context create builders
       - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3.0.0
         with:
           endpoint: builders
       - name: '[execution] build and push manifests'
         run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
             CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
           else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
             CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
           fi
-
           docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
+          
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le-alpine"
+            
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64"
+          fi

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/nightly-security-scan.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
 
 jobs:
   client-dependencies:

.github/workflows/pr-security.yml

@@ -14,7 +14,7 @@ on:
       - '.github/workflows/pr-security.yml'
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/test.yaml

@@ -1,17 +1,26 @@
 name: Test
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 on:
+  workflow_dispatch:
   pull_request:
+    branches:
+      - master
+      - develop
+      - release/*
     types:
       - opened
       - reopened
       - synchronize
       - ready_for_review
   push:
+    branches:
+      - master
+      - develop
+      - release/*
 
 jobs:
   test-client:
@@ -19,15 +28,22 @@ jobs:
     if: github.event.pull_request.draft == false
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - name: 'checkout the current branch'
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: 'set up node.js'
+        uses: actions/setup-node@v4.0.1
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
+
       - run: yarn --frozen-lockfile
 
       - name: Run tests
         run: make test-client ARGS="--maxWorkers=2 --minWorkers=1"
+
   test-server:
     strategy:
       matrix:
@@ -40,9 +56,21 @@ jobs:
     if: github.event.pull_request.draft == false
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - name: 'checkout the current branch'
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: 'set up golang'
+        uses: actions/setup-go@v5.0.0
         with:
           go-version: ${{ env.GO_VERSION }}
-      - name: Run tests
+
+      - name: 'install dependencies'
+        run: make test-deps PLATFORM=linux ARCH=amd64
+
+      - name: 'update $PATH'
+        run: echo "$(pwd)/dist" >> $GITHUB_PATH
+
+      - name: 'run tests'
         run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -13,7 +13,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.storybook/preview.tsx

@@ -8,7 +8,6 @@ import { QueryClient, QueryClientProvider } from 'react-query';
 initMSW(
   {
     onUnhandledRequest: ({ method, url }) => {
-      console.log(method, url);
       if (url.startsWith('/api')) {
         console.error(`Unhandled ${method} request to ${url}.
 

Makefile

@@ -64,6 +64,9 @@ clean: ## Remove all build and download artifacts
 .PHONY: test test-client test-server
 test: test-server test-client ## Run all tests
 
+test-deps: init-dist
+	./build/download_docker_compose_binary.sh $(PLATFORM) $(ARCH) $(shell jq -r '.dockerCompose' < "./binary-version.json")
+
 test-client: ## Run client tests
 	yarn test $(ARGS)
 

api/backup/backup.go

@@ -82,7 +82,8 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 }
 
 func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
-	_, err := datastore.Backup(filepath.Join(backupDirPath, "portainer.db"))
+	dbFileName := datastore.Connection().GetDatabaseFileName()
+	_, err := datastore.Backup(filepath.Join(backupDirPath, dbFileName))
 	return err
 }
 

api/backup/restore.go

@@ -26,7 +26,7 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
 	if password != "" {
 		archive, err = decrypt(archive, password)
 		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
+			return errors.Wrap(err, "failed to decrypt the archive. Please ensure the password is correct and try again")
 		}
 	}
 

api/chisel/schedules.go

@@ -5,6 +5,17 @@ import (
 	"github.com/portainer/portainer/api/internal/edge/cache"
 )
 
+// EdgeJobs retrieves the edge jobs for the given environment
+func (service *Service) EdgeJobs(endpointID portainer.EndpointID) []portainer.EdgeJob {
+	service.mu.RLock()
+	defer service.mu.RUnlock()
+
+	return append(
+		make([]portainer.EdgeJob, 0, len(service.edgeJobs[endpointID])),
+		service.edgeJobs[endpointID]...,
+	)
+}
+
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
 func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portainer.EdgeJob) {
 	if endpoint.Edge.AsyncMode {
@@ -12,10 +23,10 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	}
 
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpoint.ID)
+	defer service.mu.Unlock()
 
 	existingJobIndex := -1
-	for idx, existingJob := range tunnel.Jobs {
+	for idx, existingJob := range service.edgeJobs[endpoint.ID] {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
 
@@ -24,30 +35,28 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	}
 
 	if existingJobIndex == -1 {
-		tunnel.Jobs = append(tunnel.Jobs, *edgeJob)
+		service.edgeJobs[endpoint.ID] = append(service.edgeJobs[endpoint.ID], *edgeJob)
 	} else {
-		tunnel.Jobs[existingJobIndex] = *edgeJob
+		service.edgeJobs[endpoint.ID][existingJobIndex] = *edgeJob
 	}
 
 	cache.Del(endpoint.ID)
-
-	service.mu.Unlock()
 }
 
 // RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
 func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
 
-	for endpointID, tunnel := range service.tunnelDetailsMap {
+	for endpointID := range service.edgeJobs {
 		n := 0
-		for _, edgeJob := range tunnel.Jobs {
+		for _, edgeJob := range service.edgeJobs[endpointID] {
 			if edgeJob.ID != edgeJobID {
-				tunnel.Jobs[n] = edgeJob
+				service.edgeJobs[endpointID][n] = edgeJob
 				n++
 			}
 		}
 
-		tunnel.Jobs = tunnel.Jobs[:n]
+		service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 		cache.Del(endpointID)
 	}
@@ -57,19 +66,17 @@ func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 
 func (service *Service) RemoveEdgeJobFromEndpoint(endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
+	defer service.mu.Unlock()
 
 	n := 0
-	for _, edgeJob := range tunnel.Jobs {
+	for _, edgeJob := range service.edgeJobs[endpointID] {
 		if edgeJob.ID != edgeJobID {
-			tunnel.Jobs[n] = edgeJob
+			service.edgeJobs[endpointID][n] = edgeJob
 			n++
 		}
 	}
 
-	tunnel.Jobs = tunnel.Jobs[:n]
+	service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 	cache.Del(endpointID)
-
-	service.mu.Unlock()
 }

api/chisel/service.go

@@ -19,7 +19,6 @@ import (
 
 const (
 	tunnelCleanupInterval = 10 * time.Second
-	requiredTimeout       = 15 * time.Second
 	activeTimeout         = 4*time.Minute + 30*time.Second
 	pingTimeout           = 3 * time.Second
 )
@@ -28,32 +27,54 @@ const (
 // It is used to start a reverse tunnel server and to manage the connection status of each tunnel
 // connected to the tunnel server.
 type Service struct {
-	serverFingerprint string
-	serverPort        string
-	tunnelDetailsMap  map[portainer.EndpointID]*portainer.TunnelDetails
-	dataStore         dataservices.DataStore
-	snapshotService   portainer.SnapshotService
-	chiselServer      *chserver.Server
-	shutdownCtx       context.Context
-	ProxyManager      *proxy.Manager
-	mu                sync.Mutex
-	fileService       portainer.FileService
+	serverFingerprint      string
+	serverPort             string
+	activeTunnels          map[portainer.EndpointID]*portainer.TunnelDetails
+	edgeJobs               map[portainer.EndpointID][]portainer.EdgeJob
+	dataStore              dataservices.DataStore
+	snapshotService        portainer.SnapshotService
+	chiselServer           *chserver.Server
+	shutdownCtx            context.Context
+	ProxyManager           *proxy.Manager
+	mu                     sync.RWMutex
+	fileService            portainer.FileService
+	defaultCheckinInterval int
 }
 
 // NewService returns a pointer to a new instance of Service
 func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
+	defaultCheckinInterval := portainer.DefaultEdgeAgentCheckinIntervalInSeconds
+
+	settings, err := dataStore.Settings().Settings()
+	if err == nil {
+		defaultCheckinInterval = settings.EdgeAgentCheckinInterval
+	} else {
+		log.Error().Err(err).Msg("unable to retrieve the settings from the database")
+	}
+
 	return &Service{
-		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
-		dataStore:        dataStore,
-		shutdownCtx:      shutdownCtx,
-		fileService:      fileService,
+		activeTunnels:          make(map[portainer.EndpointID]*portainer.TunnelDetails),
+		edgeJobs:               make(map[portainer.EndpointID][]portainer.EdgeJob),
+		dataStore:              dataStore,
+		shutdownCtx:            shutdownCtx,
+		fileService:            fileService,
+		defaultCheckinInterval: defaultCheckinInterval,
 	}
 }
 
 // pingAgent ping the given agent so that the agent can keep the tunnel alive
 func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
-	tunnel := service.GetTunnelDetails(endpointID)
-	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
+	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
+	if err != nil {
+		return err
+	}
+
+	tunnelAddr, err := service.TunnelAddr(endpoint)
+	if err != nil {
+		return err
+	}
+
+	requestURL := fmt.Sprintf("http://%s/ping", tunnelAddr)
 	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
 	if err != nil {
 		return err
@@ -76,47 +97,49 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
 func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
-	go func() {
-		log.Debug().
-			Int("endpoint_id", int(endpointID)).
-			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("KeepTunnelAlive: start")
+	go service.keepTunnelAlive(endpointID, ctx, maxAlive)
+}
 
-		maxAliveTicker := time.NewTicker(maxAlive)
-		defer maxAliveTicker.Stop()
+func (service *Service) keepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
+	log.Debug().
+		Int("endpoint_id", int(endpointID)).
+		Float64("max_alive_minutes", maxAlive.Minutes()).
+		Msg("KeepTunnelAlive: start")
 
-		pingTicker := time.NewTicker(tunnelCleanupInterval)
-		defer pingTicker.Stop()
+	maxAliveTicker := time.NewTicker(maxAlive)
+	defer maxAliveTicker.Stop()
 
-		for {
-			select {
-			case <-pingTicker.C:
-				service.SetTunnelStatusToActive(endpointID)
-				err := service.pingAgent(endpointID)
-				if err != nil {
-					log.Debug().
-						Int("endpoint_id", int(endpointID)).
-						Err(err).
-						Msg("KeepTunnelAlive: ping agent")
-				}
-			case <-maxAliveTicker.C:
-				log.Debug().
-					Int("endpoint_id", int(endpointID)).
-					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("KeepTunnelAlive: tunnel keep alive timeout")
+	pingTicker := time.NewTicker(tunnelCleanupInterval)
+	defer pingTicker.Stop()
 
-				return
-			case <-ctx.Done():
-				err := ctx.Err()
+	for {
+		select {
+		case <-pingTicker.C:
+			service.UpdateLastActivity(endpointID)
+
+			if err := service.pingAgent(endpointID); err != nil {
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("KeepTunnelAlive: tunnel stop")
-
-				return
+					Msg("KeepTunnelAlive: ping agent")
 			}
+		case <-maxAliveTicker.C:
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Float64("timeout_minutes", maxAlive.Minutes()).
+				Msg("KeepTunnelAlive: tunnel keep alive timeout")
+
+			return
+		case <-ctx.Done():
+			err := ctx.Err()
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("KeepTunnelAlive: tunnel stop")
+
+			return
 		}
-	}()
+	}
 }
 
 // StartTunnelServer starts a tunnel server on the specified addr and port.
@@ -126,7 +149,6 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
 	privateKeyFile, err := service.retrievePrivateKeyFile()
-
 	if err != nil {
 		return err
 	}
@@ -144,21 +166,21 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	service.serverFingerprint = chiselServer.GetFingerprint()
 	service.serverPort = port
 
-	err = chiselServer.Start(addr, port)
-	if err != nil {
+	if err := chiselServer.Start(addr, port); err != nil {
 		return err
 	}
+
 	service.chiselServer = chiselServer
 
 	// TODO: work-around Chisel default behavior.
 	// By default, Chisel will allow anyone to connect if no user exists.
 	username, password := generateRandomCredentials()
-	err = service.chiselServer.AddUser(username, password, "127.0.0.1")
-	if err != nil {
+	if err = service.chiselServer.AddUser(username, password, "127.0.0.1"); err != nil {
 		return err
 	}
 
 	service.snapshotService = snapshotService
+
 	go service.startTunnelVerificationLoop()
 
 	return nil
@@ -172,37 +194,39 @@ func (service *Service) StopTunnelServer() error {
 func (service *Service) retrievePrivateKeyFile() (string, error) {
 	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
 
-	exist, _ := service.fileService.FileExists(privateKeyFile)
-	if !exist {
-		log.Debug().
-			Str("private-key", privateKeyFile).
-			Msg("Chisel private key file does not exist")
-
-		privateKey, err := ccrypto.GenerateKey("")
-		if err != nil {
-			log.Error().
-				Err(err).
-				Msg("Failed to generate chisel private key")
-			return "", err
-		}
-
-		err = service.fileService.StoreChiselPrivateKey(privateKey)
-		if err != nil {
-			log.Error().
-				Err(err).
-				Msg("Failed to save Chisel private key to disk")
-			return "", err
-		} else {
-			log.Info().
-				Str("private-key", privateKeyFile).
-				Msg("Generated a new Chisel private key file")
-		}
-	} else {
+	if exists, _ := service.fileService.FileExists(privateKeyFile); exists {
 		log.Info().
 			Str("private-key", privateKeyFile).
-			Msg("Found Chisel private key file on disk")
+			Msg("found Chisel private key file on disk")
+
+		return privateKeyFile, nil
 	}
 
+	log.Debug().
+		Str("private-key", privateKeyFile).
+		Msg("chisel private key file does not exist")
+
+	privateKey, err := ccrypto.GenerateKey("")
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to generate chisel private key")
+
+		return "", err
+	}
+
+	if err = service.fileService.StoreChiselPrivateKey(privateKey); err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to save Chisel private key to disk")
+
+		return "", err
+	}
+
+	log.Info().
+		Str("private-key", privateKeyFile).
+		Msg("generated a new Chisel private key file")
+
 	return privateKeyFile, nil
 }
 
@@ -230,63 +254,45 @@ func (service *Service) startTunnelVerificationLoop() {
 	}
 }
 
+// checkTunnels finds the first tunnel that has not had any activity recently
+// and attempts to take a snapshot, then closes it and returns
 func (service *Service) checkTunnels() {
-	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
+	service.mu.RLock()
 
-	service.mu.Lock()
-	for key, tunnel := range service.tunnelDetailsMap {
-		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
-			continue
-		}
-
-		tunnels[key] = *tunnel
-	}
-	service.mu.Unlock()
-
-	for endpointID, tunnel := range tunnels {
+	for endpointID, tunnel := range service.activeTunnels {
 		elapsed := time.Since(tunnel.LastActivity)
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
-			Str("status", tunnel.Status).
-			Float64("status_time_seconds", elapsed.Seconds()).
+			Float64("last_activity_seconds", elapsed.Seconds()).
 			Msg("environment tunnel monitoring")
 
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", requiredTimeout.Seconds()).
-				Msg("REQUIRED state timeout exceeded")
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed < activeTimeout {
+			continue
 		}
 
-		if tunnel.Status == portainer.EdgeAgentActive && elapsed > activeTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", activeTimeout.Seconds()).
-				Msg("ACTIVE state timeout exceeded")
+		tunnelPort := tunnel.Port
 
-			err := service.snapshotEnvironment(endpointID, tunnel.Port)
-			if err != nil {
-				log.Error().
-					Int("endpoint_id", int(endpointID)).
-					Err(err).
-					Msg("unable to snapshot Edge environment")
-			}
+		service.mu.RUnlock()
+
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Float64("last_activity_seconds", elapsed.Seconds()).
+			Float64("timeout_seconds", activeTimeout.Seconds()).
+			Msg("last activity timeout exceeded")
+
+		if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
+			log.Error().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("unable to snapshot Edge environment")
 		}
 
-		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
+		service.close(portainer.EndpointID(endpointID))
+
+		return
 	}
+
+	service.mu.RUnlock()
 }
 
 func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {

api/chisel/service_test.go

@@ -7,14 +7,22 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestPingAgentPanic(t *testing.T) {
-	endpointID := portainer.EndpointID(1)
+	endpoint := &portainer.Endpoint{
+		ID:          1,
+		EdgeID:      "test-edge-id",
+		Type:        portainer.EdgeAgentOnDockerEnvironment,
+		UserTrusted: true,
+	}
 
-	s := NewService(nil, nil, nil)
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	s := NewService(store, nil, nil)
 
 	defer func() {
 		require.Nil(t, recover())
@@ -32,8 +40,9 @@ func TestPingAgentPanic(t *testing.T) {
 		require.NoError(t, http.Serve(ln, mux))
 	}()
 
-	s.getTunnelDetails(endpointID)
-	s.tunnelDetailsMap[endpointID].Port = ln.Addr().(*net.TCPAddr).Port
+	err = s.Open(endpoint)
+	require.NoError(t, err)
+	s.activeTunnels[endpoint.ID].Port = ln.Addr().(*net.TCPAddr).Port
 
-	require.Error(t, s.pingAgent(endpointID))
+	require.Error(t, s.pingAgent(endpoint.ID))
 }

api/chisel/tunnel.go

@@ -5,14 +5,18 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"strings"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/pkg/libcrypto"
 
 	"github.com/dchest/uniuri"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -20,18 +24,191 @@ const (
 	maxAvailablePort = 65535
 )
 
+var (
+	ErrNonEdgeEnv = errors.New("cannot open a tunnel for non-edge environments")
+	ErrAsyncEnv   = errors.New("cannot open a tunnel for async edge environments")
+	ErrInvalidEnv = errors.New("cannot open a tunnel for an invalid environment")
+)
+
+// Open will mark the tunnel as REQUIRED so the agent opens it
+func (s *Service) Open(endpoint *portainer.Endpoint) error {
+	if !endpointutils.IsEdgeEndpoint(endpoint) {
+		return ErrNonEdgeEnv
+	}
+
+	if endpoint.Edge.AsyncMode {
+		return ErrAsyncEnv
+	}
+
+	if endpoint.ID == 0 || endpoint.EdgeID == "" || !endpoint.UserTrusted {
+		return ErrInvalidEnv
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.activeTunnels[endpoint.ID]; ok {
+		return nil
+	}
+
+	defer cache.Del(endpoint.ID)
+
+	tun := &portainer.TunnelDetails{
+		Status:       portainer.EdgeAgentManagementRequired,
+		Port:         s.getUnusedPort(),
+		LastActivity: time.Now(),
+	}
+
+	username, password := generateRandomCredentials()
+
+	if s.chiselServer != nil {
+		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tun.Port)
+
+		if err := s.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
+			return err
+		}
+	}
+
+	credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
+	if err != nil {
+		return err
+	}
+
+	tun.Credentials = credentials
+
+	s.activeTunnels[endpoint.ID] = tun
+
+	return nil
+}
+
+// close removes the tunnel from the map so the agent will close it
+func (s *Service) close(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tun, ok := s.activeTunnels[endpointID]
+	if !ok {
+		return
+	}
+
+	if len(tun.Credentials) > 0 && s.chiselServer != nil {
+		user, _, _ := strings.Cut(tun.Credentials, ":")
+		s.chiselServer.DeleteUser(user)
+	}
+
+	if s.ProxyManager != nil {
+		s.ProxyManager.DeleteEndpointProxy(endpointID)
+	}
+
+	delete(s.activeTunnels, endpointID)
+
+	cache.Del(endpointID)
+}
+
+// Config returns the tunnel details needed for the agent to connect
+func (s *Service) Config(endpointID portainer.EndpointID) portainer.TunnelDetails {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		return *tun
+	}
+
+	return portainer.TunnelDetails{Status: portainer.EdgeAgentIdle}
+}
+
+// TunnelAddr returns the address of the local tunnel, including the port, it
+// will block until the tunnel is ready
+func (s *Service) TunnelAddr(endpoint *portainer.Endpoint) (string, error) {
+	if err := s.Open(endpoint); err != nil {
+		return "", err
+	}
+
+	tun := s.Config(endpoint.ID)
+	checkinInterval := time.Duration(s.tryEffectiveCheckinInterval(endpoint)) * time.Second
+
+	for t0 := time.Now(); ; {
+		if time.Since(t0) > 2*checkinInterval {
+			s.close(endpoint.ID)
+
+			return "", errors.New("unable to open the tunnel")
+		}
+
+		// Check if the tunnel is established
+		conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: tun.Port})
+		if err != nil {
+			time.Sleep(checkinInterval / 100)
+
+			continue
+		}
+
+		conn.Close()
+
+		break
+	}
+
+	s.UpdateLastActivity(endpoint.ID)
+
+	return fmt.Sprintf("127.0.0.1:%d", tun.Port), nil
+}
+
+// tryEffectiveCheckinInterval avoids a potential deadlock by returning a
+// previous known value after a timeout
+func (s *Service) tryEffectiveCheckinInterval(endpoint *portainer.Endpoint) int {
+	ch := make(chan int, 1)
+
+	go func() {
+		ch <- edge.EffectiveCheckinInterval(s.dataStore, endpoint)
+	}()
+
+	select {
+	case <-time.After(50 * time.Millisecond):
+		s.mu.RLock()
+		defer s.mu.RUnlock()
+
+		return s.defaultCheckinInterval
+	case i := <-ch:
+		s.mu.Lock()
+		s.defaultCheckinInterval = i
+		s.mu.Unlock()
+
+		return i
+	}
+}
+
+// UpdateLastActivity sets the current timestamp to avoid the tunnel timeout
+func (s *Service) UpdateLastActivity(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		tun.LastActivity = time.Now()
+	}
+}
+
 // NOTE: it needs to be called with the lock acquired
 // getUnusedPort is used to generate an unused random port in the dynamic port range.
 // Dynamic ports (also called private ports) are 49152 to 65535.
 func (service *Service) getUnusedPort() int {
 	port := randomInt(minAvailablePort, maxAvailablePort)
 
-	for _, tunnel := range service.tunnelDetailsMap {
+	for _, tunnel := range service.activeTunnels {
 		if tunnel.Port == port {
 			return service.getUnusedPort()
 		}
 	}
 
+	conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
+	if err == nil {
+		conn.Close()
+
+		log.Debug().
+			Int("port", port).
+			Msg("selected port is in use, trying a different one")
+
+		return service.getUnusedPort()
+	}
+
 	return port
 }
 
@@ -39,152 +216,10 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
-// NOTE: it needs to be called with the lock acquired
-func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
-
-	if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
-		return tunnel
-	}
-
-	tunnel := &portainer.TunnelDetails{
-		Status: portainer.EdgeAgentIdle,
-	}
-
-	service.tunnelDetailsMap[endpointID] = tunnel
-
-	cache.Del(endpointID)
-
-	return tunnel
-}
-
-// GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
-func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	return *service.getTunnelDetails(endpointID)
-}
-
-// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
-	if endpoint.Edge.AsyncMode {
-		return portainer.TunnelDetails{}, errors.New("cannot open tunnel on async endpoint")
-	}
-
-	tunnel := service.GetTunnelDetails(endpoint.ID)
-
-	if tunnel.Status == portainer.EdgeAgentActive {
-		// update the LastActivity
-		service.SetTunnelStatusToActive(endpoint.ID)
-	}
-
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := service.dataStore.Settings().Settings()
-			if err != nil {
-				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
-	}
-
-	return service.GetTunnelDetails(endpoint.ID), nil
-}
-
-// SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to ACTIVE.
-func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentActive
-	tunnel.Credentials = ""
-	tunnel.LastActivity = time.Now()
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to IDLE.
-// It removes any existing credentials associated to the tunnel.
-func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentIdle
-	tunnel.Port = 0
-	tunnel.LastActivity = time.Now()
-
-	credentials := tunnel.Credentials
-	if credentials != "" {
-		tunnel.Credentials = ""
-
-		if service.chiselServer != nil {
-			service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
-		}
-	}
-
-	service.ProxyManager.DeleteEndpointProxy(endpointID)
-
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to REQUIRED.
-// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
-// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
-// Credentials are encrypted using the Edge ID associated to the environment(endpoint).
-func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
-	defer cache.Del(endpointID)
-
-	tunnel := service.getTunnelDetails(endpointID)
-
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	if tunnel.Port == 0 {
-		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
-		if err != nil {
-			return err
-		}
-
-		tunnel.Status = portainer.EdgeAgentManagementRequired
-		tunnel.Port = service.getUnusedPort()
-		tunnel.LastActivity = time.Now()
-
-		username, password := generateRandomCredentials()
-		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-
-		if service.chiselServer != nil {
-			err = service.chiselServer.AddUser(username, password, authorizedRemote)
-			if err != nil {
-				return err
-			}
-		}
-
-		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
-		if err != nil {
-			return err
-		}
-		tunnel.Credentials = credentials
-	}
-
-	return nil
-}
-
 func generateRandomCredentials() (string, string) {
 	username := uniuri.NewLen(8)
 	password := uniuri.NewLen(8)
+
 	return username, password
 }
 

api/cli/cli.go

@@ -62,7 +62,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
-		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()

api/cmd/portainer/log.go

@@ -42,6 +42,13 @@ func setLoggingMode(mode string) {
 			TimeFormat:    "2006/01/02 03:04PM",
 			FormatMessage: formatMessage,
 		})
+	case "NOCOLOR":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage,
+			NoColor:       true,
+		})
 	case "JSON":
 		log.Logger = log.Output(os.Stderr)
 	}

api/cmd/portainer/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/datastore/migrator"
+	"github.com/portainer/portainer/api/datastore/postinit"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -457,19 +458,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	authorizationService := authorization.NewService(dataStore)
 	authorizationService.K8sClientFactory = kubernetesClientFactory
 
-	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
-
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed initializing snapshot service")
-	}
-	snapshotService.Start()
-
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
 
-	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
+	proxyManager := proxy.NewManager(kubernetesClientFactory)
 
 	reverseTunnelService.ProxyManager = proxyManager
 
@@ -489,6 +482,16 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
+	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, dockerClientFactory, authorizationService, shutdownCtx, *flags.Assets, kubernetesDeployer)
+
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
+	}
+	snapshotService.Start()
+
+	proxyManager.NewProxyFactory(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService)
+
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing helm package manager")
@@ -578,10 +581,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	// but some more complex migrations require access to a kubernetes or docker
 	// client. Therefore we run a separate migration process just before
 	// starting the server.
-	postInitMigrator := datastore.NewPostInitMigrator(
+	postInitMigrator := postinit.NewPostInitMigrator(
 		kubernetesClientFactory,
 		dockerClientFactory,
 		dataStore,
+		*flags.Assets,
+		kubernetesDeployer,
 	)
 	if err := postInitMigrator.PostInitMigrate(); err != nil {
 		log.Fatal().Err(err).Msg("failure during post init migrations")
@@ -650,6 +655,7 @@ func main() {
 			Msg("starting Portainer")
 
 		err := server.Start()
+
 		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/crypto/aes.go

@@ -1,52 +1,216 @@
 package crypto
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
 	"io"
 
+	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/scrypt"
 )
 
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
+const (
+	// AES GCM settings
+	aesGcmHeader    = "AES256-GCM" // The encrypted file header
+	aesGcmBlockSize = 1024 * 1024  // 1MB block for aes gcm
 
-var emptySalt []byte = make([]byte, 0)
+	// Argon2 settings
+	// Recommded settings lower memory hardware according to current OWASP recommendations
+	// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
+	// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
+	argon2MemoryCost = 12 * 1024
+	argon2TimeCost   = 3
+	argon2Threads    = 1
+	argon2KeyLength  = 32
+)
 
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
+// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
 func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	err := aesEncryptGCM(input, output, passphrase)
 	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
+		return fmt.Errorf("error encrypting file: %w", err)
 	}
 
 	return nil
 }
 
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
 func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Read file header to determine how it was encrypted
+	inputReader := bufio.NewReader(input)
+	header, err := inputReader.Peek(len(aesGcmHeader))
+	if err != nil {
+		return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
+	}
+
+	if string(header) == aesGcmHeader {
+		reader, err := aesDecryptGCM(inputReader, passphrase)
+		if err != nil {
+			return nil, fmt.Errorf("error decrypting file: %w", err)
+		}
+
+		return reader, nil
+	}
+
+	// Use the previous decryption routine which has no header (to support older archives)
+	reader, err := aesDecryptOFB(inputReader, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
+	}
+
+	return reader, nil
+}
+
+// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
+func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
+	// Derive key using argon2 with a random salt
+	salt := make([]byte, 16) // 16 bytes salt
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		return err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return err
+	}
+
+	// Generate nonce
+	nonce, err := NewRandomNonce(aesgcm.NonceSize())
+	if err != nil {
+		return err
+	}
+
+	// write the header
+	if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
+		return err
+	}
+
+	// Write nonce and salt to the output file
+	if _, err := output.Write(salt); err != nil {
+		return err
+	}
+	if _, err := output.Write(nonce.Value()); err != nil {
+		return err
+	}
+
+	// Buffer for reading plaintext blocks
+	buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
+	ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
+
+	// Encrypt plaintext in blocks
+	for {
+		n, err := io.ReadFull(input, buf)
+		if n == 0 {
+			break // end of plaintext input
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return err
+		}
+
+		// Seal encrypts the plaintext using the nonce returning the updated slice.
+		ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
+
+		_, err = output.Write(ciphertext)
+		if err != nil {
+			return err
+		}
+
+		nonce.Increment()
+	}
+
+	return nil
+}
+
+// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
+func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Reader & verify header
+	header := make([]byte, len(aesGcmHeader))
+	if _, err := io.ReadFull(input, header); err != nil {
+		return nil, err
+	}
+
+	if string(header) != aesGcmHeader {
+		return nil, fmt.Errorf("invalid header")
+	}
+
+	// Read salt
+	salt := make([]byte, 16) // Salt size
+	if _, err := io.ReadFull(input, salt); err != nil {
+		return nil, err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+
+	// Initialize AES cipher block
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create GCM mode with the cipher block
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read nonce from the input reader
+	nonce := NewNonce(aesgcm.NonceSize())
+	if err := nonce.Read(input); err != nil {
+		return nil, err
+	}
+
+	// Initialize a buffer to store decrypted data
+	buf := bytes.Buffer{}
+	plaintext := make([]byte, aesGcmBlockSize)
+
+	// Decrypt the ciphertext in blocks
+	for {
+		// Read a block of ciphertext from the input reader
+		ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
+		n, err := io.ReadFull(input, ciphertextBlock)
+		if n == 0 {
+			break // end of ciphertext
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return nil, err
+		}
+
+		// Decrypt the block of ciphertext
+		plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = buf.Write(plaintext)
+		if err != nil {
+			return nil, err
+		}
+
+		nonce.Increment()
+	}
+
+	return &buf, nil
+}
+
+// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+// note: This function used to decrypt files that were encrypted without a header i.e. old archives
+func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
+	var emptySalt []byte = make([]byte, 0)
+
 	// making a 32 bytes key that would correspond to AES-256
 	// don't necessarily need a salt, so just kept in empty
 	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
@@ -59,11 +223,9 @@ func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
 		return nil, err
 	}
 
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
+	// If the key is unique for each ciphertext, then it's ok to use a zero IV.
 	var iv [aes.BlockSize]byte
 	stream := cipher.NewOFB(block, iv[:])
-
 	reader := &cipher.StreamReader{S: stream, R: input}
 
 	return reader, nil

api/crypto/aes_test.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -9,7 +10,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func randBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	const passphrase = "passphrase"
+
 	tmpdir := t.TempDir()
 
 	var (
@@ -18,17 +31,99 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024*1024*100 + 523)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withStrongPassphrase(t *testing.T) {
+	const passphrase = "A strong passphrase with special characters: !@#$%^&*()_+"
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withTheSamePasswordSmallFile(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
 
 	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
 	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
@@ -57,7 +152,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024 * 50)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -96,7 +191,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1034)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -117,11 +212,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	decryptedFileWriter, _ := os.Create(decryptedFilePath)
 	defer decryptedFileWriter.Close()
 
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := os.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+	_, err = AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.NotNil(t, err, "Should not allow decrypt with wrong passphrase")
 }

api/crypto/nonce.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+type Nonce struct {
+	val []byte
+}
+
+func NewNonce(size int) *Nonce {
+	return &Nonce{val: make([]byte, size)}
+}
+
+// NewRandomNonce generates a new initial nonce with the lower byte set to a random value
+// This ensures there are plenty of nonce values availble before rolling over
+// Based on ideas from the Secure Programming Cookbook for C and C++ by John Viega, Matt Messier
+// https://www.oreilly.com/library/view/secure-programming-cookbook/0596003943/ch04s09.html
+func NewRandomNonce(size int) (*Nonce, error) {
+	randomBytes := 1
+	if size <= randomBytes {
+		return nil, errors.New("nonce size must be greater than the number of random bytes")
+	}
+
+	randomPart := make([]byte, randomBytes)
+	if _, err := rand.Read(randomPart); err != nil {
+		return nil, err
+	}
+
+	zeroPart := make([]byte, size-randomBytes)
+	nonceVal := append(randomPart, zeroPart...)
+	return &Nonce{val: nonceVal}, nil
+}
+
+func (n *Nonce) Read(stream io.Reader) error {
+	_, err := io.ReadFull(stream, n.val)
+	return err
+}
+
+func (n *Nonce) Value() []byte {
+	return n.val
+}
+
+func (n *Nonce) Increment() error {
+	// Start incrementing from the least significant byte
+	for i := len(n.val) - 1; i >= 0; i-- {
+		// Increment the current byte
+		n.val[i]++
+
+		// Check for overflow
+		if n.val[i] != 0 {
+			// No overflow, nonce is successfully incremented
+			return nil
+		}
+	}
+
+	// If we reach here, it means the nonce has overflowed
+	return errors.New("nonce overflow")
+}

api/crypto/tls.go

@@ -22,6 +22,12 @@ func CreateTLSConfiguration() *tls.Config {
 			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
 		},
 	}
 }

api/dataservices/interface.go

@@ -36,6 +36,7 @@ type (
 	}
 
 	DataStore interface {
+		Connection() portainer.Connection
 		Open() (newStore bool, err error)
 		Init() error
 		Close() error
@@ -73,6 +74,7 @@ type (
 	PendingActionsService interface {
 		BaseCRUD[portainer.PendingActions, portainer.PendingActionsID]
 		GetNextIdentifier() int
+		DeleteByEndpointID(ID portainer.EndpointID) error
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks

api/dataservices/pendingactions/pendingactions.go

@@ -1,10 +1,12 @@
 package pendingactions
 
 import (
+	"fmt"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -45,6 +47,12 @@ func (s Service) Update(ID portainer.PendingActionsID, config *portainer.Pending
 	})
 }
 
+func (s Service) DeleteByEndpointID(ID portainer.EndpointID) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).DeleteByEndpointID(ID)
+	})
+}
+
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
 		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]{
@@ -68,6 +76,29 @@ func (s ServiceTx) Update(ID portainer.PendingActionsID, config *portainer.Pendi
 	return s.BaseDataServiceTx.Update(ID, config)
 }
 
+func (s ServiceTx) DeleteByEndpointID(ID portainer.EndpointID) error {
+	log.Debug().Int("endpointId", int(ID)).Msg("deleting pending actions for endpoint")
+	pendingActions, err := s.BaseDataServiceTx.ReadAll()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve pending-actions for endpoint (%d): %w", ID, err)
+	}
+
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == ID {
+			err := s.BaseDataServiceTx.Delete(pendingAction.ID)
+			if err != nil {
+				log.Debug().Int("endpointId", int(ID)).Msgf("failed to delete pending action: %v", err)
+			}
+		}
+	}
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
 	return service.Connection.GetNextIdentifier(BucketName)

api/dataservices/team/team.go

@@ -19,8 +19,7 @@ type Service struct {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
-	if err != nil {
+	if err := connection.SetServiceName(BucketName); err != nil {
 		return nil, err
 	}
 
@@ -32,6 +31,16 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
+	}
+}
+
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	var t portainer.Team

api/dataservices/team/tx.go

@@ -0,0 +1,48 @@
+package team
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]
+}
+
+// TeamByName returns a team by name.
+func (service ServiceTx) TeamByName(name string) (*portainer.Team, error) {
+	var t portainer.Team
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Team{},
+		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+			return strings.EqualFold(e.Name, name)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &t, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// CreateTeam creates a new Team.
+func (service ServiceTx) Create(team *portainer.Team) error {
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, any) {
+			team.ID = portainer.TeamID(id)
+			return int(team.ID), team
+		},
+	)
+}

api/datastore/migrate_data.go

@@ -86,6 +86,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		EdgeStackService:        store.EdgeStackService,
 		EdgeJobService:          store.EdgeJobService,
 		TunnelServerService:     store.TunnelServerService,
+		PendingActionsService:   store.PendingActionsService,
 	}
 }
 

api/datastore/migrate_post_init.go

@@ -1,117 +0,0 @@
-package datastore
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
-	"github.com/portainer/portainer/api/kubernetes/cli"
-
-	"github.com/rs/zerolog/log"
-)
-
-type PostInitMigrator struct {
-	kubeFactory   *cli.ClientFactory
-	dockerFactory *dockerclient.ClientFactory
-	dataStore     dataservices.DataStore
-}
-
-func NewPostInitMigrator(kubeFactory *cli.ClientFactory, dockerFactory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *PostInitMigrator {
-	return &PostInitMigrator{
-		kubeFactory:   kubeFactory,
-		dockerFactory: dockerFactory,
-		dataStore:     dataStore,
-	}
-}
-
-func (migrator *PostInitMigrator) PostInitMigrate() error {
-	if err := migrator.PostInitMigrateIngresses(); err != nil {
-		return err
-	}
-
-	migrator.PostInitMigrateGPUs()
-
-	return nil
-}
-
-func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
-	endpoints, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for i := range endpoints {
-		// Early exit if we do not need to migrate!
-		if !endpoints[i].PostInitMigrations.MigrateIngresses {
-			return nil
-		}
-
-		err := migrator.kubeFactory.MigrateEndpointIngresses(&endpoints[i])
-		if err != nil {
-			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
-		}
-	}
-
-	return nil
-}
-
-// PostInitMigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
-// If there's an error getting the containers, we'll log it and move on
-func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
-	environments, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		log.Err(err).Msg("failure getting endpoints")
-		return
-	}
-
-	for i := range environments {
-		if environments[i].Type == portainer.DockerEnvironment {
-			// // Early exit if we do not need to migrate!
-			if !environments[i].PostInitMigrations.MigrateGPUs {
-				return
-			}
-
-			// set the MigrateGPUs flag to false so we don't run this again
-			environments[i].PostInitMigrations.MigrateGPUs = false
-			migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-			// create a docker client
-			dockerClient, err := migrator.dockerFactory.CreateClient(&environments[i], "", nil)
-			if err != nil {
-				log.Err(err).Msg("failure creating docker client for environment: " + environments[i].Name)
-				return
-			}
-			defer dockerClient.Close()
-
-			// get all containers
-			containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
-			if err != nil {
-				log.Err(err).Msg("failed to list containers")
-				return
-			}
-
-			// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole endpoint
-		containersLoop:
-			for _, container := range containers {
-				// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
-				containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
-				if err != nil {
-					log.Err(err).Msg("failed to inspect container")
-					return
-				}
-
-				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
-				for _, deviceRequest := range deviceRequests {
-					if deviceRequest.Driver == "nvidia" {
-						environments[i].EnableGPUManagement = true
-						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-						break containersLoop
-					}
-				}
-			}
-		}
-	}
-}

api/datastore/migrator/migrate_dbversion110.go

@@ -23,3 +23,29 @@ func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
 
 	return migrator.settingsService.UpdateSettings(settings)
 }
+
+// In PortainerCE the resource overcommit option should always be true across all endpoints
+func (migrator *Migrator) updateResourceOverCommitToDB110() error {
+	log.Info().Msg("updating resource overcommit setting to true")
+
+	endpoints, err := migrator.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.Type == portainer.KubernetesLocalEnvironment ||
+			endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
+			endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+
+			endpoint.Kubernetes.Configuration.EnableResourceOverCommit = true
+
+			err = migrator.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion111.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
+)
+
+func (migrator *Migrator) cleanPendingActionsForDeletedEndpointsForDB111() error {
+	log.Info().Msg("cleaning up pending actions for deleted endpoints")
+
+	pendingActions, err := migrator.pendingActionsService.ReadAll()
+	if err != nil {
+		return err
+	}
+
+	endpoints := make(map[portainer.EndpointID]struct{})
+	for _, action := range pendingActions {
+		endpoints[action.EndpointID] = struct{}{}
+	}
+
+	for endpointId := range endpoints {
+		_, err := migrator.endpointService.Endpoint(endpointId)
+		if dataservices.IsErrObjectNotFound(err) {
+			err := migrator.pendingActionsService.DeleteByEndpointID(endpointId)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -14,6 +14,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
 	"github.com/portainer/portainer/api/dataservices/extension"
 	"github.com/portainer/portainer/api/dataservices/fdoprofile"
+	"github.com/portainer/portainer/api/dataservices/pendingactions"
 	"github.com/portainer/portainer/api/dataservices/registry"
 	"github.com/portainer/portainer/api/dataservices/resourcecontrol"
 	"github.com/portainer/portainer/api/dataservices/role"
@@ -58,6 +59,7 @@ type (
 		edgeStackService        *edgestack.Service
 		edgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		pendingActionsService   *pendingactions.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -85,6 +87,7 @@ type (
 		EdgeStackService        *edgestack.Service
 		EdgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		PendingActionsService   *pendingactions.Service
 	}
 )
 
@@ -114,6 +117,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		edgeStackService:        parameters.EdgeStackService,
 		edgeJobService:          parameters.EdgeJobService,
 		TunnelServerService:     parameters.TunnelServerService,
+		pendingActionsService:   parameters.PendingActionsService,
 	}
 
 	migrator.initMigrations()
@@ -230,6 +234,10 @@ func (m *Migrator) initMigrations() {
 	)
 	m.addMigrations("2.20",
 		m.updateAppTemplatesVersionForDB110,
+		m.updateResourceOverCommitToDB110,
+	)
+	m.addMigrations("2.20.2",
+		m.cleanPendingActionsForDeletedEndpointsForDB111,
 	)
 
 	// Add new migrations below...

api/datastore/pendingactions_test.go

@@ -0,0 +1,95 @@
+package datastore
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+)
+
+func Test_ConvertCleanNAPWithOverridePoliciesPayload(t *testing.T) {
+	t.Run("test ConvertCleanNAPWithOverridePoliciesPayload", func(t *testing.T) {
+
+		_, store := MustNewTestStore(t, true, false)
+		defer store.Close()
+
+		testData := []struct {
+			Name          string
+			PendingAction portainer.PendingActions
+			Expected      *actions.CleanNAPWithOverridePoliciesPayload
+			Err           bool
+		}{
+			{
+				Name: "test actiondata with EndpointGroupID 1",
+				PendingAction: portainer.PendingActions{
+					EndpointID: 1,
+					Action:     "CleanNAPWithOverridePolicies",
+					ActionData: &actions.CleanNAPWithOverridePoliciesPayload{
+						EndpointGroupID: 1,
+					},
+				},
+				Expected: &actions.CleanNAPWithOverridePoliciesPayload{
+					EndpointGroupID: 1,
+				},
+			},
+			{
+				Name: "test actionData nil",
+				PendingAction: portainer.PendingActions{
+					EndpointID: 2,
+					Action:     "CleanNAPWithOverridePolicies",
+					ActionData: nil,
+				},
+				Expected: nil,
+			},
+			{
+				Name: "test actionData empty and expected error",
+				PendingAction: portainer.PendingActions{
+					EndpointID: 2,
+					Action:     "CleanNAPWithOverridePolicies",
+					ActionData: "",
+				},
+				Expected: nil,
+				Err:      true,
+			},
+		}
+
+		for _, d := range testData {
+			err := store.PendingActions().Create(&d.PendingAction)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			pendingActions, err := store.PendingActions().ReadAll()
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			for _, endpointPendingAction := range pendingActions {
+				t.Run(d.Name, func(t *testing.T) {
+					if endpointPendingAction.Action == "CleanNAPWithOverridePolicies" {
+						actionData, err := actions.ConvertCleanNAPWithOverridePoliciesPayload(endpointPendingAction.ActionData)
+						if d.Err && err == nil {
+							t.Error(err)
+						}
+
+						if d.Expected == nil && actionData != nil {
+							t.Errorf("expected nil , got %d", actionData)
+						}
+
+						if d.Expected != nil && actionData == nil {
+							t.Errorf("expected not nil , got %d", actionData)
+						}
+
+						if d.Expected != nil && actionData.EndpointGroupID != d.Expected.EndpointGroupID {
+							t.Errorf("expected EndpointGroupID %d , got %d", d.Expected.EndpointGroupID, actionData.EndpointGroupID)
+						}
+					}
+				})
+			}
+
+			store.PendingActions().Delete(d.PendingAction.ID)
+		}
+	})
+}

api/datastore/postinit/migrate_post_init.go

@@ -0,0 +1,203 @@
+package postinit
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerClient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+
+	"github.com/rs/zerolog/log"
+)
+
+type PostInitMigrator struct {
+	kubeFactory        *cli.ClientFactory
+	dockerFactory      *dockerClient.ClientFactory
+	dataStore          dataservices.DataStore
+	assetsPath         string
+	kubernetesDeployer portainer.KubernetesDeployer
+}
+
+func NewPostInitMigrator(
+	kubeFactory *cli.ClientFactory,
+	dockerFactory *dockerClient.ClientFactory,
+	dataStore dataservices.DataStore,
+	assetsPath string,
+	kubernetesDeployer portainer.KubernetesDeployer,
+) *PostInitMigrator {
+	return &PostInitMigrator{
+		kubeFactory:        kubeFactory,
+		dockerFactory:      dockerFactory,
+		dataStore:          dataStore,
+		assetsPath:         assetsPath,
+		kubernetesDeployer: kubernetesDeployer,
+	}
+}
+
+// PostInitMigrate will run all post-init migrations, which require docker/kube clients for all edge or non-edge environments
+func (postInitMigrator *PostInitMigrator) PostInitMigrate() error {
+	environments, err := postInitMigrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		log.Error().Err(err).Msg("Error getting environments")
+		return err
+	}
+
+	for _, environment := range environments {
+		// edge environments will run after the server starts, in pending actions
+		if endpointutils.IsEdgeEndpoint(&environment) {
+			log.Info().Msgf("Adding pending action 'PostInitMigrateEnvironment' for environment %d", environment.ID)
+			err = postInitMigrator.createPostInitMigrationPendingAction(environment.ID)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error creating pending action for environment %d", environment.ID)
+			}
+		} else {
+			// non-edge environments will run before the server starts.
+			err = postInitMigrator.MigrateEnvironment(&environment)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error running post-init migrations for non-edge environment %d", environment.ID)
+			}
+		}
+
+	}
+
+	return nil
+}
+
+// try to create a post init migration pending action. If it already exists, do nothing
+// this function exists for readability, not reusability
+// TODO: This should be moved into pending actions as part of the pending action migration
+func (postInitMigrator *PostInitMigrator) createPostInitMigrationPendingAction(environmentID portainer.EndpointID) error {
+	migrateEnvPendingAction := portainer.PendingActions{
+		EndpointID: environmentID,
+		Action:     actions.PostInitMigrateEnvironment,
+	}
+
+	// Get all pending actions and filter them by endpoint, action and action args that are equal to the migrateEnvPendingAction
+	pendingActions, err := postInitMigrator.dataStore.PendingActions().ReadAll()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error retrieving pending actions")
+		return fmt.Errorf("failed to retrieve pending actions for environment %d: %w", environmentID, err)
+	}
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == environmentID &&
+			pendingAction.Action == migrateEnvPendingAction.Action &&
+			reflect.DeepEqual(pendingAction.ActionData, migrateEnvPendingAction.ActionData) {
+			log.Debug().Msgf("Migration pending action for environment %d already exists, skipping creating another", environmentID)
+			return nil
+		}
+	}
+
+	// If there are no pending actions for the given endpoint, create one
+	err = postInitMigrator.dataStore.PendingActions().Create(&migrateEnvPendingAction)
+	if err != nil {
+		log.Error().Err(err).Msgf("Error creating pending action for environment %d", environmentID)
+	}
+	return nil
+}
+
+// MigrateEnvironment runs migrations on a single environment
+func (migrator *PostInitMigrator) MigrateEnvironment(environment *portainer.Endpoint) error {
+	log.Info().Msgf("Executing post init migration for environment %d", environment.ID)
+
+	switch {
+	case endpointutils.IsKubernetesEndpoint(environment):
+		// get the kubeclient for the environment, and skip all kube migrations if there's an error
+		kubeclient, err := migrator.kubeFactory.GetKubeClient(environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating kubeclient for environment: %d", environment.ID)
+			return err
+		}
+		// if one environment fails, it is logged and the next migration runs. The error is returned at the end and handled by pending actions
+		err = migrator.MigrateIngresses(*environment, kubeclient)
+		if err != nil {
+			return err
+		}
+		return nil
+	case endpointutils.IsDockerEndpoint(environment):
+		// get the docker client for the environment, and skip all docker migrations if there's an error
+		dockerClient, err := migrator.dockerFactory.CreateClient(environment, "", nil)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating docker client for environment: %d", environment.ID)
+			return err
+		}
+		defer dockerClient.Close()
+		migrator.MigrateGPUs(*environment, dockerClient)
+	}
+
+	return nil
+}
+
+func (migrator *PostInitMigrator) MigrateIngresses(environment portainer.Endpoint, kubeclient *cli.KubeClient) error {
+	// Early exit if we do not need to migrate!
+	if !environment.PostInitMigrations.MigrateIngresses {
+		return nil
+	}
+	log.Debug().Msgf("Migrating ingresses for environment %d", environment.ID)
+
+	err := migrator.kubeFactory.MigrateEndpointIngresses(&environment, migrator.dataStore, kubeclient)
+	if err != nil {
+		log.Error().Err(err).Msgf("Error migrating ingresses for environment %d", environment.ID)
+		return err
+	}
+	return nil
+}
+
+// MigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
+// If there's an error getting the containers, we'll log it and move on
+func (migrator *PostInitMigrator) MigrateGPUs(e portainer.Endpoint, dockerClient *client.Client) error {
+	return migrator.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		environment, err := tx.Endpoint().Endpoint(e.ID)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting environment %d", environment.ID)
+			return err
+		}
+		// Early exit if we do not need to migrate!
+		if !environment.PostInitMigrations.MigrateGPUs {
+			return nil
+		}
+		log.Debug().Msgf("Migrating GPUs for environment %d", e.ID)
+
+		// get all containers
+		containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{All: true})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to list containers for environment %d", environment.ID)
+			return err
+		}
+
+		// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole environment
+	containersLoop:
+		for _, container := range containers {
+			// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
+			containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to inspect container")
+				continue
+			}
+
+			deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
+			for _, deviceRequest := range deviceRequests {
+				if deviceRequest.Driver == "nvidia" {
+					environment.EnableGPUManagement = true
+					break containersLoop
+				}
+			}
+		}
+
+		// set the MigrateGPUs flag to false so we don't run this again
+		environment.PostInitMigrations.MigrateGPUs = false
+		err = tx.Endpoint().UpdateEndpoint(environment.ID, environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error updating EnableGPUManagement flag for environment %d", environment.ID)
+			return err
+		}
+
+		return nil
+	})
+}

api/datastore/services.go

@@ -402,7 +402,6 @@ type storeExport struct {
 }
 
 func (store *Store) Export(filename string) (err error) {
-
 	backup := storeExport{}
 
 	if c, err := store.CustomTemplate().ReadAll(); err != nil {
@@ -606,6 +605,7 @@ func (store *Store) Export(filename string) (err error) {
 	if err != nil {
 		return err
 	}
+
 	return os.WriteFile(filename, b, 0600)
 }
 

api/datastore/services_tx.go

@@ -16,7 +16,9 @@ func (tx *StoreTx) IsErrObjectNotFound(err error) bool {
 
 func (tx *StoreTx) CustomTemplate() dataservices.CustomTemplateService { return nil }
 
-func (tx *StoreTx) PendingActions() dataservices.PendingActionsService { return nil }
+func (tx *StoreTx) PendingActions() dataservices.PendingActionsService {
+	return tx.store.PendingActionsService.Tx(tx.tx)
+}
 
 func (tx *StoreTx) EdgeGroup() dataservices.EdgeGroupService {
 	return tx.store.EdgeGroupService.Tx(tx.tx)
@@ -78,7 +80,10 @@ func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
 	return tx.store.TeamMembershipService.Tx(tx.tx)
 }
 
-func (tx *StoreTx) Team() dataservices.TeamService                 { return nil }
+func (tx *StoreTx) Team() dataservices.TeamService {
+	return tx.store.TeamService.Tx(tx.tx)
+}
+
 func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }
 
 func (tx *StoreTx) User() dataservices.UserService {

api/datastore/test_data/output_24_to_latest.json

@@ -631,6 +631,7 @@
     "LogoURL": "",
     "OAuthSettings": {
       "AccessTokenURI": "",
+      "AuthStyle": 0,
       "AuthorizationURI": "",
       "ClientID": "",
       "DefaultTeamID": 0,
@@ -677,6 +678,7 @@
             "Architecture": "",
             "BridgeNfIp6tables": false,
             "BridgeNfIptables": false,
+            "CDISpecDirs": null,
             "CPUSet": false,
             "CPUShares": false,
             "CgroupDriver": "",
@@ -939,6 +941,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.21.1\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/client/client.go

@@ -3,7 +3,6 @@ package client
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"io"
 	"maps"
 	"net/http"
@@ -13,7 +12,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	"github.com/segmentio/encoding/json"
 )
@@ -50,12 +49,12 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
 	case portainer.AgentOnDockerEnvironment:
 		return createAgentClient(endpoint, endpoint.URL, factory.signatureService, nodeName, timeout)
 	case portainer.EdgeAgentOnDockerEnvironment:
-		tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint)
+		tunnelAddr, err := factory.reverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return nil, err
 		}
 
-		endpointURL := fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+		endpointURL := "http://" + tunnelAddr
 
 		return createAgentClient(endpoint, endpointURL, factory.signatureService, nodeName, timeout)
 	}
@@ -93,11 +92,17 @@ func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*cli
 		return nil, err
 	}
 
-	return client.NewClientWithOpts(
+	opts := []client.Opt{
 		client.WithHost(endpoint.URL),
 		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
-	)
+	}
+
+	if nnTransport, ok := httpCli.Transport.(*NodeNameTransport); ok && nnTransport.TLSClientConfig != nil {
+		opts = append(opts, client.WithScheme("https"))
+	}
+
+	return client.NewClientWithOpts(opts...)
 }
 
 func createAgentClient(endpoint *portainer.Endpoint, endpointURL string, signatureService portainer.DigitalSignatureService, nodeName string, timeout *time.Duration) (*client.Client, error) {
@@ -159,7 +164,7 @@ func (t *NodeNameTransport) RoundTrip(req *http.Request) (*http.Response, error)
 	resp.Body = io.NopCloser(bytes.NewReader(body))
 
 	var rs []struct {
-		types.ImageSummary
+		image.Summary
 		Portainer struct {
 			Agent struct {
 				NodeName string

api/docker/container.go

@@ -4,15 +4,17 @@ import (
 	"context"
 	"strings"
 
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/docker/images"
+
+	"github.com/Masterminds/semver"
 	"github.com/docker/docker/api/types"
 	dockercontainer "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
-	"github.com/portainer/portainer/api/docker/images"
 	"github.com/rs/zerolog/log"
 )
 
@@ -30,6 +32,44 @@ func NewContainerService(factory *dockerclient.ClientFactory, dataStore dataserv
 	}
 }
 
+// applyVersionConstraint uses the version to apply a transformation function to
+// the value when the constraint is satisfied
+func applyVersionConstraint[T any](currentVersion, versionConstraint string, value T, transform func(T) T) (T, error) {
+	newValue := value
+
+	constraint, err := semver.NewConstraint(versionConstraint)
+	if err != nil {
+		return newValue, errors.New("invalid version constraint specified")
+	}
+
+	currentVer, err := semver.NewVersion(currentVersion)
+	if err != nil {
+		log.Warn().Err(err).Msg("Unable to parse the Docker client version")
+
+		return newValue, nil
+	}
+
+	if satisfiesConstraint, _ := constraint.Validate(currentVer); satisfiesConstraint {
+		newValue = transform(value)
+	}
+
+	return newValue, nil
+}
+
+func clearMacAddrs(n network.NetworkingConfig) network.NetworkingConfig {
+	netConfig := network.NetworkingConfig{
+		EndpointsConfig: make(map[string]*network.EndpointSettings),
+	}
+
+	for k := range n.EndpointsConfig {
+		endpointConfig := n.EndpointsConfig[k].Copy()
+		endpointConfig.MacAddress = ""
+		netConfig.EndpointsConfig[k] = endpointConfig
+	}
+
+	return netConfig
+}
+
 // Recreate a container
 func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerId string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error) {
 	cli, err := c.factory.CreateClient(endpoint, nodeName, nil)
@@ -90,7 +130,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 		return nil, errors.Wrap(err, "rename container error")
 	}
 
-	networkWithCreation := network.NetworkingConfig{
+	initialNetwork := network.NetworkingConfig{
 		EndpointsConfig: make(map[string]*network.EndpointSettings),
 	}
 
@@ -103,10 +143,10 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 		}
 
 		// 5. get the first network attached to the current container
-		if len(networkWithCreation.EndpointsConfig) == 0 {
+		if len(initialNetwork.EndpointsConfig) == 0 {
 			// Retrieve the first network that is linked to the present container, which
 			// will be utilized when creating the container.
-			networkWithCreation.EndpointsConfig[name] = network
+			initialNetwork.EndpointsConfig[name] = network
 		}
 	}
 	c.sr.enable()
@@ -119,7 +159,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 		for _, network := range container.NetworkSettings.Networks {
 			cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
 		}
-		cli.ContainerStart(ctx, containerId, types.ContainerStartOptions{})
+		cli.ContainerStart(ctx, containerId, dockercontainer.StartOptions{})
 	})
 
 	log.Debug().Str("container", strings.Split(container.Name, "/")[1]).Msg("starting to create a new container")
@@ -130,12 +170,20 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 	// to retain the same network settings we have to connect on creation to one of the old
 	// container's networks, and connect to the other networks after creation.
 	// see: https://portainer.atlassian.net/browse/EE-5448
-	create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &networkWithCreation, nil, container.Name)
+
+	// Docker API < 1.44 does not support specifying MAC addresses
+	// https://github.com/moby/moby/blob/6aea26b431ea152a8b085e453da06ea403f89886/client/container_create.go#L44-L46
+	initialNetwork, err = applyVersionConstraint(cli.ClientVersion(), "< 1.44", initialNetwork, clearMacAddrs)
+	if err != nil {
+		return nil, err
+	}
+
+	create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &initialNetwork, nil, container.Name)
 
 	c.sr.push(func() {
 		log.Debug().Str("container_id", create.ID).Msg("removing the new container")
 		cli.ContainerStop(ctx, create.ID, dockercontainer.StopOptions{})
-		cli.ContainerRemove(ctx, create.ID, types.ContainerRemoveOptions{})
+		cli.ContainerRemove(ctx, create.ID, dockercontainer.RemoveOptions{})
 	})
 
 	if err != nil {
@@ -150,8 +198,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 	log.Debug().Str("container_id", newContainerId).Msg("connecting networks to container")
 	networks := container.NetworkSettings.Networks
 	for key, network := range networks {
-		_, ok := networkWithCreation.EndpointsConfig[key]
-		if ok {
+		if _, ok := initialNetwork.EndpointsConfig[key]; ok {
 			// skip the network that is used during container creation
 			continue
 		}
@@ -164,14 +211,14 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 
 	// 8. start the new container
 	log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
-	err = cli.ContainerStart(ctx, newContainerId, types.ContainerStartOptions{})
+	err = cli.ContainerStart(ctx, newContainerId, dockercontainer.StartOptions{})
 	if err != nil {
 		return nil, errors.Wrap(err, "start container error")
 	}
 
 	// 9. delete the old container
 	log.Debug().Str("container_id", containerId).Msg("starting to remove the old container")
-	_ = cli.ContainerRemove(ctx, containerId, types.ContainerRemoveOptions{})
+	_ = cli.ContainerRemove(ctx, containerId, dockercontainer.RemoveOptions{})
 
 	c.sr.disable()
 

api/docker/container_test.go

@@ -0,0 +1,52 @@
+package docker
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types/network"
+	"github.com/stretchr/testify/require"
+)
+
+func TestApplyVersionConstraint(t *testing.T) {
+	initialNet := network.NetworkingConfig{
+		EndpointsConfig: map[string]*network.EndpointSettings{
+			"key1": {
+				MacAddress: "mac1",
+				EndpointID: "endpointID1",
+			},
+			"key2": {
+				MacAddress: "mac2",
+				EndpointID: "endpointID2",
+			},
+		},
+	}
+
+	f := func(currentVer string, constraint string, success, emptyMac bool) {
+		t.Helper()
+
+		transformedNet, err := applyVersionConstraint(currentVer, constraint, initialNet, clearMacAddrs)
+		if success {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err)
+		}
+
+		require.Len(t, transformedNet.EndpointsConfig, len(initialNet.EndpointsConfig))
+
+		for k := range initialNet.EndpointsConfig {
+			if emptyMac {
+				require.NotEqual(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
+				require.Empty(t, transformedNet.EndpointsConfig[k].MacAddress)
+
+				continue
+			}
+
+			require.Equal(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
+		}
+	}
+
+	f("1.45", "< 1.44", true, false)  // No transformation
+	f("1.43", "< 1.44", true, true)   // Transformation
+	f("a.b.", "< 1.44", true, false)  // Invalid current version
+	f("1.45", "z 1.44", false, false) // Invalid version constraint
+}

api/docker/images/status.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	portainer "github.com/portainer/portainer/api"
 	consts "github.com/portainer/portainer/api/docker/consts"
@@ -157,7 +158,7 @@ func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string,
 		return Error, nil
 	}
 
-	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
+	containers, err := cli.ContainerList(ctx, container.ListOptions{
 		All:     true,
 		Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+serviceID)),
 	})

api/docker/snapshot.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	_container "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
@@ -147,7 +148,7 @@ func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Clien
 }
 
 func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
-	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
 	if err != nil {
 		return err
 	}

api/exec/swarm_stack.go

@@ -3,7 +3,6 @@ package exec
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"os"
 	"os/exec"
 	"path"
@@ -186,11 +185,11 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, config
 
 	endpointURL := endpoint.URL
 	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		tunnel, err := manager.reverseTunnelService.GetActiveTunnel(endpoint)
+		tunnelAddr, err := manager.reverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return "", nil, err
 		}
-		endpointURL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnel.Port)
+		endpointURL = "tcp://" + tunnelAddr
 	}
 
 	args = append(args, "-H", endpointURL)

api/filesystem/filesystem.go

@@ -934,7 +934,7 @@ func FileExists(filePath string) (bool, error) {
 func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
 	// 1. Backup the source directory to a different folder
 	backupDir := fmt.Sprintf("%s-%s", filepath.Dir(originalPath), "backup")
-	err := MoveDirectory(originalPath, backupDir)
+	err := MoveDirectory(originalPath, backupDir, false)
 	if err != nil {
 		return fmt.Errorf("failed to backup source directory: %w", err)
 	}
@@ -973,14 +973,14 @@ func restoreBackup(src, backupDir string) error {
 		return fmt.Errorf("failed to delete destination directory: %w", err)
 	}
 
-	err = MoveDirectory(backupDir, src)
+	err = MoveDirectory(backupDir, src, false)
 	if err != nil {
 		return fmt.Errorf("failed to restore backup directory: %w", err)
 	}
 	return nil
 }
 
-func MoveDirectory(originalPath, newPath string) error {
+func MoveDirectory(originalPath, newPath string, overwriteTargetPath bool) error {
 	if _, err := os.Stat(originalPath); err != nil {
 		return err
 	}
@@ -991,7 +991,13 @@ func MoveDirectory(originalPath, newPath string) error {
 	}
 
 	if alreadyExists {
-		return errors.New("Target path already exists")
+		if !overwriteTargetPath {
+			return fmt.Errorf("Target path already exists")
+		}
+
+		if err = os.RemoveAll(newPath); err != nil {
+			return fmt.Errorf("failed to overwrite path %s: %s", newPath, err.Error())
+		}
 	}
 
 	return os.Rename(originalPath, newPath)

api/filesystem/filesystem_move_test.go

@@ -16,7 +16,7 @@ func Test_movePath_shouldFailIfSourceDirDoesNotExist(t *testing.T) {
 	file1 := addFile(destinationDir, "dir", "file")
 	file2 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when source path is missing")
 	assert.FileExists(t, file1, "destination dir contents should remain")
 	assert.FileExists(t, file2, "destination dir contents should remain")
@@ -30,7 +30,7 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	file3 := addFile(destinationDir, "dir", "file")
 	file4 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when destination directory already exists")
 	assert.FileExists(t, file1, "source dir contents should remain")
 	assert.FileExists(t, file2, "source dir contents should remain")
@@ -38,6 +38,22 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	assert.FileExists(t, file4, "destination dir contents should remain")
 }
 
+func Test_movePath_succesIfOverwriteSetWhenDestinationDirExists(t *testing.T) {
+	sourceDir := t.TempDir()
+	file1 := addFile(sourceDir, "dir", "file")
+	file2 := addFile(sourceDir, "file")
+	destinationDir := t.TempDir()
+	file3 := addFile(destinationDir, "dir", "file")
+	file4 := addFile(destinationDir, "file")
+
+	err := MoveDirectory(sourceDir, destinationDir, true)
+	assert.NoError(t, err)
+	assert.NoFileExists(t, file1, "source dir contents should be moved")
+	assert.NoFileExists(t, file2, "source dir contents should be moved")
+	assert.FileExists(t, file3, "destination dir contents should remain")
+	assert.FileExists(t, file4, "destination dir contents should remain")
+}
+
 func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T) {
 	tmp := t.TempDir()
 	sourceDir := path.Join(tmp, "source")
@@ -46,7 +62,7 @@ func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T)
 	file2 := addFile(sourceDir, "file")
 	destinationDir := path.Join(tmp, "destination")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.NoError(t, err)
 	assert.NoFileExists(t, file1, "source dir contents should be moved")
 	assert.NoFileExists(t, file2, "source dir contents should be moved")

api/git/backup.go

@@ -38,7 +38,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 		}
 	}
 
-	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath)
+	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath, true)
 	if err != nil {
 		return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
 	}
@@ -48,7 +48,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 	err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password, options.TLSSkipVerify)
 	if err != nil {
 		cleanUp = false
-		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath)
+		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath, false)
 		if restoreError != nil {
 			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
 		}

api/git/git.go

@@ -143,6 +143,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 		ReferenceName:   plumbing.ReferenceName(opt.referenceName),
 		Auth:            getAuth(opt.username, opt.password),
 		InsecureSkipTLS: opt.tlsSkipVerify,
+		Tags:            git.NoTags,
 	}
 
 	repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
@@ -166,7 +167,10 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 	}
 
 	var allPaths []string
+
 	w := object.NewTreeWalker(tree, true, nil)
+	defer w.Close()
+
 	for {
 		name, entry, err := w.Next()
 		if err != nil {

api/git/git_test.go

@@ -91,6 +91,29 @@ func Test_latestCommitID(t *testing.T) {
 	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
 }
 
+func Test_ListRefs(t *testing.T) {
+	service := Service{git: NewGitClient(true)}
+
+	repositoryURL := setup(t)
+
+	fs, err := service.ListRefs(repositoryURL, "", "", false, false)
+
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"refs/heads/main"}, fs)
+}
+
+func Test_ListFiles(t *testing.T) {
+	service := Service{git: NewGitClient(true)}
+
+	repositoryURL := setup(t)
+	referenceName := "refs/heads/main"
+
+	fs, err := service.ListFiles(repositoryURL, referenceName, "", "", false, false, []string{".yml"}, false)
+
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"docker-compose.yml"}, fs)
+}
+
 func getCommitHistoryLength(t *testing.T, err error, dir string) int {
 	repo, err := git.PlainOpen(dir)
 	if err != nil {

api/git/service.go

@@ -9,6 +9,7 @@ import (
 
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/sync/singleflight"
 )
 
 const (
@@ -223,11 +224,23 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 	return refs, nil
 }
 
+var singleflightGroup = &singleflight.Group{}
+
 // ListFiles will list all the files of the target repository with specific extensions.
 // If extension is not provided, it will list all the files under the target repository
 func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
 	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
 
+	fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) {
+		return service.listFiles(repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify)
+	})
+
+	return filterFiles(fs.([]string), includedExts), err
+}
+
+func (service *Service) listFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
+	repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
+
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
 		service.repoFileCache.Remove(repoKey)
@@ -235,14 +248,9 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 
 	if service.repoFileCache != nil {
 		// lookup the files cache first
-		cache, ok := service.repoFileCache.Get(repoKey)
-		if ok {
-			files, success := cache.([]string)
-			if success {
-				// For the case while searching files in a repository without include extensions for the first time,
-				// but with include extensions for the second time
-				includedFiles := filterFiles(files, includedExts)
-				return includedFiles, nil
+		if cache, ok := service.repoFileCache.Get(repoKey); ok {
+			if files, ok := cache.([]string); ok {
+				return files, nil
 			}
 		}
 	}
@@ -274,12 +282,11 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
 		}
 	}
 
-	includedFiles := filterFiles(files, includedExts)
 	if service.cacheEnabled && service.repoFileCache != nil {
-		service.repoFileCache.Add(repoKey, includedFiles)
-		return includedFiles, nil
+		service.repoFileCache.Add(repoKey, files)
 	}
-	return includedFiles, nil
+
+	return files, nil
 }
 
 func (service *Service) purgeCache() {

api/http/csrf/csrf.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"net/http"
+	"os"
 
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
@@ -13,6 +14,13 @@ import (
 )
 
 func WithProtect(handler http.Handler) (http.Handler, error) {
+	// IsDockerDesktopExtension is used to check if we should skip csrf checks in the request bouncer (ShouldSkipCSRFCheck)
+	// DOCKER_EXTENSION is set to '1' in build/docker-extension/docker-compose.yml
+	isDockerDesktopExtension := false
+	if val, ok := os.LookupEnv("DOCKER_EXTENSION"); ok && val == "1" {
+		isDockerDesktopExtension = true
+	}
+
 	handler = withSendCSRFToken(handler)
 
 	token := make([]byte, 32)
@@ -21,9 +29,13 @@ func WithProtect(handler http.Handler) (http.Handler, error) {
 		return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
 	}
 
-	handler = gorillacsrf.Protect([]byte(token), gorillacsrf.Path("/"))(handler)
+	handler = gorillacsrf.Protect(
+		[]byte(token),
+		gorillacsrf.Path("/"),
+		gorillacsrf.Secure(false),
+	)(handler)
 
-	return withSkipCSRF(handler), nil
+	return withSkipCSRF(handler, isDockerDesktopExtension), nil
 }
 
 func withSendCSRFToken(handler http.Handler) http.Handler {
@@ -44,10 +56,10 @@ func withSendCSRFToken(handler http.Handler) http.Handler {
 	})
 }
 
-func withSkipCSRF(handler http.Handler) http.Handler {
+func withSkipCSRF(handler http.Handler, isDockerDesktopExtension bool) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 
-		skip, err := security.ShouldSkipCSRFCheck(r)
+		skip, err := security.ShouldSkipCSRFCheck(r, isDockerDesktopExtension)
 		if err != nil {
 			httperror.WriteError(w, http.StatusForbidden, err.Error(), err)
 			return

api/http/handler/auth/authenticate.go

@@ -75,7 +75,12 @@ func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *h
 		if settings.AuthenticationMethod == portainer.AuthenticationInternal ||
 			settings.AuthenticationMethod == portainer.AuthenticationOAuth ||
 			(settings.AuthenticationMethod == portainer.AuthenticationLDAP && !settings.LDAPSettings.AutoCreateUsers) {
-			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+			// avoid username enumeration timing attack by creating a fake user
+			// https://en.wikipedia.org/wiki/Timing_attack
+			user = &portainer.User{
+				Username: "portainer-fake-username",
+				Password: "$2a$10$abcdefghijklmnopqrstuvwx..ABCDEFGHIJKLMNOPQRSTUVWXYZ12", // fake but valid format bcrypt hash
+			}
 		}
 	}
 
@@ -112,7 +117,11 @@ func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portai
 func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
 	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
 	if err != nil {
-		return httperror.Forbidden("Only initial admin is allowed to login without oauth", err)
+		if errors.Is(err, httperrors.ErrUnauthorized) {
+			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+		}
+
+		return httperror.InternalServerError("Unable to authenticate user against LDAP", err)
 	}
 
 	if user == nil {

api/http/handler/auth/logout.go

@@ -28,5 +28,7 @@ func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperro
 
 	security.RemoveAuthCookie(w)
 
+	handler.bouncer.RevokeJWT(tokenData.Token)
+
 	return response.Empty(w)
 }

api/http/handler/customtemplates/customtemplate_git_fetch.go

@@ -70,8 +70,7 @@ func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Re
 	if err != nil {
 		log.Warn().Err(err).Msg("failed to download git repository")
 
-		if err != nil {
-			rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
+		if rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath); rbErr != nil {
 			return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr)
 		}
 

api/http/handler/customtemplates/customtemplate_git_fetch_test.go

@@ -2,6 +2,7 @@ package customtemplates
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"io/fs"
 	"net/http"
@@ -19,6 +20,7 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
@@ -49,6 +51,19 @@ func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]
 	return os.ReadFile(filepath.Join(projectPath, configFilePath))
 }
 
+type InvalidTestGitService struct {
+	portainer.GitService
+	targetFilePath string
+}
+
+func (g *InvalidTestGitService) CloneRepository(dest, repoUrl, refName, username, password string, tlsSkipVerify bool) error {
+	return errors.New("simulate network error")
+}
+
+func (g *InvalidTestGitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
+	return "", nil
+}
+
 func createTestFile(targetPath string) error {
 	f, err := os.Create(targetPath)
 	if err != nil {
@@ -174,4 +189,28 @@ func Test_customTemplateGitFetch(t *testing.T) {
 
 		singleAPIRequest(h, jwt2, is, "gfedcba")
 	})
+
+	t.Run("restore git repository if it is failed to download the new git repository", func(t *testing.T) {
+		invalidGitService := &InvalidTestGitService{
+			targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath),
+		}
+		h := NewHandler(requestBouncer, store, fileService, invalidGitService)
+
+		req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
+		testhelpers.AddTestSecurityCookie(req, jwt1)
+
+		rr := httptest.NewRecorder()
+		h.ServeHTTP(rr, req)
+
+		is.Equal(http.StatusInternalServerError, rr.Code)
+
+		var errResp httperror.HandlerError
+		err = json.NewDecoder(rr.Body).Decode(&errResp)
+		assert.NoError(t, err, "failed to parse error body")
+
+		assert.FileExists(t, gitService.targetFilePath, "previous git repository is not restored")
+		fileContent, err := os.ReadFile(gitService.targetFilePath)
+		assert.NoError(t, err, "failed to read target file")
+		assert.Equal(t, "gfedcba", string(fileContent))
+	})
 }

api/http/handler/docker/containers/handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 )
@@ -30,7 +31,7 @@ func NewHandler(routePrefix string, bouncer security.BouncerService, dataStore d
 	}
 
 	router := h.PathPrefix(routePrefix).Subrouter()
-	router.Use(bouncer.AuthenticatedAccess)
+	router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
 
 	router.Handle("/{containerId}/gpus", httperror.LoggerHandler(h.containerGpusInspect)).Methods(http.MethodGet)
 	router.Handle("/{containerId}/recreate", httperror.LoggerHandler(h.recreate)).Methods(http.MethodPost)

api/http/handler/docker/handler.go

@@ -40,14 +40,14 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	}
 
 	// endpoints
-	endpointRouter := h.PathPrefix("/{id}").Subrouter()
+	endpointRouter := h.PathPrefix("/docker/{id}").Subrouter()
 	endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
 	endpointRouter.Use(dockerOnlyMiddleware)
 
-	containersHandler := containers.NewHandler("/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
+	containersHandler := containers.NewHandler("/docker/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
 	endpointRouter.PathPrefix("/containers").Handler(containersHandler)
 
-	imagesHandler := images.NewHandler("/{id}/images", bouncer, dockerClientFactory)
+	imagesHandler := images.NewHandler("/docker/{id}/images", bouncer, dockerClientFactory)
 	endpointRouter.PathPrefix("/images").Handler(imagesHandler)
 	return h
 }

api/http/handler/docker/images/handler.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
@@ -25,7 +26,7 @@ func NewHandler(routePrefix string, bouncer security.BouncerService, dockerClien
 	}
 
 	router := h.PathPrefix(routePrefix).Subrouter()
-	router.Use(bouncer.AuthenticatedAccess)
+	router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
 
 	router.Handle("", httperror.LoggerHandler(h.imagesList)).Methods(http.MethodGet)
 	return h

api/http/handler/docker/images/images_list.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 )
 
 type ImageResponse struct {
@@ -63,7 +64,9 @@ func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *http
 
 	imageUsageSet := set.Set[string]{}
 	if withUsage {
-		containers, err := cli.ContainerList(r.Context(), types.ContainerListOptions{})
+		containers, err := cli.ContainerList(r.Context(), container.ListOptions{
+			All: true,
+		})
 		if err != nil {
 			return httperror.InternalServerError("Unable to retrieve Docker containers", err)
 		}

api/http/handler/edgegroups/edgegroup_delete.go

@@ -19,8 +19,9 @@ import (
 // @security jwt
 // @param id path int true "EdgeGroup Id"
 // @success 204
+// @failure 409 "Edge group is in use by an Edge stack or Edge job"
 // @failure 503 "Edge compute features are disabled"
-// @failure 500
+// @failure 500 "Server error"
 // @router /edge_groups/{id} [delete]
 func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	edgeGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")

api/http/handler/edgestacks/edgestack_status_update.go

@@ -135,6 +135,11 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht
 }
 
 func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeStack, deploymentStatus portainer.EdgeStackDeploymentStatus) {
+	if deploymentStatus.Type == portainer.EdgeStackStatusRemoved {
+		delete(stack.Status, environmentId)
+		return
+	}
+
 	environmentStatus, ok := stack.Status[environmentId]
 	if !ok {
 		environmentStatus = portainer.EdgeStackStatus{

api/http/handler/endpointedge/endpointedge_status_inspect.go

@@ -15,10 +15,13 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+
+	"github.com/rs/zerolog/log"
 )
 
 type stackStatusResponse struct {
@@ -92,6 +95,8 @@ func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http
 		return httperror.Forbidden("Permission denied to access environment", errors.New("the device has not been trusted yet"))
 	}
 
+	firstConn := endpoint.LastCheckInDate == 0
+
 	err = handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint)
 	if err != nil {
 		return httperror.Forbidden("Permission denied to access environment", err)
@@ -106,7 +111,7 @@ func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http
 
 	var statusResponse *endpointEdgeStatusInspectResponse
 	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
-		statusResponse, err = handler.inspectStatus(tx, r, portainer.EndpointID(endpointID))
+		statusResponse, err = handler.inspectStatus(tx, r, portainer.EndpointID(endpointID), firstConn)
 		return err
 	})
 	if err != nil {
@@ -121,7 +126,7 @@ func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http
 	return cacheResponse(w, endpoint.ID, *statusResponse)
 }
 
-func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Request, endpointID portainer.EndpointID) (*endpointEdgeStatusInspectResponse, error) {
+func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Request, endpointID portainer.EndpointID, firstConn bool) (*endpointEdgeStatusInspectResponse, error) {
 	endpoint, err := tx.Endpoint().Endpoint(endpointID)
 	if err != nil {
 		return nil, err
@@ -133,8 +138,10 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
 	}
 
 	// Take an initial snapshot
-	if endpoint.LastCheckInDate == 0 {
-		handler.ReverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+	if firstConn {
+		if err := handler.ReverseTunnelService.Open(endpoint); err != nil {
+			log.Error().Err(err).Msg("could not open the tunnel")
+		}
 	}
 
 	agentPlatform, agentPlatformErr := parseAgentPlatform(r)
@@ -153,34 +160,21 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
 		return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err)
 	}
 
-	checkinInterval := endpoint.EdgeCheckinInterval
-	if endpoint.EdgeCheckinInterval == 0 {
-		settings, err := tx.Settings().Settings()
-		if err != nil {
-			return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
-		}
-		checkinInterval = settings.EdgeAgentCheckinInterval
-	}
-
-	tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)
+	tunnel := handler.ReverseTunnelService.Config(endpoint.ID)
 
 	statusResponse := endpointEdgeStatusInspectResponse{
 		Status:          tunnel.Status,
 		Port:            tunnel.Port,
-		CheckinInterval: checkinInterval,
+		CheckinInterval: edge.EffectiveCheckinInterval(tx, endpoint),
 		Credentials:     tunnel.Credentials,
 	}
 
-	schedules, handlerErr := handler.buildSchedules(endpoint.ID, tunnel)
+	schedules, handlerErr := handler.buildSchedules(endpoint.ID)
 	if handlerErr != nil {
 		return nil, handlerErr
 	}
 	statusResponse.Schedules = schedules
 
-	if tunnel.Status == portainer.EdgeAgentManagementRequired {
-		handler.ReverseTunnelService.SetTunnelStatusToActive(endpoint.ID)
-	}
-
 	edgeStacksStatus, handlerErr := handler.buildEdgeStacks(tx, endpoint.ID)
 	if handlerErr != nil {
 		return nil, handlerErr
@@ -213,9 +207,9 @@ func parseAgentPlatform(r *http.Request) (portainer.EndpointType, error) {
 	}
 }
 
-func (handler *Handler) buildSchedules(endpointID portainer.EndpointID, tunnel portainer.TunnelDetails) ([]edgeJobResponse, *httperror.HandlerError) {
+func (handler *Handler) buildSchedules(endpointID portainer.EndpointID) ([]edgeJobResponse, *httperror.HandlerError) {
 	schedules := []edgeJobResponse{}
-	for _, job := range tunnel.Jobs {
+	for _, job := range handler.ReverseTunnelService.EdgeJobs(endpointID) {
 		var collectLogs bool
 		if _, ok := job.GroupLogsCollection[endpointID]; ok {
 			collectLogs = job.GroupLogsCollection[endpointID].CollectLogs

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -8,6 +8,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/tag"
+	pendingActionActions "github.com/portainer/portainer/api/pendingactions/actions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -159,7 +160,9 @@ func (handler *Handler) updateEndpointGroup(tx dataservices.DataStoreTx, endpoin
 							err := handler.PendingActionsService.Create(portainer.PendingActions{
 								EndpointID: endpointID,
 								Action:     "CleanNAPWithOverridePolicies",
-								ActionData: endpointGroupID,
+								ActionData: &pendingActionActions.CleanNAPWithOverridePoliciesPayload{
+									EndpointGroupID: endpointGroupID,
+								},
 							})
 							if err != nil {
 								log.Error().Err(err).Msgf("Unable to create pending action to clean NAP with override policies for endpoint (%d) and endpoint group (%d).", endpointID, endpointGroupID)

api/http/handler/endpointproxy/proxy_docker.go

@@ -34,7 +34,7 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 			return httperror.InternalServerError("No Edge agent registered with the environment", errors.New("No agent available"))
 		}
 
-		_, err := handler.ReverseTunnelService.GetActiveTunnel(endpoint)
+		_, err := handler.ReverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return httperror.InternalServerError("Unable to get the active tunnel", err)
 		}

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -34,7 +34,7 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 			return httperror.InternalServerError("No Edge agent registered with the environment", errors.New("No agent available"))
 		}
 
-		_, err := handler.ReverseTunnelService.GetActiveTunnel(endpoint)
+		_, err := handler.ReverseTunnelService.TunnelAddr(endpoint)
 		if err != nil {
 			return httperror.InternalServerError("Unable to get the active tunnel", err)
 		}

api/http/handler/endpoints/endpoint_association_delete.go

@@ -59,8 +59,6 @@ func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http
 		return httperror.InternalServerError("Failed persisting environment in database", err)
 	}
 
-	handler.ReverseTunnelService.SetTunnelStatusToIdle(endpoint.ID)
-
 	return response.Empty(w)
 }
 

api/http/handler/endpoints/endpoint_create.go

@@ -201,6 +201,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @param Gpus formData string false "List of GPUs - json stringified array of {name, value} structs"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
+// @failure 409 "Name is not unique"
 // @failure 500 "Server error"
 // @router /endpoints [post]
 func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/endpoints/endpoint_delete.go

@@ -2,6 +2,7 @@ package endpoints
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"slices"
 	"strconv"
@@ -17,19 +18,40 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+type endpointDeleteRequest struct {
+	ID            int  `json:"id"`
+	DeleteCluster bool `json:"deleteCluster"`
+}
+
+type endpointDeleteBatchPayload struct {
+	Endpoints []endpointDeleteRequest `json:"endpoints"`
+}
+
+type endpointDeleteBatchPartialResponse struct {
+	Deleted []int `json:"deleted"`
+	Errors  []int `json:"errors"`
+}
+
+func (payload *endpointDeleteBatchPayload) Validate(r *http.Request) error {
+	if payload == nil || len(payload.Endpoints) == 0 {
+		return fmt.Errorf("invalid request payload. You must provide a list of environments to delete")
+	}
+
+	return nil
+}
+
 // @id EndpointDelete
-// @summary Remove an environment(endpoint)
-// @description Remove an environment(endpoint).
-// @description **Access policy**: administrator
+// @summary Remove an environment
+// @description Remove the environment associated to the specified identifier and optionally clean-up associated resources.
+// @description **Access policy**: Administrator only.
 // @tags endpoints
-// @security ApiKeyAuth
-// @security jwt
+// @security ApiKeyAuth || jwt
 // @param id path int true "Environment(Endpoint) identifier"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "Environment(Endpoint) not found"
-// @failure 500 "Server error"
+// @success 204 "Environment successfully deleted."
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 403 "Unauthorized access or operation not allowed."
+// @failure 404 "Unable to find the environment with the specified identifier inside the database."
+// @failure 500 "Server error occurred while attempting to delete the environment."
 // @router /endpoints/{id} [delete]
 func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
@@ -62,6 +84,63 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 	return response.Empty(w)
 }
 
+// @id EndpointDeleteBatch
+// @summary Remove multiple environments
+// @description Remove multiple environments and optionally clean-up associated resources.
+// @description **Access policy**: Administrator only.
+// @tags endpoints
+// @security ApiKeyAuth || jwt
+// @accept json
+// @produce json
+// @param body body endpointDeleteBatchPayload true "List of environments to delete, with optional deleteCluster flag to clean-up assocaited resources (cloud environments only)"
+// @success 204 "Environment(s) successfully deleted."
+// @failure 207 {object} endpointDeleteBatchPartialResponse "Partial success. Some environments were deleted successfully, while others failed."
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 403 "Unauthorized access or operation not allowed."
+// @failure 500 "Server error occurred while attempting to delete the specified environments."
+// @router /endpoints [delete]
+func (handler *Handler) endpointDeleteBatch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var p endpointDeleteBatchPayload
+	if err := request.DecodeAndValidateJSONPayload(r, &p); err != nil {
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	resp := endpointDeleteBatchPartialResponse{
+		Deleted: []int{},
+		Errors:  []int{},
+	}
+
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		for _, e := range p.Endpoints {
+			if handler.demoService.IsDemoEnvironment(portainer.EndpointID(e.ID)) {
+				resp.Errors = append(resp.Errors, e.ID)
+				log.Warn().Err(httperrors.ErrNotAvailableInDemo).Msgf("Unable to remove demo environment %d", e.ID)
+
+				continue
+			}
+
+			if err := handler.deleteEndpoint(tx, portainer.EndpointID(e.ID), e.DeleteCluster); err != nil {
+				resp.Errors = append(resp.Errors, e.ID)
+				log.Warn().Err(err).Int("environment_id", e.ID).Msg("Unable to remove environment")
+
+				continue
+			}
+
+			resp.Deleted = append(resp.Deleted, e.ID)
+		}
+
+		return nil
+	}); err != nil {
+		return httperror.InternalServerError("Unable to delete environments", err)
+	}
+
+	if len(resp.Errors) > 0 {
+		return response.JSONWithStatus(w, resp, http.StatusPartialContent)
+	}
+
+	return response.Empty(w)
+}
+
 func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID portainer.EndpointID, deleteCluster bool) error {
 	endpoint, err := tx.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if tx.IsErrObjectNotFound(err) {
@@ -78,23 +157,20 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 		}
 	}
 
-	err = tx.Snapshot().Delete(endpointID)
-	if err != nil {
-		log.Warn().Err(err).Msgf("Unable to remove the snapshot from the database")
+	if err := tx.Snapshot().Delete(endpointID); err != nil {
+		log.Warn().Err(err).Msg("Unable to remove the snapshot from the database")
 	}
 
 	handler.ProxyManager.DeleteEndpointProxy(endpoint.ID)
 
 	if len(endpoint.UserAccessPolicies) > 0 || len(endpoint.TeamAccessPolicies) > 0 {
-		err = handler.AuthorizationService.UpdateUsersAuthorizationsTx(tx)
-		if err != nil {
-			log.Warn().Err(err).Msgf("Unable to update user authorizations")
+		if err := handler.AuthorizationService.UpdateUsersAuthorizationsTx(tx); err != nil {
+			log.Warn().Err(err).Msg("Unable to update user authorizations")
 		}
 	}
 
-	err = tx.EndpointRelation().DeleteEndpointRelation(endpoint.ID)
-	if err != nil {
-		log.Warn().Err(err).Msgf("Unable to remove environment relation from the database")
+	if err := tx.EndpointRelation().DeleteEndpointRelation(endpoint.ID); err != nil {
+		log.Warn().Err(err).Msg("Unable to remove environment relation from the database")
 	}
 
 	for _, tagID := range endpoint.TagIDs {
@@ -106,9 +182,9 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 		}
 
 		if handler.DataStore.IsErrObjectNotFound(err) {
-			log.Warn().Err(err).Msgf("Unable to find tag inside the database")
+			log.Warn().Err(err).Msg("Unable to find tag inside the database")
 		} else if err != nil {
-			log.Warn().Err(err).Msgf("Unable to delete tag relation from the database")
+			log.Warn().Err(err).Msg("Unable to delete tag relation from the database")
 		}
 	}
 
@@ -122,40 +198,39 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 			return e == endpoint.ID
 		})
 
-		err = tx.EdgeGroup().Update(edgeGroup.ID, &edgeGroup)
-		if err != nil {
-			log.Warn().Err(err).Msgf("Unable to update edge group")
+		if err := tx.EdgeGroup().Update(edgeGroup.ID, &edgeGroup); err != nil {
+			log.Warn().Err(err).Msg("Unable to update edge group")
 		}
 	}
 
 	edgeStacks, err := tx.EdgeStack().EdgeStacks()
 	if err != nil {
-		log.Warn().Err(err).Msgf("Unable to retrieve edge stacks from the database")
+		log.Warn().Err(err).Msg("Unable to retrieve edge stacks from the database")
 	}
 
 	for idx := range edgeStacks {
 		edgeStack := &edgeStacks[idx]
 		if _, ok := edgeStack.Status[endpoint.ID]; ok {
 			delete(edgeStack.Status, endpoint.ID)
-			err = tx.EdgeStack().UpdateEdgeStack(edgeStack.ID, edgeStack)
-			if err != nil {
-				log.Warn().Err(err).Msgf("Unable to update edge stack")
+
+			if err := tx.EdgeStack().UpdateEdgeStack(edgeStack.ID, edgeStack); err != nil {
+				log.Warn().Err(err).Msg("Unable to update edge stack")
 			}
 		}
 	}
 
 	registries, err := tx.Registry().ReadAll()
 	if err != nil {
-		log.Warn().Err(err).Msgf("Unable to retrieve registries from the database")
+		log.Warn().Err(err).Msg("Unable to retrieve registries from the database")
 	}
 
 	for idx := range registries {
 		registry := &registries[idx]
 		if _, ok := registry.RegistryAccesses[endpoint.ID]; ok {
 			delete(registry.RegistryAccesses, endpoint.ID)
-			err = tx.Registry().Update(registry.ID, registry)
-			if err != nil {
-				log.Warn().Err(err).Msgf("Unable to update registry accesses")
+
+			if err := tx.Registry().Update(registry.ID, registry); err != nil {
+				log.Warn().Err(err).Msg("Unable to update registry accesses")
 			}
 		}
 	}
@@ -163,7 +238,7 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 	if endpointutils.IsEdgeEndpoint(endpoint) {
 		edgeJobs, err := handler.DataStore.EdgeJob().ReadAll()
 		if err != nil {
-			log.Warn().Err(err).Msgf("Unable to retrieve edge jobs from the database")
+			log.Warn().Err(err).Msg("Unable to retrieve edge jobs from the database")
 		}
 
 		for idx := range edgeJobs {
@@ -171,14 +246,18 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 			if _, ok := edgeJob.Endpoints[endpoint.ID]; ok {
 				delete(edgeJob.Endpoints, endpoint.ID)
 
-				err = tx.EdgeJob().Update(edgeJob.ID, edgeJob)
-				if err != nil {
-					log.Warn().Err(err).Msgf("Unable to update edge job")
+				if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
+					log.Warn().Err(err).Msg("Unable to update edge job")
 				}
 			}
 		}
 	}
 
+	// delete the pending actions
+	if err := tx.PendingActions().DeleteByEndpointID(endpoint.ID); err != nil {
+		log.Warn().Err(err).Int("endpointId", int(endpoint.ID)).Msg("Unable to delete pending actions")
+	}
+
 	err = tx.Endpoint().DeleteEndpoint(portainer.EndpointID(endpointID))
 	if err != nil {
 		return httperror.InternalServerError("Unable to delete the environment from the database", err)

api/http/handler/endpoints/endpoint_delete_test.go

@@ -21,7 +21,8 @@ func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
 
 	handler := NewHandler(testhelpers.NewTestRequestBouncer(), demo.NewService())
 	handler.DataStore = store
-	handler.ProxyManager = proxy.NewManager(nil, nil, nil, nil, nil, nil, nil)
+	handler.ProxyManager = proxy.NewManager(nil)
+	handler.ProxyManager.NewProxyFactory(nil, nil, nil, nil, nil, nil, nil, nil)
 
 	// Create all the environments and add them to the same edge group
 

api/http/handler/endpoints/endpoint_force_update_service.go

@@ -12,8 +12,8 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
-	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 )
 
@@ -39,7 +39,7 @@ func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
 // @produce json
 // @param id path int true "endpoint identifier"
 // @param body body forceUpdateServicePayload true "details"
-// @success 200 {object} dockertypes.ServiceUpdateResponse "Success"
+// @success 200 {object} swarm.ServiceUpdateResponse "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 404 "endpoint not found"
@@ -94,7 +94,7 @@ func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *htt
 	go func() {
 		images.EvictImageStatus(payload.ServiceID)
 		images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
-		containers, _ := dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{
+		containers, _ := dockerClient.ContainerList(context.TODO(), container.ListOptions{
 			All:     true,
 			Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+payload.ServiceID)),
 		})

api/http/handler/endpoints/endpoint_registries_list.go

@@ -3,15 +3,16 @@ package endpoints
 import (
 	"net/http"
 
+	"github.com/pkg/errors"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/pkg/errors"
 )
 
 // @id endpointRegistriesList
@@ -127,7 +128,7 @@ func (handler *Handler) isNamespaceAuthorized(endpoint *portainer.Endpoint, name
 		return true, nil
 	}
 
-	if namespace == "default" {
+	if !endpoint.Kubernetes.Configuration.RestrictDefaultNamespace && namespace == kubernetes.DefaultNamespace {
 		return true, nil
 	}
 

api/http/handler/endpoints/endpoint_update.go

@@ -69,6 +69,7 @@ func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
+// @failure 409 "Name is not unique"
 // @failure 500 "Server error"
 // @router /endpoints/{id} [put]
 func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/endpoints/filter.go

@@ -334,11 +334,16 @@ func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portai
 		status := endpoint.Status
 		if endpointutils.IsEdgeEndpoint(&endpoint) {
 			isCheckValid := false
+
 			edgeCheckinInterval := endpoint.EdgeCheckinInterval
-			if endpoint.EdgeCheckinInterval == 0 {
+			if edgeCheckinInterval == 0 {
 				edgeCheckinInterval = settings.EdgeAgentCheckinInterval
 			}
 
+			if endpoint.Edge.AsyncMode {
+				edgeCheckinInterval = getShortestAsyncInterval(&endpoint, settings)
+			}
+
 			if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 {
 				isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd)
 			}
@@ -622,9 +627,36 @@ func getEdgeStackStatusParam(r *http.Request) (*portainer.EdgeStackStatusType, e
 		portainer.EdgeStackStatusRunning,
 		portainer.EdgeStackStatusDeploying,
 		portainer.EdgeStackStatusRemoving,
+		portainer.EdgeStackStatusCompleted,
 	}, edgeStackStatus) {
 		return nil, errors.New("invalid edgeStackStatus parameter")
 	}
 
 	return &edgeStackStatus, nil
 }
+
+func getShortestAsyncInterval(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
+	var edgeIntervalUseDefault int = -1
+	pingInterval := endpoint.Edge.PingInterval
+	if pingInterval == edgeIntervalUseDefault {
+		pingInterval = settings.Edge.PingInterval
+	}
+	shortestAsyncInterval := pingInterval
+
+	snapshotInterval := endpoint.Edge.SnapshotInterval
+	if snapshotInterval == edgeIntervalUseDefault {
+		snapshotInterval = settings.Edge.SnapshotInterval
+	}
+	if shortestAsyncInterval > snapshotInterval {
+		shortestAsyncInterval = snapshotInterval
+	}
+
+	commandInterval := endpoint.Edge.CommandInterval
+	if commandInterval == edgeIntervalUseDefault {
+		commandInterval = settings.Edge.CommandInterval
+	}
+	if shortestAsyncInterval > commandInterval {
+		shortestAsyncInterval = commandInterval
+	}
+	return shortestAsyncInterval
+}

api/http/handler/endpoints/handler.go

@@ -71,6 +71,8 @@ func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Han
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoints/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoints",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDeleteBatch))).Methods(http.MethodDelete)
 	h.Handle("/endpoints/{id}/dockerhub/{registryId}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
 	h.Handle("/endpoints/{id}/snapshot",

api/http/handler/handler.go

@@ -85,7 +85,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.20.0
+// @version 2.21.1
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -199,7 +199,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case strings.HasPrefix(r.URL.Path, "/api/kubernetes"):
 		http.StripPrefix("/api", h.KubernetesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/docker"):
-		http.StripPrefix("/api/docker", h.DockerHandler).ServeHTTP(w, r)
+		http.StripPrefix("/api", h.DockerHandler).ServeHTTP(w, r)
 
 	// Helm subpath under kubernetes -> /api/endpoints/{id}/kubernetes/helm
 	case strings.HasPrefix(r.URL.Path, "/api/endpoints/") && strings.Contains(r.URL.Path, "/kubernetes/helm"):

api/http/handler/helm/handler.go

@@ -38,19 +38,20 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 		kubeClusterAccessService: kubeClusterAccessService,
 	}
 
-	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
+	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"),
+		bouncer.AuthenticatedAccess)
 
 	// `helm list -o json`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmList))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmList)).Methods(http.MethodGet)
 
 	// `helm delete RELEASE_NAME`
 	h.Handle("/{id}/kubernetes/helm/{release}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmDelete))).Methods(http.MethodDelete)
+		httperror.LoggerHandler(h.helmDelete)).Methods(http.MethodDelete)
 
 	// `helm install [NAME] [CHART] flags`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmInstall))).Methods(http.MethodPost)
+		httperror.LoggerHandler(h.helmInstall)).Methods(http.MethodPost)
 
 	// Deprecated
 	h.Handle("/{id}/kubernetes/helm/repositories",
@@ -69,12 +70,14 @@ func NewTemplateHandler(bouncer security.BouncerService, helmPackageManager libh
 		requestBouncer:     bouncer,
 	}
 
+	h.Use(bouncer.AuthenticatedAccess)
+
 	h.Handle("/templates/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmRepoSearch))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmRepoSearch)).Methods(http.MethodGet)
 
 	// helm show [COMMAND] [CHART] [REPO] flags
 	h.Handle("/templates/helm/{command:chart|values|readme}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmShow))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmShow)).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/helm/helm_install.go

@@ -61,8 +61,7 @@ func (handler *Handler) helmInstall(w http.ResponseWriter, r *http.Request) *htt
 		return httperror.InternalServerError("Unable to install a chart", err)
 	}
 
-	w.WriteHeader(http.StatusCreated)
-	return response.JSON(w, release)
+	return response.JSONWithStatus(w, release, http.StatusCreated)
 }
 
 func (p *installChartPayload) Validate(_ *http.Request) error {

api/http/handler/hostmanagement/openamt/amtrpc.go

@@ -155,7 +155,7 @@ func pullImage(ctx context.Context, docker *client.Client, imageName string) err
 // runContainer should be used to run a short command that returns information to stdout
 // TODO: add k8s support
 func runContainer(ctx context.Context, docker *client.Client, imageName, containerName string, cmdLine []string) (output string, err error) {
-	opts := types.ContainerListOptions{All: true}
+	opts := container.ListOptions{All: true}
 	opts.Filters = filters.NewArgs()
 	opts.Filters.Add("name", containerName)
 	existingContainers, err := docker.ContainerList(ctx, opts)
@@ -170,7 +170,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 	}
 
 	if len(existingContainers) > 0 {
-		err = docker.ContainerRemove(ctx, existingContainers[0].ID, types.ContainerRemoveOptions{Force: true})
+		err = docker.ContainerRemove(ctx, existingContainers[0].ID, container.RemoveOptions{Force: true})
 		if err != nil {
 			log.Error().
 				Str("image_name", imageName).
@@ -211,7 +211,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 		return "", err
 	}
 
-	err = docker.ContainerStart(ctx, created.ID, types.ContainerStartOptions{})
+	err = docker.ContainerStart(ctx, created.ID, container.StartOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).
@@ -243,14 +243,14 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 
 	log.Debug().Int64("status", statusCode).Msg("container wait status")
 
-	out, err := docker.ContainerLogs(ctx, created.ID, types.ContainerLogsOptions{ShowStdout: true})
+	out, err := docker.ContainerLogs(ctx, created.ID, container.LogsOptions{ShowStdout: true})
 	if err != nil {
 		log.Error().Err(err).Str("image_name", imageName).Str("container_name", containerName).Msg("getting container log")
 
 		return "", err
 	}
 
-	err = docker.ContainerRemove(ctx, created.ID, types.ContainerRemoveOptions{})
+	err = docker.ContainerRemove(ctx, created.ID, container.RemoveOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).

api/http/handler/registries/handler.go

@@ -7,12 +7,16 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/pendingactions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 
 	"github.com/gorilla/mux"
+
+	"github.com/pkg/errors"
 )
 
 func hideFields(registry *portainer.Registry, hideAccesses bool) {
@@ -83,29 +87,88 @@ func (handler *Handler) registriesHaveSameURLAndCredentials(r1, r2 *portainer.Re
 	return hasSameUrl && hasSameCredentials && r1.Gitlab.ProjectPath == r2.Gitlab.ProjectPath
 }
 
-func (handler *Handler) userHasRegistryAccess(r *http.Request) (hasAccess bool, isAdmin bool, err error) {
+// this function validates that
+//
+// 1. user has the appropriate authorizations to perform the request
+//
+// 2. user has a direct or indirect access to the registry
+func (handler *Handler) userHasRegistryAccess(r *http.Request, registry *portainer.Registry) (hasAccess bool, isAdmin bool, err error) {
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
 		return false, false, err
 	}
 
+	user, err := handler.DataStore.User().Read(securityContext.UserID)
+	if err != nil {
+		return false, false, err
+	}
+
+	// Portainer admins always have access to everything
 	if securityContext.IsAdmin {
 		return true, true, nil
 	}
 
-	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
-	if err != nil {
-		return false, false, err
-	}
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	// mandatory query param that should become a path param
+	endpointIdStr, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
 	if err != nil {
 		return false, false, err
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	endpointId := portainer.EndpointID(endpointIdStr)
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointId)
 	if err != nil {
 		return false, false, err
 	}
 
-	return true, false, nil
+	// validate that the request is allowed for the user (READ/WRITE authorization on request path)
+	if err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint); errors.Is(err, security.ErrAuthorizationRequired) {
+		return false, false, nil
+	} else if err != nil {
+		return false, false, err
+	}
+
+	memberships, err := handler.DataStore.TeamMembership().TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return false, false, nil
+	}
+
+	// validate access for kubernetes namespaces (leverage registry.RegistryAccesses[endpointId].Namespaces)
+	if endpointutils.IsKubernetesEndpoint(endpoint) {
+		kcl, err := handler.K8sClientFactory.GetKubeClient(endpoint)
+		if err != nil {
+			return false, false, errors.Wrap(err, "unable to retrieve kubernetes client to validate registry access")
+		}
+		accessPolicies, err := kcl.GetNamespaceAccessPolicies()
+		if err != nil {
+			return false, false, errors.Wrap(err, "unable to retrieve environment's namespaces policies to validate registry access")
+		}
+
+		authorizedNamespaces := registry.RegistryAccesses[endpointId].Namespaces
+
+		for _, namespace := range authorizedNamespaces {
+			// when the default namespace is authorized to use a registry, all users have the ability to use it
+			// unless the default namespace is restricted: in this case continue to search for other potential accesses authorizations
+			if namespace == kubernetes.DefaultNamespace && !endpoint.Kubernetes.Configuration.RestrictDefaultNamespace {
+				return true, false, nil
+			}
+
+			namespacePolicy := accessPolicies[namespace]
+			if security.AuthorizedAccess(user.ID, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies) {
+				return true, false, nil
+			}
+		}
+		return false, false, nil
+	}
+
+	// validate access for docker environments
+	// leverage registry.RegistryAccesses[endpointId].UserAccessPolicies (direct access)
+	// and registry.RegistryAccesses[endpointId].TeamAccessPolicies (indirect access via his teams)
+	if security.AuthorizedRegistryAccess(registry, user, memberships, endpoint.ID) {
+		return true, false, nil
+	}
+
+	// when user has no access via their role, direct grant or indirect grant
+	// then they don't have access to the registry
+	return false, false, nil
 }

api/http/handler/registries/registry_create.go

@@ -89,6 +89,7 @@ func (payload *registryCreatePayload) Validate(_ *http.Request) error {
 // @param body body registryCreatePayload true "Registry details"
 // @success 200 {object} portainer.Registry "Success"
 // @failure 400 "Invalid request"
+// @failure 409 "Another registry with the same name or same URL & credentials already exists"
 // @failure 500 "Server error"
 // @router /registries [post]
 func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/registries/registry_delete.go

@@ -8,6 +8,7 @@ import (
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/pendingactions"
+	"github.com/portainer/portainer/api/pendingactions/actions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -91,7 +92,7 @@ func (handler *Handler) deleteKubernetesSecrets(registry *portainer.Registry) er
 			if len(failedNamespaces) > 0 {
 				handler.PendingActionsService.Create(portainer.PendingActions{
 					EndpointID: endpointId,
-					Action:     pendingactions.DeletePortainerK8sRegistrySecrets,
+					Action:     actions.DeletePortainerK8sRegistrySecrets,
 
 					// When extracting the data, this is the type we need to pull out
 					// i.e. pendingactions.DeletePortainerK8sRegistrySecretsData

api/http/handler/registries/registry_inspect.go

@@ -26,14 +26,6 @@ import (
 // @failure 500 "Server error"
 // @router /registries/{id} [get]
 func (handler *Handler) registryInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	hasAccess, isAdmin, err := handler.userHasRegistryAccess(r)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve info from request context", err)
-	}
-	if !hasAccess {
-		return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
-	}
-
 	registryID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return httperror.BadRequest("Invalid registry identifier route variable", err)
@@ -46,6 +38,14 @@ func (handler *Handler) registryInspect(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to find a registry with the specified identifier inside the database", err)
 	}
 
+	hasAccess, isAdmin, err := handler.userHasRegistryAccess(r, registry)
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve info from request context", err)
+	}
+	if !hasAccess {
+		return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
+	}
+
 	hideFields(registry, !isAdmin)
 	return response.JSON(w, registry)
 }

api/http/handler/registries/registry_update.go

@@ -52,7 +52,7 @@ func (payload *registryUpdatePayload) Validate(r *http.Request) error {
 // @success 200 {object} portainer.Registry "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Registry not found"
-// @failure 409 "Another registry with the same URL already exists"
+// @failure 409 "Another registry with the same name or same URL & credentials already exists"
 // @failure 500 "Server error"
 // @router /registries/{id} [put]
 func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/resourcecontrols/resourcecontrol_create.go

@@ -63,7 +63,7 @@ func (payload *resourceControlCreatePayload) Validate(r *http.Request) error {
 // @param body body resourceControlCreatePayload true "Resource control details"
 // @success 200 {object} portainer.ResourceControl "Success"
 // @failure 400 "Invalid request"
-// @failure 409 "Resource control already exists"
+// @failure 409 "A resource control is already associated to this resource"
 // @failure 500 "Server error"
 // @router /resource_controls [post]
 func (handler *Handler) resourceControlCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/settings/settings_update.go

@@ -13,6 +13,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"golang.org/x/oauth2"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
@@ -95,6 +96,11 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 		}
 	}
 
+	if payload.OAuthSettings != nil {
+		if payload.OAuthSettings.AuthStyle < oauth2.AuthStyleAutoDetect || payload.OAuthSettings.AuthStyle > oauth2.AuthStyleInHeader {
+			return errors.New("Invalid OAuth AuthStyle")
+		}
+	}
 	return nil
 }
 
@@ -225,6 +231,7 @@ func (handler *Handler) updateSettings(tx dataservices.DataStoreTx, payload sett
 		settings.OAuthSettings = *payload.OAuthSettings
 		settings.OAuthSettings.ClientSecret = clientSecret
 		settings.OAuthSettings.KubeSecretKey = kubeSecret
+		settings.OAuthSettings.AuthStyle = payload.OAuthSettings.AuthStyle
 	}
 
 	if payload.EnableEdgeComputeFeatures != nil {

api/http/handler/stacks/create_compose_stack.go

@@ -229,6 +229,7 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 // @param body body composeStackFromGitRepositoryPayload true "stack config"
 // @success 200 {object} portainer.Stack
 // @failure 400 "Invalid request"
+// @failure 409 "Stack name or webhook ID already exists"
 // @failure 500 "Server error"
 // @router /stacks/create/standalone/repository [post]
 func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {

api/http/handler/stacks/create_kubernetes_stack.go

@@ -195,6 +195,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 // @param endpointId query int true "Identifier of the environment that will be used to deploy the stack"
 // @success 200 {object} portainer.Stack
 // @failure 400 "Invalid request"
+// @failure 409 "Stack name or webhook ID already exists"
 // @failure 500 "Server error"
 // @router /stacks/create/kubernetes/repository [post]
 func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {

api/http/handler/stacks/create_swarm_stack.go

@@ -188,6 +188,7 @@ func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference
 // @param body body swarmStackFromGitRepositoryPayload true "stack config"
 // @success 200 {object} portainer.Stack
 // @failure 400 "Invalid request"
+// @failure 409 "Stack name or webhook ID already exists"
 // @failure 500 "Server error"
 // @router /stacks/create/swarm/repository [post]
 func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {

api/http/handler/stacks/handler.go

@@ -21,6 +21,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 )
@@ -190,7 +191,7 @@ func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoin
 		}
 	}
 
-	containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{All: true})
 	if err != nil {
 		return false, err
 	}

api/http/handler/stacks/stack_list.go

@@ -23,6 +23,7 @@ type stackListOperationFilters struct {
 // @description List all stacks based on the current user authorizations.
 // @description Will return all stacks if using an administrator account otherwise it
 // @description will only return the list of stacks the user have access to.
+// @description Limited stacks will not be returned by this endpoint.
 // @description **Access policy**: authenticated
 // @tags stacks
 // @security ApiKeyAuth
@@ -91,25 +92,55 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 	return response.JSON(w, stacks)
 }
 
+// filterStacks refines a collection of Stack instances using specified criteria.
+// This function examines the provided filters: EndpointID, SwarmID, and IncludeOrphanedStacks.
+// - If both EndpointID is zero and SwarmID is an empty string, the function directly returns the original stack list without any modifications.
+// - If either filter is specified, it proceeds to selectively include stacks that match the criteria.
+
+// Key Points on Business Logic:
+// 1. Determining Inclusion of Orphaned Stacks:
+//    - The decision to include orphaned stacks is influenced by the user's role and usually set by the client (UI).
+//    - Administrators or environment administrators can include orphaned stacks by setting IncludeOrphanedStacks to true, reflecting their broader access rights.
+//    - For non-administrative users, this is typically set to false, limiting their visibility to only stacks within their purview.
+
+// 2. Inclusion Criteria for Orphaned Stacks:
+//    - When IncludeOrphanedStacks is true and an EndpointID is specified (not zero), the function selects:
+//      a) Stacks linked to the specified EndpointID.
+//      b) Orphaned stacks that don't have a naming conflict with any stack associated with the EndpointID.
+//    - This approach is designed to avoid name conflicts within Docker Compose, which restricts the creation of multiple stacks with the same name.
+
+// 3. Type Matching for Orphaned Stacks:
+//    - The function ensures that orphaned stacks are compatible with the environment's stack type (compose or swarm).
+//    - It filters out orphaned swarm stacks in Docker standalone environments
+//    - It filters out orphaned standalone stack in Docker swarm environments
+//    - This ensures that re-association respects the constraints of the environment and stack type.
+
+// The outcome is a new list of stacks that align with these filtering and business logic criteria.
 func filterStacks(stacks []portainer.Stack, filters *stackListOperationFilters, endpoints []portainer.Endpoint) []portainer.Stack {
 	if filters.EndpointID == 0 && filters.SwarmID == "" {
 		return stacks
 	}
 
 	filteredStacks := make([]portainer.Stack, 0, len(stacks))
+	uniqueStackNames := make(map[string]struct{})
 	for _, stack := range stacks {
-		if filters.IncludeOrphanedStacks && isOrphanedStack(stack, endpoints) {
-			if (stack.Type == portainer.DockerComposeStack && filters.SwarmID == "") || (stack.Type == portainer.DockerSwarmStack && filters.SwarmID != "") {
-				filteredStacks = append(filteredStacks, stack)
-			}
-			continue
-		}
-
 		if stack.Type == portainer.DockerComposeStack && stack.EndpointID == portainer.EndpointID(filters.EndpointID) {
 			filteredStacks = append(filteredStacks, stack)
+			uniqueStackNames[stack.Name] = struct{}{}
 		}
 		if stack.Type == portainer.DockerSwarmStack && stack.SwarmID == filters.SwarmID {
 			filteredStacks = append(filteredStacks, stack)
+			uniqueStackNames[stack.Name] = struct{}{}
+		}
+	}
+
+	for _, stack := range stacks {
+		if filters.IncludeOrphanedStacks && isOrphanedStack(stack, endpoints) {
+			if (stack.Type == portainer.DockerComposeStack && filters.SwarmID == "") || (stack.Type == portainer.DockerSwarmStack && filters.SwarmID != "") {
+				if _, exists := uniqueStackNames[stack.Name]; !exists {
+					filteredStacks = append(filteredStacks, stack)
+				}
+			}
 		}
 	}
 

api/http/handler/stacks/stack_list_test.go

@@ -0,0 +1,74 @@
+package stacks
+
+import (
+	"sort"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterStacks(t *testing.T) {
+	t.Run("filter stacks against particular endpoint and all orphaned stacks", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: true}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}, {ID: 2}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+
+	t.Run("filter unique stacks against particular endpoint and all orphaned stacks and an orphaned stack has the same name with normal stack", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+			{ID: 4, EndpointID: 4, Name: "normal_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: true}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}, {ID: 2}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+
+	t.Run("only filter stacks against particular endpoint and no orphaned stacks", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+			{ID: 4, EndpointID: 4, Name: "normal_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: false}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+}
+
+func isEqualStacks(t *testing.T, expectStacks, actualStacks []portainer.Stack) {
+	expectStackIDs := make([]int, len(expectStacks))
+	for i, stack := range expectStacks {
+		expectStackIDs[i] = int(stack.ID)
+	}
+	sort.Ints(expectStackIDs)
+
+	actualStackIDs := make([]int, len(actualStacks))
+	for i, stack := range actualStacks {
+		actualStackIDs[i] = int(stack.ID)
+	}
+	sort.Ints(actualStackIDs)
+
+	assert.Equal(t, expectStackIDs, actualStackIDs)
+}

api/http/handler/stacks/stack_migrate.go

@@ -46,6 +46,7 @@ func (payload *stackMigratePayload) Validate(r *http.Request) error {
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 404 "Stack not found"
+// @failure 409 "A stack with the same name is already running on the target environment(endpoint)"
 // @failure 500 "Server error"
 // @router /stacks/{id}/migrate [post]
 func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/stacks/stack_start.go

@@ -29,6 +29,7 @@ import (
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 404 "Not found"
+// @failure 409 "Stack name is not unique"
 // @failure 500 "Server error"
 // @router /stacks/{id}/start [post]
 func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -27,6 +27,8 @@ type stackGitRedployPayload struct {
 	Prune                    bool
 	// Force a pulling to current image with the original tag though the image is already the latest
 	PullImage bool `example:"false"`
+
+	StackName string
 }
 
 func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
@@ -44,7 +46,7 @@ func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
 // @produce json
 // @param id path int true "Stack identifier"
 // @param endpointId query int false "Stacks created before version 1.18.0 might not have an associated environment(endpoint) identifier. Use this optional parameter to set the environment(endpoint) identifier used by the stack."
-// @param body body stackGitRedployPayload true "Git configs for pull and redeploy a stack"
+// @param body body stackGitRedployPayload true "Git configs for pull and redeploy of a stack. **StackName** may only be populated for Kuberenetes stacks, and if specified with a blank string, it will be set to blank"
 // @success 200 {object} portainer.Stack "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
@@ -136,6 +138,10 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
+	if stack.Type == portainer.KubernetesStack {
+		stack.Name = payload.StackName
+	}
+
 	repositoryUsername := ""
 	repositoryPassword := ""
 	if payload.RepositoryAuthentication {

api/http/handler/stacks/webhook_invoke.go

@@ -19,7 +19,7 @@ import (
 // @param webhookID path string true "Stack identifier"
 // @success 200 "Success"
 // @failure 400 "Invalid request"
-// @failure 409 "Conflict"
+// @failure 409 "Autoupdate for the stack isn't available"
 // @failure 500 "Server error"
 // @router /stacks/webhooks/{webhookID} [post]
 func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/tags/tag_create.go

@@ -35,7 +35,7 @@ func (payload *tagCreatePayload) Validate(r *http.Request) error {
 // @produce json
 // @param body body tagCreatePayload true "Tag details"
 // @success 200 {object} portainer.Tag "Success"
-// @failure 409 "Tag name exists"
+// @failure 409 "This name is already associated to a tag"
 // @failure 500 "Server error"
 // @router /tags [post]
 func (handler *Handler) tagCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/teams/team_create.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -23,6 +24,7 @@ func (payload *teamCreatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid team name")
 	}
+
 	return nil
 }
 
@@ -38,31 +40,47 @@ func (payload *teamCreatePayload) Validate(r *http.Request) error {
 // @param body body teamCreatePayload true "details"
 // @success 200 {object} portainer.Team "Success"
 // @failure 400 "Invalid request"
-// @failure 409 "Team already exists"
+// @failure 409 "A team with the same name already exists"
 // @failure 500 "Server error"
 // @router /teams [post]
 func (handler *Handler) teamCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	var payload teamCreatePayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	team, err := handler.DataStore.Team().TeamByName(payload.Name)
-	if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.InternalServerError("Unable to retrieve teams from the database", err)
+	var team *portainer.Team
+
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		var err error
+		team, err = createTeam(tx, payload)
+
+		return err
+	}); err != nil {
+		var httpErr *httperror.HandlerError
+		if errors.As(err, &httpErr) {
+			return httpErr
+		}
+
+		return httperror.InternalServerError("Unexpected error", err)
+	}
+
+	return response.JSON(w, team)
+}
+
+func createTeam(tx dataservices.DataStoreTx, payload teamCreatePayload) (*portainer.Team, error) {
+	team, err := tx.Team().TeamByName(payload.Name)
+	if err != nil && !tx.IsErrObjectNotFound(err) {
+		return nil, httperror.InternalServerError("Unable to retrieve teams from the database", err)
 	}
 	if team != nil {
-		return httperror.Conflict("A team with the same name already exists", errors.New("Team already exists"))
+		return nil, httperror.Conflict("A team with the same name already exists", errors.New("Team already exists"))
 	}
 
-	team = &portainer.Team{
-		Name: payload.Name,
-	}
+	team = &portainer.Team{Name: payload.Name}
 
-	err = handler.DataStore.Team().Create(team)
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist the team inside the database", err)
+	if err := tx.Team().Create(team); err != nil {
+		return nil, httperror.InternalServerError("Unable to persist the team inside the database", err)
 	}
 
 	for _, teamLeader := range payload.TeamLeaders {
@@ -72,11 +90,10 @@ func (handler *Handler) teamCreate(w http.ResponseWriter, r *http.Request) *http
 			Role:   portainer.TeamLeader,
 		}
 
-		err = handler.DataStore.TeamMembership().Create(membership)
-		if err != nil {
-			return httperror.InternalServerError("Unable to persist team leadership inside the database", err)
+		if err := tx.TeamMembership().Create(membership); err != nil {
+			return nil, httperror.InternalServerError("Unable to persist team leadership inside the database", err)
 		}
 	}
 
-	return response.JSON(w, team)
+	return team, nil
 }

api/http/handler/teams/team_create_test.go

@@ -0,0 +1,65 @@
+package teams
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/portainer/portainer/api/datastore"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+func TestConcurrentTeamCreation(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, false)
+
+	h := &Handler{
+		DataStore: store,
+	}
+
+	tcp := teamCreatePayload{
+		Name: "portainer",
+	}
+
+	m, err := json.Marshal(tcp)
+	require.NoError(t, err)
+
+	errGroup := &errgroup.Group{}
+
+	n := 100
+
+	for i := 0; i < n; i++ {
+		errGroup.Go(func() error {
+			req, err := http.NewRequest(http.MethodPost, "/teams", bytes.NewReader(m))
+			if err != nil {
+				return err
+			}
+
+			if err := h.teamCreate(httptest.NewRecorder(), req); err != nil {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	err = errGroup.Wait()
+	require.Error(t, err)
+
+	teams, err := store.Team().ReadAll()
+	require.NotEmpty(t, teams)
+	require.NoError(t, err)
+
+	teamCreated := false
+	for _, team := range teams {
+		if team.Name == tcp.Name {
+			require.False(t, teamCreated)
+			teamCreated = true
+		}
+	}
+
+	require.True(t, teamCreated)
+}

api/http/handler/users/user_create.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -54,12 +55,33 @@ func (handler *Handler) userCreate(w http.ResponseWriter, r *http.Request) *http
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	user, err := handler.DataStore.User().UserByUsername(payload.Username)
-	if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.InternalServerError("Unable to retrieve users from the database", err)
+	var user *portainer.User
+
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		var err error
+		user, err = handler.createUser(tx, payload)
+
+		return err
+	}); err != nil {
+		var httpErr *httperror.HandlerError
+		if errors.As(err, &httpErr) {
+			return httpErr
+		}
+
+		return httperror.InternalServerError("Unexpected error", err)
 	}
+
+	return response.JSON(w, user)
+}
+
+func (handler *Handler) createUser(tx dataservices.DataStoreTx, payload userCreatePayload) (*portainer.User, error) {
+	user, err := tx.User().UserByUsername(payload.Username)
+	if err != nil && !tx.IsErrObjectNotFound(err) {
+		return nil, httperror.InternalServerError("Unable to retrieve users from the database", err)
+	}
+
 	if user != nil {
-		return httperror.Conflict("Another user with the same username already exists", errUserAlreadyExists)
+		return nil, httperror.Conflict("Another user with the same username already exists", errUserAlreadyExists)
 	}
 
 	user = &portainer.User{
@@ -67,33 +89,33 @@ func (handler *Handler) userCreate(w http.ResponseWriter, r *http.Request) *http
 		Role:     portainer.UserRole(payload.Role),
 	}
 
-	settings, err := handler.DataStore.Settings().Settings()
+	settings, err := tx.Settings().Settings()
 	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve settings from the database", err)
+		return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
 	}
 
-	// when ldap/oauth is on, can only add users without password
+	// When LDAP/OAuth is on, can only add users without password
 	if (settings.AuthenticationMethod == portainer.AuthenticationLDAP || settings.AuthenticationMethod == portainer.AuthenticationOAuth) && payload.Password != "" {
-		errMsg := "A user with password can not be created when authentication method is Oauth or LDAP"
-		return httperror.BadRequest(errMsg, errors.New(errMsg))
+		errMsg := "a user with password can not be created when authentication method is Oauth or LDAP"
+		return nil, httperror.BadRequest(errMsg, errors.New(errMsg))
 	}
 
 	if settings.AuthenticationMethod == portainer.AuthenticationInternal {
 		if !handler.passwordStrengthChecker.Check(payload.Password) {
-			return httperror.BadRequest("Password does not meet the requirements", nil)
+			return nil, httperror.BadRequest("Password does not meet the requirements", nil)
 		}
 
 		user.Password, err = handler.CryptoService.Hash(payload.Password)
 		if err != nil {
-			return httperror.InternalServerError("Unable to hash user password", errCryptoHashFailure)
+			return nil, httperror.InternalServerError("Unable to hash user password", errCryptoHashFailure)
 		}
 	}
 
-	err = handler.DataStore.User().Create(user)
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist user inside the database", err)
+	if err := tx.User().Create(user); err != nil {
+		return nil, httperror.InternalServerError("Unable to persist user inside the database", err)
 	}
 
 	hideFields(user)
-	return response.JSON(w, user)
+
+	return user, nil
 }

api/http/handler/users/user_create_access_token.go

@@ -2,6 +2,7 @@ package users
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
@@ -20,9 +21,6 @@ type userAccessTokenCreatePayload struct {
 }
 
 func (payload *userAccessTokenCreatePayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.Password) {
-		return errors.New("invalid password: cannot be empty")
-	}
 	if govalidator.IsNull(payload.Description) {
 		return errors.New("invalid description: cannot be empty")
 	}
@@ -44,6 +42,7 @@ type accessTokenResponse struct {
 // @summary Generate an API key for a user
 // @description Generates an API key for a user.
 // @description Only the calling user can generate a token for themselves.
+// @description Password is required only for internal authentication.
 // @description **Access policy**: restricted
 // @tags users
 // @security jwt
@@ -51,7 +50,7 @@ type accessTokenResponse struct {
 // @produce json
 // @param id path int true "User identifier"
 // @param body body userAccessTokenCreatePayload true "details"
-// @success 201 {object} accessTokenResponse "Created"
+// @success 200 {object} accessTokenResponse "Created"
 // @failure 400 "Invalid request"
 // @failure 401 "Unauthorized"
 // @failure 403 "Permission denied"
@@ -60,8 +59,13 @@ type accessTokenResponse struct {
 // @router /users/{id}/tokens [post]
 func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	// specifically require Cookie auth for this endpoint since API-Key based auth is not supported
-	if jwt, _ := handler.bouncer.CookieAuthLookup(r); jwt == nil {
-		return httperror.Unauthorized("Auth not supported", errors.New("Cookie Authentication required"))
+	jwt, _ := handler.bouncer.CookieAuthLookup(r)
+	if jwt == nil {
+		jwt, _ = handler.bouncer.JWTAuthLookup(r)
+	}
+
+	if jwt == nil {
+		return httperror.Unauthorized("Auth not supported", errors.New("Authentication required"))
 	}
 
 	var payload userAccessTokenCreatePayload
@@ -89,9 +93,21 @@ func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Req
 		return httperror.InternalServerError("Unable to find a user with the specified identifier inside the database", err)
 	}
 
-	err = handler.CryptoService.CompareHashAndData(user.Password, payload.Password)
+	internalAuth, err := handler.usesInternalAuthentication(portainer.UserID(userID))
 	if err != nil {
-		return httperror.Forbidden("Current password doesn't match", errors.New("Current password does not match the password provided. Please try again"))
+		return httperror.InternalServerError("Unable to determine the authentication method", err)
+	}
+
+	if internalAuth {
+		// Internal auth requires the password field and must not be empty
+		if govalidator.IsNull(payload.Password) {
+			return httperror.BadRequest("Invalid request payload", errors.New("invalid password: cannot be empty"))
+		}
+
+		err = handler.CryptoService.CompareHashAndData(user.Password, payload.Password)
+		if err != nil {
+			return httperror.Forbidden("Current password doesn't match", errors.New("Current password does not match the password provided. Please try again"))
+		}
 	}
 
 	rawAPIKey, apiKey, err := handler.apiKeyService.GenerateApiKey(*user, payload.Description)
@@ -99,6 +115,20 @@ func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Req
 		return httperror.InternalServerError("Internal Server Error", err)
 	}
 
-	w.WriteHeader(http.StatusCreated)
-	return response.JSON(w, accessTokenResponse{rawAPIKey, *apiKey})
+	return response.JSONWithStatus(w, accessTokenResponse{rawAPIKey, *apiKey}, http.StatusCreated)
+}
+
+func (handler *Handler) usesInternalAuthentication(userID portainer.UserID) (bool, error) {
+	// userID 1 is the admin user and always uses internal auth
+	if userID == 1 {
+		return true, nil
+	}
+
+	// otherwise determine the auth method from the settings
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return false, fmt.Errorf("unable to retrieve the settings from the database: %w", err)
+	}
+
+	return settings.AuthenticationMethod == portainer.AuthenticationInternal, nil
 }

api/http/handler/users/user_create_access_token_test.go

@@ -107,7 +107,7 @@ func Test_userCreateAccessToken(t *testing.T) {
 
 		body, err := io.ReadAll(rr.Body)
 		is.NoError(err, "ReadAll should not return error")
-		is.Equal(`{"message":"Auth not supported","details":"Cookie Authentication required"}`, string(body))
+		is.Equal(`{"message":"Auth not supported","details":"Authentication required"}`, string(body))
 	})
 }