Revert "fix(kube): fix text in activity and authentication logs teasers [EE-6742]" (#11727)

Co-authored-by: testa113 <testa113>

.eslintrc.yml

@@ -10,6 +10,7 @@ globals:
 extends:
   - 'eslint:recommended'
   - 'plugin:storybook/recommended'
+  - 'plugin:import/typescript'
   - prettier
 
 plugins:
@@ -29,6 +30,7 @@ rules:
   no-empty: warn
   no-empty-function: warn
   no-useless-escape: 'off'
+  import/named: error
   import/order:
     [
       'error',
@@ -43,6 +45,12 @@ rules:
         pathGroupsExcludedImportTypes: ['internal'],
       },
     ]
+  no-restricted-imports:
+    - error
+    - patterns:
+        - group:
+            - '@/react/test-utils/*'
+          message: 'These utils are just for test files'
 
 settings:
   'import/resolver':
@@ -51,6 +59,8 @@ settings:
         - ['@@', './app/react/components']
         - ['@', './app']
       extensions: ['.js', '.ts', '.tsx']
+    typescript: true
+    node: true
 
 overrides:
   - files:
@@ -75,6 +85,7 @@ overrides:
     settings:
       react:
         version: 'detect'
+
     rules:
       import/order:
         [
@@ -108,6 +119,12 @@ overrides:
       'no-await-in-loop': 'off'
       'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
       'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
+      '@typescript-eslint/no-restricted-imports':
+        - error
+        - patterns:
+            - group:
+                - '@/react/test-utils/*'
+              message: 'These utils are just for test files'
     overrides: # allow props spreading for hoc files
       - files:
           - app/**/with*.ts{,x}
@@ -121,7 +138,11 @@ overrides:
       'vitest/env': true
     rules:
       'react/jsx-no-constructed-context-values': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
   - files:
       - app/**/*.stories.*
     rules:
       'no-alert': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off

.github/workflows/ci.yaml

@@ -5,7 +5,7 @@ on:
   push:
     branches:
       - 'develop'
-      - '!release/*'
+      - 'release/*'
   pull_request:
     branches:
       - 'develop'
@@ -20,9 +20,9 @@ on:
       - ready_for_review
 
 env:
-  DOCKER_HUB_REPO: portainerci/portainer
-  NODE_ENV: testing
-  GO_VERSION: 1.21.6
+  DOCKER_HUB_REPO: portainerci/portainer-ce
+  EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:
@@ -30,81 +30,59 @@ jobs:
     strategy:
       matrix:
         config:
-          - { platform: linux, arch: amd64 }
-          - { platform: linux, arch: arm64 }
+          - { platform: linux, arch: amd64, version: "" }
+          - { platform: linux, arch: arm64, version: "" }
+          - { platform: linux, arch: arm, version: "" }
+          - { platform: linux, arch: ppc64le, version: "" }
+          - { platform: linux, arch: s390x, version: "" }
           - { platform: windows, arch: amd64, version: 1809 }
           - { platform: windows, arch: amd64, version: ltsc2022 }
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
     steps:
       - name: '[preparation] checkout the current branch'
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4.1.1
         with:
           ref: ${{ github.event.inputs.branch }}
       - name: '[preparation] set up golang'
-        uses: actions/setup-go@v4.0.1
+        uses: actions/setup-go@v5.0.0
         with:
           go-version: ${{ env.GO_VERSION }}
-          cache: false
-      - name: '[preparation] cache paths'
-        id: cache-dir-path
-        run: |
-          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
-          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
-      - name: '[preparation] cache go'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache-dir-path.outputs.go-build-dir }}
-            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
-          enableCrossOsArchive: true
       - name: '[preparation] set up node.js'
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4.0.1
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: ''
-      - name: '[preparation] cache yarn'
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
-          enableCrossOsArchive: true
+          cache: 'yarn'
       - name: '[preparation] set up qemu'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3.0.0
       - name: '[preparation] set up docker context for buildx'
         run: docker context create builders
       - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3.0.0
         with:
           endpoint: builders
       - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: '[preparation] set the container image tag'
         run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
             CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
           else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
             CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
           fi
-
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
-          else 
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
-          fi
-
-          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
+          
+          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}" >> $GITHUB_ENV
       - name: '[execution] build linux & windows portainer binaries'
         run: |
           export YARN_VERSION=$(yarn --version)
@@ -112,6 +90,12 @@ jobs:
           export BUILDNUMBER=${GITHUB_RUN_NUMBER}
           GIT_COMMIT_HASH_LONG=${{ github.sha }}
           export GIT_COMMIT_HASH_SHORT={GIT_COMMIT_HASH_LONG:0:7}
+          
+          NODE_ENV="testing"
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            NODE_ENV="production"
+          fi
+          
           make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
         env:
           CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
@@ -123,35 +107,70 @@ jobs:
           else 
             docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
             docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            
+            if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            fi
           fi
         env:
           CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
   build_manifests:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     if: github.event.pull_request.draft == false
     needs: [build_images]
     steps:
       - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: '[preparation] set up docker context for buildx'
         run: docker version && docker context create builders
       - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3.0.0
         with:
           endpoint: builders
       - name: '[execution] build and push manifests'
         run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
             CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
           else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
             CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
           fi
 
           docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
             "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
+          
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
+            
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x"
+            
+            docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
+          fi

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/nightly-security-scan.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
 
 jobs:
   client-dependencies:

.github/workflows/pr-security.yml

@@ -14,7 +14,7 @@ on:
       - '.github/workflows/pr-security.yml'
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

.github/workflows/test.yaml

@@ -1,17 +1,25 @@
 name: Test
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 on:
   pull_request:
+    branches:
+      - master
+      - develop
+      - release/*
     types:
       - opened
       - reopened
       - synchronize
       - ready_for_review
   push:
+    branches:
+      - master
+      - develop
+      - release/*
 
 jobs:
   test-client:

.github/workflows/validate-openapi-spec.yaml

@@ -13,7 +13,7 @@ on:
       - ready_for_review
 
 env:
-  GO_VERSION: 1.21.6
+  GO_VERSION: 1.21.9
   NODE_VERSION: 18.x
 
 jobs:

api/backup/restore.go

@@ -26,7 +26,7 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
 	if password != "" {
 		archive, err = decrypt(archive, password)
 		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
+			return errors.Wrap(err, "failed to decrypt the archive. Please ensure the password is correct and try again")
 		}
 	}
 

api/cmd/portainer/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/datastore/migrator"
+	"github.com/portainer/portainer/api/datastore/postinit"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -457,14 +458,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	authorizationService := authorization.NewService(dataStore)
 	authorizationService.K8sClientFactory = kubernetesClientFactory
 
-	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
-
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed initializing snapshot service")
-	}
-	snapshotService.Start()
-
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
@@ -489,6 +482,14 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
+	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, dockerClientFactory, authorizationService, shutdownCtx, *flags.Assets, kubernetesDeployer)
+
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
+	}
+	snapshotService.Start()
+
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing helm package manager")
@@ -578,10 +579,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	// but some more complex migrations require access to a kubernetes or docker
 	// client. Therefore we run a separate migration process just before
 	// starting the server.
-	postInitMigrator := datastore.NewPostInitMigrator(
+	postInitMigrator := postinit.NewPostInitMigrator(
 		kubernetesClientFactory,
 		dockerClientFactory,
 		dataStore,
+		*flags.Assets,
+		kubernetesDeployer,
 	)
 	if err := postInitMigrator.PostInitMigrate(); err != nil {
 		log.Fatal().Err(err).Msg("failure during post init migrations")
@@ -650,6 +653,7 @@ func main() {
 			Msg("starting Portainer")
 
 		err := server.Start()
+
 		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/crypto/aes.go

@@ -1,52 +1,216 @@
 package crypto
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
 	"io"
 
+	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/scrypt"
 )
 
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
+const (
+	// AES GCM settings
+	aesGcmHeader    = "AES256-GCM" // The encrypted file header
+	aesGcmBlockSize = 1024 * 1024  // 1MB block for aes gcm
 
-var emptySalt []byte = make([]byte, 0)
+	// Argon2 settings
+	// Recommded settings lower memory hardware according to current OWASP recommendations
+	// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
+	// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
+	argon2MemoryCost = 12 * 1024
+	argon2TimeCost   = 3
+	argon2Threads    = 1
+	argon2KeyLength  = 32
+)
 
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
+// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
 func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	err := aesEncryptGCM(input, output, passphrase)
 	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
+		return fmt.Errorf("error encrypting file: %w", err)
 	}
 
 	return nil
 }
 
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
 func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Read file header to determine how it was encrypted
+	inputReader := bufio.NewReader(input)
+	header, err := inputReader.Peek(len(aesGcmHeader))
+	if err != nil {
+		return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
+	}
+
+	if string(header) == aesGcmHeader {
+		reader, err := aesDecryptGCM(inputReader, passphrase)
+		if err != nil {
+			return nil, fmt.Errorf("error decrypting file: %w", err)
+		}
+
+		return reader, nil
+	}
+
+	// Use the previous decryption routine which has no header (to support older archives)
+	reader, err := aesDecryptOFB(inputReader, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
+	}
+
+	return reader, nil
+}
+
+// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
+func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
+	// Derive key using argon2 with a random salt
+	salt := make([]byte, 16) // 16 bytes salt
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		return err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return err
+	}
+
+	// Generate nonce
+	nonce, err := NewRandomNonce(aesgcm.NonceSize())
+	if err != nil {
+		return err
+	}
+
+	// write the header
+	if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
+		return err
+	}
+
+	// Write nonce and salt to the output file
+	if _, err := output.Write(salt); err != nil {
+		return err
+	}
+	if _, err := output.Write(nonce.Value()); err != nil {
+		return err
+	}
+
+	// Buffer for reading plaintext blocks
+	buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
+	ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
+
+	// Encrypt plaintext in blocks
+	for {
+		n, err := io.ReadFull(input, buf)
+		if n == 0 {
+			break // end of plaintext input
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return err
+		}
+
+		// Seal encrypts the plaintext using the nonce returning the updated slice.
+		ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
+
+		_, err = output.Write(ciphertext)
+		if err != nil {
+			return err
+		}
+
+		nonce.Increment()
+	}
+
+	return nil
+}
+
+// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
+func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Reader & verify header
+	header := make([]byte, len(aesGcmHeader))
+	if _, err := io.ReadFull(input, header); err != nil {
+		return nil, err
+	}
+
+	if string(header) != aesGcmHeader {
+		return nil, fmt.Errorf("invalid header")
+	}
+
+	// Read salt
+	salt := make([]byte, 16) // Salt size
+	if _, err := io.ReadFull(input, salt); err != nil {
+		return nil, err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+
+	// Initialize AES cipher block
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create GCM mode with the cipher block
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read nonce from the input reader
+	nonce := NewNonce(aesgcm.NonceSize())
+	if err := nonce.Read(input); err != nil {
+		return nil, err
+	}
+
+	// Initialize a buffer to store decrypted data
+	buf := bytes.Buffer{}
+	plaintext := make([]byte, aesGcmBlockSize)
+
+	// Decrypt the ciphertext in blocks
+	for {
+		// Read a block of ciphertext from the input reader
+		ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
+		n, err := io.ReadFull(input, ciphertextBlock)
+		if n == 0 {
+			break // end of ciphertext
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return nil, err
+		}
+
+		// Decrypt the block of ciphertext
+		plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = buf.Write(plaintext)
+		if err != nil {
+			return nil, err
+		}
+
+		nonce.Increment()
+	}
+
+	return &buf, nil
+}
+
+// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+// note: This function used to decrypt files that were encrypted without a header i.e. old archives
+func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
+	var emptySalt []byte = make([]byte, 0)
+
 	// making a 32 bytes key that would correspond to AES-256
 	// don't necessarily need a salt, so just kept in empty
 	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
@@ -59,11 +223,9 @@ func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
 		return nil, err
 	}
 
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
+	// If the key is unique for each ciphertext, then it's ok to use a zero IV.
 	var iv [aes.BlockSize]byte
 	stream := cipher.NewOFB(block, iv[:])
-
 	reader := &cipher.StreamReader{S: stream, R: input}
 
 	return reader, nil

api/crypto/aes_test.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -9,7 +10,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func randBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	const passphrase = "passphrase"
+
 	tmpdir := t.TempDir()
 
 	var (
@@ -18,17 +31,99 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024*1024*100 + 523)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withStrongPassphrase(t *testing.T) {
+	const passphrase = "A strong passphrase with special characters: !@#$%^&*()_+"
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withTheSamePasswordSmallFile(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
 
 	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
 	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
@@ -57,7 +152,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024 * 50)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -96,7 +191,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1034)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -117,11 +212,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	decryptedFileWriter, _ := os.Create(decryptedFilePath)
 	defer decryptedFileWriter.Close()
 
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := os.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+	_, err = AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.NotNil(t, err, "Should not allow decrypt with wrong passphrase")
 }

api/crypto/nonce.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+type Nonce struct {
+	val []byte
+}
+
+func NewNonce(size int) *Nonce {
+	return &Nonce{val: make([]byte, size)}
+}
+
+// NewRandomNonce generates a new initial nonce with the lower byte set to a random value
+// This ensures there are plenty of nonce values availble before rolling over
+// Based on ideas from the Secure Programming Cookbook for C and C++ by John Viega, Matt Messier
+// https://www.oreilly.com/library/view/secure-programming-cookbook/0596003943/ch04s09.html
+func NewRandomNonce(size int) (*Nonce, error) {
+	randomBytes := 1
+	if size <= randomBytes {
+		return nil, errors.New("nonce size must be greater than the number of random bytes")
+	}
+
+	randomPart := make([]byte, randomBytes)
+	if _, err := rand.Read(randomPart); err != nil {
+		return nil, err
+	}
+
+	zeroPart := make([]byte, size-randomBytes)
+	nonceVal := append(randomPart, zeroPart...)
+	return &Nonce{val: nonceVal}, nil
+}
+
+func (n *Nonce) Read(stream io.Reader) error {
+	_, err := io.ReadFull(stream, n.val)
+	return err
+}
+
+func (n *Nonce) Value() []byte {
+	return n.val
+}
+
+func (n *Nonce) Increment() error {
+	// Start incrementing from the least significant byte
+	for i := len(n.val) - 1; i >= 0; i-- {
+		// Increment the current byte
+		n.val[i]++
+
+		// Check for overflow
+		if n.val[i] != 0 {
+			// No overflow, nonce is successfully incremented
+			return nil
+		}
+	}
+
+	// If we reach here, it means the nonce has overflowed
+	return errors.New("nonce overflow")
+}

api/dataservices/interface.go

@@ -73,6 +73,7 @@ type (
 	PendingActionsService interface {
 		BaseCRUD[portainer.PendingActions, portainer.PendingActionsID]
 		GetNextIdentifier() int
+		DeleteByEndpointID(ID portainer.EndpointID) error
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks

api/dataservices/pendingactions/pendingactions.go

@@ -1,10 +1,12 @@
 package pendingactions
 
 import (
+	"fmt"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -45,6 +47,12 @@ func (s Service) Update(ID portainer.PendingActionsID, config *portainer.Pending
 	})
 }
 
+func (s Service) DeleteByEndpointID(ID portainer.EndpointID) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).DeleteByEndpointID(ID)
+	})
+}
+
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
 		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]{
@@ -68,6 +76,29 @@ func (s ServiceTx) Update(ID portainer.PendingActionsID, config *portainer.Pendi
 	return s.BaseDataServiceTx.Update(ID, config)
 }
 
+func (s ServiceTx) DeleteByEndpointID(ID portainer.EndpointID) error {
+	log.Debug().Int("endpointId", int(ID)).Msg("deleting pending actions for endpoint")
+	pendingActions, err := s.BaseDataServiceTx.ReadAll()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve pending-actions for endpoint (%d): %w", ID, err)
+	}
+
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == ID {
+			err := s.BaseDataServiceTx.Delete(pendingAction.ID)
+			if err != nil {
+				log.Debug().Int("endpointId", int(ID)).Msgf("failed to delete pending action: %v", err)
+			}
+		}
+	}
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
 	return service.Connection.GetNextIdentifier(BucketName)

api/datastore/migrate_data.go

@@ -86,6 +86,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		EdgeStackService:        store.EdgeStackService,
 		EdgeJobService:          store.EdgeJobService,
 		TunnelServerService:     store.TunnelServerService,
+		PendingActionsService:   store.PendingActionsService,
 	}
 }
 

api/datastore/migrate_post_init.go

@@ -1,117 +0,0 @@
-package datastore
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
-	"github.com/portainer/portainer/api/kubernetes/cli"
-
-	"github.com/rs/zerolog/log"
-)
-
-type PostInitMigrator struct {
-	kubeFactory   *cli.ClientFactory
-	dockerFactory *dockerclient.ClientFactory
-	dataStore     dataservices.DataStore
-}
-
-func NewPostInitMigrator(kubeFactory *cli.ClientFactory, dockerFactory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *PostInitMigrator {
-	return &PostInitMigrator{
-		kubeFactory:   kubeFactory,
-		dockerFactory: dockerFactory,
-		dataStore:     dataStore,
-	}
-}
-
-func (migrator *PostInitMigrator) PostInitMigrate() error {
-	if err := migrator.PostInitMigrateIngresses(); err != nil {
-		return err
-	}
-
-	migrator.PostInitMigrateGPUs()
-
-	return nil
-}
-
-func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
-	endpoints, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for i := range endpoints {
-		// Early exit if we do not need to migrate!
-		if !endpoints[i].PostInitMigrations.MigrateIngresses {
-			return nil
-		}
-
-		err := migrator.kubeFactory.MigrateEndpointIngresses(&endpoints[i])
-		if err != nil {
-			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
-		}
-	}
-
-	return nil
-}
-
-// PostInitMigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
-// If there's an error getting the containers, we'll log it and move on
-func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
-	environments, err := migrator.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		log.Err(err).Msg("failure getting endpoints")
-		return
-	}
-
-	for i := range environments {
-		if environments[i].Type == portainer.DockerEnvironment {
-			// // Early exit if we do not need to migrate!
-			if !environments[i].PostInitMigrations.MigrateGPUs {
-				return
-			}
-
-			// set the MigrateGPUs flag to false so we don't run this again
-			environments[i].PostInitMigrations.MigrateGPUs = false
-			migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-			// create a docker client
-			dockerClient, err := migrator.dockerFactory.CreateClient(&environments[i], "", nil)
-			if err != nil {
-				log.Err(err).Msg("failure creating docker client for environment: " + environments[i].Name)
-				return
-			}
-			defer dockerClient.Close()
-
-			// get all containers
-			containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
-			if err != nil {
-				log.Err(err).Msg("failed to list containers")
-				return
-			}
-
-			// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole endpoint
-		containersLoop:
-			for _, container := range containers {
-				// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
-				containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
-				if err != nil {
-					log.Err(err).Msg("failed to inspect container")
-					return
-				}
-
-				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
-				for _, deviceRequest := range deviceRequests {
-					if deviceRequest.Driver == "nvidia" {
-						environments[i].EnableGPUManagement = true
-						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
-
-						break containersLoop
-					}
-				}
-			}
-		}
-	}
-}

api/datastore/migrator/migrate_dbversion110.go

@@ -23,3 +23,29 @@ func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
 
 	return migrator.settingsService.UpdateSettings(settings)
 }
+
+// In PortainerCE the resource overcommit option should always be true across all endpoints
+func (migrator *Migrator) updateResourceOverCommitToDB110() error {
+	log.Info().Msg("updating resource overcommit setting to true")
+
+	endpoints, err := migrator.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.Type == portainer.KubernetesLocalEnvironment ||
+			endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
+			endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+
+			endpoint.Kubernetes.Configuration.EnableResourceOverCommit = true
+
+			err = migrator.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion111.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
+)
+
+func (migrator *Migrator) cleanPendingActionsForDeletedEndpointsForDB111() error {
+	log.Info().Msg("cleaning up pending actions for deleted endpoints")
+
+	pendingActions, err := migrator.pendingActionsService.ReadAll()
+	if err != nil {
+		return err
+	}
+
+	endpoints := make(map[portainer.EndpointID]struct{})
+	for _, action := range pendingActions {
+		endpoints[action.EndpointID] = struct{}{}
+	}
+
+	for endpointId := range endpoints {
+		_, err := migrator.endpointService.Endpoint(endpointId)
+		if dataservices.IsErrObjectNotFound(err) {
+			err := migrator.pendingActionsService.DeleteByEndpointID(endpointId)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -14,6 +14,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
 	"github.com/portainer/portainer/api/dataservices/extension"
 	"github.com/portainer/portainer/api/dataservices/fdoprofile"
+	"github.com/portainer/portainer/api/dataservices/pendingactions"
 	"github.com/portainer/portainer/api/dataservices/registry"
 	"github.com/portainer/portainer/api/dataservices/resourcecontrol"
 	"github.com/portainer/portainer/api/dataservices/role"
@@ -58,6 +59,7 @@ type (
 		edgeStackService        *edgestack.Service
 		edgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		pendingActionsService   *pendingactions.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -85,6 +87,7 @@ type (
 		EdgeStackService        *edgestack.Service
 		EdgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
+		PendingActionsService   *pendingactions.Service
 	}
 )
 
@@ -114,6 +117,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		edgeStackService:        parameters.EdgeStackService,
 		edgeJobService:          parameters.EdgeJobService,
 		TunnelServerService:     parameters.TunnelServerService,
+		pendingActionsService:   parameters.PendingActionsService,
 	}
 
 	migrator.initMigrations()
@@ -230,6 +234,10 @@ func (m *Migrator) initMigrations() {
 	)
 	m.addMigrations("2.20",
 		m.updateAppTemplatesVersionForDB110,
+		m.updateResourceOverCommitToDB110,
+	)
+	m.addMigrations("2.20.2",
+		m.cleanPendingActionsForDeletedEndpointsForDB111,
 	)
 
 	// Add new migrations below...

api/datastore/postinit/migrate_post_init.go

@@ -0,0 +1,203 @@
+package postinit
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerClient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+
+	"github.com/rs/zerolog/log"
+)
+
+type PostInitMigrator struct {
+	kubeFactory        *cli.ClientFactory
+	dockerFactory      *dockerClient.ClientFactory
+	dataStore          dataservices.DataStore
+	assetsPath         string
+	kubernetesDeployer portainer.KubernetesDeployer
+}
+
+func NewPostInitMigrator(
+	kubeFactory *cli.ClientFactory,
+	dockerFactory *dockerClient.ClientFactory,
+	dataStore dataservices.DataStore,
+	assetsPath string,
+	kubernetesDeployer portainer.KubernetesDeployer,
+) *PostInitMigrator {
+	return &PostInitMigrator{
+		kubeFactory:        kubeFactory,
+		dockerFactory:      dockerFactory,
+		dataStore:          dataStore,
+		assetsPath:         assetsPath,
+		kubernetesDeployer: kubernetesDeployer,
+	}
+}
+
+// PostInitMigrate will run all post-init migrations, which require docker/kube clients for all edge or non-edge environments
+func (postInitMigrator *PostInitMigrator) PostInitMigrate() error {
+	environments, err := postInitMigrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		log.Error().Err(err).Msg("Error getting environments")
+		return err
+	}
+
+	for _, environment := range environments {
+		// edge environments will run after the server starts, in pending actions
+		if endpointutils.IsEdgeEndpoint(&environment) {
+			log.Info().Msgf("Adding pending action 'PostInitMigrateEnvironment' for environment %d", environment.ID)
+			err = postInitMigrator.createPostInitMigrationPendingAction(environment.ID)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error creating pending action for environment %d", environment.ID)
+			}
+		} else {
+			// non-edge environments will run before the server starts.
+			err = postInitMigrator.MigrateEnvironment(&environment)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error running post-init migrations for non-edge environment %d", environment.ID)
+			}
+		}
+
+	}
+
+	return nil
+}
+
+// try to create a post init migration pending action. If it already exists, do nothing
+// this function exists for readability, not reusability
+// TODO: This should be moved into pending actions as part of the pending action migration
+func (postInitMigrator *PostInitMigrator) createPostInitMigrationPendingAction(environmentID portainer.EndpointID) error {
+	migrateEnvPendingAction := portainer.PendingActions{
+		EndpointID: environmentID,
+		Action:     actions.PostInitMigrateEnvironment,
+	}
+
+	// Get all pending actions and filter them by endpoint, action and action args that are equal to the migrateEnvPendingAction
+	pendingActions, err := postInitMigrator.dataStore.PendingActions().ReadAll()
+	if err != nil {
+		log.Error().Err(err).Msgf("Error retrieving pending actions")
+		return fmt.Errorf("failed to retrieve pending actions for environment %d: %w", environmentID, err)
+	}
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == environmentID &&
+			pendingAction.Action == migrateEnvPendingAction.Action &&
+			reflect.DeepEqual(pendingAction.ActionData, migrateEnvPendingAction.ActionData) {
+			log.Debug().Msgf("Migration pending action for environment %d already exists, skipping creating another", environmentID)
+			return nil
+		}
+	}
+
+	// If there are no pending actions for the given endpoint, create one
+	err = postInitMigrator.dataStore.PendingActions().Create(&migrateEnvPendingAction)
+	if err != nil {
+		log.Error().Err(err).Msgf("Error creating pending action for environment %d", environmentID)
+	}
+	return nil
+}
+
+// MigrateEnvironment runs migrations on a single environment
+func (migrator *PostInitMigrator) MigrateEnvironment(environment *portainer.Endpoint) error {
+	log.Info().Msgf("Executing post init migration for environment %d", environment.ID)
+
+	switch {
+	case endpointutils.IsKubernetesEndpoint(environment):
+		// get the kubeclient for the environment, and skip all kube migrations if there's an error
+		kubeclient, err := migrator.kubeFactory.GetKubeClient(environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating kubeclient for environment: %d", environment.ID)
+			return err
+		}
+		// if one environment fails, it is logged and the next migration runs. The error is returned at the end and handled by pending actions
+		err = migrator.MigrateIngresses(*environment, kubeclient)
+		if err != nil {
+			return err
+		}
+		return nil
+	case endpointutils.IsDockerEndpoint(environment):
+		// get the docker client for the environment, and skip all docker migrations if there's an error
+		dockerClient, err := migrator.dockerFactory.CreateClient(environment, "", nil)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error creating docker client for environment: %d", environment.ID)
+			return err
+		}
+		defer dockerClient.Close()
+		migrator.MigrateGPUs(*environment, dockerClient)
+	}
+
+	return nil
+}
+
+func (migrator *PostInitMigrator) MigrateIngresses(environment portainer.Endpoint, kubeclient *cli.KubeClient) error {
+	// Early exit if we do not need to migrate!
+	if !environment.PostInitMigrations.MigrateIngresses {
+		return nil
+	}
+	log.Debug().Msgf("Migrating ingresses for environment %d", environment.ID)
+
+	err := migrator.kubeFactory.MigrateEndpointIngresses(&environment, migrator.dataStore, kubeclient)
+	if err != nil {
+		log.Error().Err(err).Msgf("Error migrating ingresses for environment %d", environment.ID)
+		return err
+	}
+	return nil
+}
+
+// MigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
+// If there's an error getting the containers, we'll log it and move on
+func (migrator *PostInitMigrator) MigrateGPUs(e portainer.Endpoint, dockerClient *client.Client) error {
+	return migrator.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		environment, err := tx.Endpoint().Endpoint(e.ID)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting environment %d", environment.ID)
+			return err
+		}
+		// Early exit if we do not need to migrate!
+		if !environment.PostInitMigrations.MigrateGPUs {
+			return nil
+		}
+		log.Debug().Msgf("Migrating GPUs for environment %d", e.ID)
+
+		// get all containers
+		containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{All: true})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to list containers for environment %d", environment.ID)
+			return err
+		}
+
+		// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole environment
+	containersLoop:
+		for _, container := range containers {
+			// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
+			containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to inspect container")
+				continue
+			}
+
+			deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
+			for _, deviceRequest := range deviceRequests {
+				if deviceRequest.Driver == "nvidia" {
+					environment.EnableGPUManagement = true
+					break containersLoop
+				}
+			}
+		}
+
+		// set the MigrateGPUs flag to false so we don't run this again
+		environment.PostInitMigrations.MigrateGPUs = false
+		err = tx.Endpoint().UpdateEndpoint(environment.ID, environment)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error updating EnableGPUManagement flag for environment %d", environment.ID)
+			return err
+		}
+
+		return nil
+	})
+}

api/datastore/services_tx.go

@@ -16,7 +16,9 @@ func (tx *StoreTx) IsErrObjectNotFound(err error) bool {
 
 func (tx *StoreTx) CustomTemplate() dataservices.CustomTemplateService { return nil }
 
-func (tx *StoreTx) PendingActions() dataservices.PendingActionsService { return nil }
+func (tx *StoreTx) PendingActions() dataservices.PendingActionsService {
+	return tx.store.PendingActionsService.Tx(tx.tx)
+}
 
 func (tx *StoreTx) EdgeGroup() dataservices.EdgeGroupService {
 	return tx.store.EdgeGroupService.Tx(tx.tx)

api/datastore/test_data/output_24_to_latest.json

@@ -631,6 +631,7 @@
     "LogoURL": "",
     "OAuthSettings": {
       "AccessTokenURI": "",
+      "AuthStyle": 0,
       "AuthorizationURI": "",
       "ClientID": "",
       "DefaultTeamID": 0,
@@ -677,6 +678,7 @@
             "Architecture": "",
             "BridgeNfIp6tables": false,
             "BridgeNfIptables": false,
+            "CDISpecDirs": null,
             "CPUSet": false,
             "CPUShares": false,
             "CgroupDriver": "",
@@ -939,6 +941,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.20.2\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/client/client.go

@@ -13,7 +13,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 
-	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	"github.com/segmentio/encoding/json"
 )
@@ -93,11 +93,17 @@ func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*cli
 		return nil, err
 	}
 
-	return client.NewClientWithOpts(
+	opts := []client.Opt{
 		client.WithHost(endpoint.URL),
 		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
-	)
+	}
+
+	if nnTransport, ok := httpCli.Transport.(*NodeNameTransport); ok && nnTransport.TLSClientConfig != nil {
+		opts = append(opts, client.WithScheme("https"))
+	}
+
+	return client.NewClientWithOpts(opts...)
 }
 
 func createAgentClient(endpoint *portainer.Endpoint, endpointURL string, signatureService portainer.DigitalSignatureService, nodeName string, timeout *time.Duration) (*client.Client, error) {
@@ -159,7 +165,7 @@ func (t *NodeNameTransport) RoundTrip(req *http.Request) (*http.Response, error)
 	resp.Body = io.NopCloser(bytes.NewReader(body))
 
 	var rs []struct {
-		types.ImageSummary
+		image.Summary
 		Portainer struct {
 			Agent struct {
 				NodeName string

api/docker/container.go

@@ -119,7 +119,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 		for _, network := range container.NetworkSettings.Networks {
 			cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
 		}
-		cli.ContainerStart(ctx, containerId, types.ContainerStartOptions{})
+		cli.ContainerStart(ctx, containerId, dockercontainer.StartOptions{})
 	})
 
 	log.Debug().Str("container", strings.Split(container.Name, "/")[1]).Msg("starting to create a new container")
@@ -135,7 +135,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 	c.sr.push(func() {
 		log.Debug().Str("container_id", create.ID).Msg("removing the new container")
 		cli.ContainerStop(ctx, create.ID, dockercontainer.StopOptions{})
-		cli.ContainerRemove(ctx, create.ID, types.ContainerRemoveOptions{})
+		cli.ContainerRemove(ctx, create.ID, dockercontainer.RemoveOptions{})
 	})
 
 	if err != nil {
@@ -164,14 +164,14 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
 
 	// 8. start the new container
 	log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
-	err = cli.ContainerStart(ctx, newContainerId, types.ContainerStartOptions{})
+	err = cli.ContainerStart(ctx, newContainerId, dockercontainer.StartOptions{})
 	if err != nil {
 		return nil, errors.Wrap(err, "start container error")
 	}
 
 	// 9. delete the old container
 	log.Debug().Str("container_id", containerId).Msg("starting to remove the old container")
-	_ = cli.ContainerRemove(ctx, containerId, types.ContainerRemoveOptions{})
+	_ = cli.ContainerRemove(ctx, containerId, dockercontainer.RemoveOptions{})
 
 	c.sr.disable()
 

api/docker/images/status.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	portainer "github.com/portainer/portainer/api"
 	consts "github.com/portainer/portainer/api/docker/consts"
@@ -157,7 +158,7 @@ func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string,
 		return Error, nil
 	}
 
-	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
+	containers, err := cli.ContainerList(ctx, container.ListOptions{
 		All:     true,
 		Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+serviceID)),
 	})

api/docker/snapshot.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	_container "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
@@ -147,7 +148,7 @@ func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Clien
 }
 
 func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
-	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
 	if err != nil {
 		return err
 	}

api/filesystem/filesystem.go

@@ -934,7 +934,7 @@ func FileExists(filePath string) (bool, error) {
 func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
 	// 1. Backup the source directory to a different folder
 	backupDir := fmt.Sprintf("%s-%s", filepath.Dir(originalPath), "backup")
-	err := MoveDirectory(originalPath, backupDir)
+	err := MoveDirectory(originalPath, backupDir, false)
 	if err != nil {
 		return fmt.Errorf("failed to backup source directory: %w", err)
 	}
@@ -973,14 +973,14 @@ func restoreBackup(src, backupDir string) error {
 		return fmt.Errorf("failed to delete destination directory: %w", err)
 	}
 
-	err = MoveDirectory(backupDir, src)
+	err = MoveDirectory(backupDir, src, false)
 	if err != nil {
 		return fmt.Errorf("failed to restore backup directory: %w", err)
 	}
 	return nil
 }
 
-func MoveDirectory(originalPath, newPath string) error {
+func MoveDirectory(originalPath, newPath string, overwriteTargetPath bool) error {
 	if _, err := os.Stat(originalPath); err != nil {
 		return err
 	}
@@ -991,7 +991,13 @@ func MoveDirectory(originalPath, newPath string) error {
 	}
 
 	if alreadyExists {
-		return errors.New("Target path already exists")
+		if !overwriteTargetPath {
+			return fmt.Errorf("Target path already exists")
+		}
+
+		if err = os.RemoveAll(newPath); err != nil {
+			return fmt.Errorf("failed to overwrite path %s: %s", newPath, err.Error())
+		}
 	}
 
 	return os.Rename(originalPath, newPath)

api/filesystem/filesystem_move_test.go

@@ -16,7 +16,7 @@ func Test_movePath_shouldFailIfSourceDirDoesNotExist(t *testing.T) {
 	file1 := addFile(destinationDir, "dir", "file")
 	file2 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when source path is missing")
 	assert.FileExists(t, file1, "destination dir contents should remain")
 	assert.FileExists(t, file2, "destination dir contents should remain")
@@ -30,7 +30,7 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	file3 := addFile(destinationDir, "dir", "file")
 	file4 := addFile(destinationDir, "file")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.Error(t, err, "move directory should fail when destination directory already exists")
 	assert.FileExists(t, file1, "source dir contents should remain")
 	assert.FileExists(t, file2, "source dir contents should remain")
@@ -38,6 +38,22 @@ func Test_movePath_shouldFailIfDestinationDirExists(t *testing.T) {
 	assert.FileExists(t, file4, "destination dir contents should remain")
 }
 
+func Test_movePath_succesIfOverwriteSetWhenDestinationDirExists(t *testing.T) {
+	sourceDir := t.TempDir()
+	file1 := addFile(sourceDir, "dir", "file")
+	file2 := addFile(sourceDir, "file")
+	destinationDir := t.TempDir()
+	file3 := addFile(destinationDir, "dir", "file")
+	file4 := addFile(destinationDir, "file")
+
+	err := MoveDirectory(sourceDir, destinationDir, true)
+	assert.NoError(t, err)
+	assert.NoFileExists(t, file1, "source dir contents should be moved")
+	assert.NoFileExists(t, file2, "source dir contents should be moved")
+	assert.FileExists(t, file3, "destination dir contents should remain")
+	assert.FileExists(t, file4, "destination dir contents should remain")
+}
+
 func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T) {
 	tmp := t.TempDir()
 	sourceDir := path.Join(tmp, "source")
@@ -46,7 +62,7 @@ func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T)
 	file2 := addFile(sourceDir, "file")
 	destinationDir := path.Join(tmp, "destination")
 
-	err := MoveDirectory(sourceDir, destinationDir)
+	err := MoveDirectory(sourceDir, destinationDir, false)
 	assert.NoError(t, err)
 	assert.NoFileExists(t, file1, "source dir contents should be moved")
 	assert.NoFileExists(t, file2, "source dir contents should be moved")

api/git/backup.go

@@ -38,7 +38,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 		}
 	}
 
-	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath)
+	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath, true)
 	if err != nil {
 		return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
 	}
@@ -48,7 +48,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
 	err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password, options.TLSSkipVerify)
 	if err != nil {
 		cleanUp = false
-		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath)
+		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath, false)
 		if restoreError != nil {
 			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
 		}

api/http/csrf/csrf.go

@@ -21,7 +21,11 @@ func WithProtect(handler http.Handler) (http.Handler, error) {
 		return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
 	}
 
-	handler = gorillacsrf.Protect([]byte(token), gorillacsrf.Path("/"))(handler)
+	handler = gorillacsrf.Protect(
+		[]byte(token),
+		gorillacsrf.Path("/"),
+		gorillacsrf.Secure(false),
+	)(handler)
 
 	return withSkipCSRF(handler), nil
 }

api/http/handler/auth/authenticate.go

@@ -75,7 +75,12 @@ func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *h
 		if settings.AuthenticationMethod == portainer.AuthenticationInternal ||
 			settings.AuthenticationMethod == portainer.AuthenticationOAuth ||
 			(settings.AuthenticationMethod == portainer.AuthenticationLDAP && !settings.LDAPSettings.AutoCreateUsers) {
-			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+			// avoid username enumeration timing attack by creating a fake user
+			// https://en.wikipedia.org/wiki/Timing_attack
+			user = &portainer.User{
+				Username: "portainer-fake-username",
+				Password: "$2a$10$abcdefghijklmnopqrstuvwx..ABCDEFGHIJKLMNOPQRSTUVWXYZ12", // fake but valid format bcrypt hash
+			}
 		}
 	}
 
@@ -112,7 +117,11 @@ func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portai
 func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
 	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
 	if err != nil {
-		return httperror.Forbidden("Only initial admin is allowed to login without oauth", err)
+		if errors.Is(err, httperrors.ErrUnauthorized) {
+			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
+		}
+
+		return httperror.InternalServerError("Unable to authenticate user against LDAP", err)
 	}
 
 	if user == nil {

api/http/handler/docker/images/images_list.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 )
 
 type ImageResponse struct {
@@ -63,7 +64,7 @@ func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *http
 
 	imageUsageSet := set.Set[string]{}
 	if withUsage {
-		containers, err := cli.ContainerList(r.Context(), types.ContainerListOptions{})
+		containers, err := cli.ContainerList(r.Context(), container.ListOptions{})
 		if err != nil {
 			return httperror.InternalServerError("Unable to retrieve Docker containers", err)
 		}

api/http/handler/edgestacks/edgestack_status_update.go

@@ -135,6 +135,11 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht
 }
 
 func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeStack, deploymentStatus portainer.EdgeStackDeploymentStatus) {
+	if deploymentStatus.Type == portainer.EdgeStackStatusRemoved {
+		delete(stack.Status, environmentId)
+		return
+	}
+
 	environmentStatus, ok := stack.Status[environmentId]
 	if !ok {
 		environmentStatus = portainer.EdgeStackStatus{

api/http/handler/endpoints/endpoint_delete.go

@@ -179,6 +179,12 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 		}
 	}
 
+	// delete the pending actions
+	err = tx.PendingActions().DeleteByEndpointID(endpoint.ID)
+	if err != nil {
+		log.Warn().Err(err).Int("endpointId", int(endpoint.ID)).Msgf("Unable to delete pending actions")
+	}
+
 	err = tx.Endpoint().DeleteEndpoint(portainer.EndpointID(endpointID))
 	if err != nil {
 		return httperror.InternalServerError("Unable to delete the environment from the database", err)

api/http/handler/endpoints/endpoint_force_update_service.go

@@ -12,8 +12,8 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
-	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 )
 
@@ -39,7 +39,7 @@ func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
 // @produce json
 // @param id path int true "endpoint identifier"
 // @param body body forceUpdateServicePayload true "details"
-// @success 200 {object} dockertypes.ServiceUpdateResponse "Success"
+// @success 200 {object} swarm.ServiceUpdateResponse "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 404 "endpoint not found"
@@ -94,7 +94,7 @@ func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *htt
 	go func() {
 		images.EvictImageStatus(payload.ServiceID)
 		images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
-		containers, _ := dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{
+		containers, _ := dockerClient.ContainerList(context.TODO(), container.ListOptions{
 			All:     true,
 			Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+payload.ServiceID)),
 		})

api/http/handler/endpoints/filter.go

@@ -622,6 +622,7 @@ func getEdgeStackStatusParam(r *http.Request) (*portainer.EdgeStackStatusType, e
 		portainer.EdgeStackStatusRunning,
 		portainer.EdgeStackStatusDeploying,
 		portainer.EdgeStackStatusRemoving,
+		portainer.EdgeStackStatusCompleted,
 	}, edgeStackStatus) {
 		return nil, errors.New("invalid edgeStackStatus parameter")
 	}

api/http/handler/handler.go

@@ -85,7 +85,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.20.0
+// @version 2.20.2
 // @description.markdown api-description.md
 // @termsOfService
 

api/http/handler/helm/handler.go

@@ -38,19 +38,20 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 		kubeClusterAccessService: kubeClusterAccessService,
 	}
 
-	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
+	h.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"),
+		bouncer.AuthenticatedAccess)
 
 	// `helm list -o json`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmList))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmList)).Methods(http.MethodGet)
 
 	// `helm delete RELEASE_NAME`
 	h.Handle("/{id}/kubernetes/helm/{release}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmDelete))).Methods(http.MethodDelete)
+		httperror.LoggerHandler(h.helmDelete)).Methods(http.MethodDelete)
 
 	// `helm install [NAME] [CHART] flags`
 	h.Handle("/{id}/kubernetes/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmInstall))).Methods(http.MethodPost)
+		httperror.LoggerHandler(h.helmInstall)).Methods(http.MethodPost)
 
 	// Deprecated
 	h.Handle("/{id}/kubernetes/helm/repositories",
@@ -69,12 +70,14 @@ func NewTemplateHandler(bouncer security.BouncerService, helmPackageManager libh
 		requestBouncer:     bouncer,
 	}
 
+	h.Use(bouncer.AuthenticatedAccess)
+
 	h.Handle("/templates/helm",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmRepoSearch))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmRepoSearch)).Methods(http.MethodGet)
 
 	// helm show [COMMAND] [CHART] [REPO] flags
 	h.Handle("/templates/helm/{command:chart|values|readme}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmShow))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.helmShow)).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/helm/helm_install.go

@@ -61,8 +61,7 @@ func (handler *Handler) helmInstall(w http.ResponseWriter, r *http.Request) *htt
 		return httperror.InternalServerError("Unable to install a chart", err)
 	}
 
-	w.WriteHeader(http.StatusCreated)
-	return response.JSON(w, release)
+	return response.JSONWithStatus(w, release, http.StatusCreated)
 }
 
 func (p *installChartPayload) Validate(_ *http.Request) error {

api/http/handler/hostmanagement/openamt/amtrpc.go

@@ -155,7 +155,7 @@ func pullImage(ctx context.Context, docker *client.Client, imageName string) err
 // runContainer should be used to run a short command that returns information to stdout
 // TODO: add k8s support
 func runContainer(ctx context.Context, docker *client.Client, imageName, containerName string, cmdLine []string) (output string, err error) {
-	opts := types.ContainerListOptions{All: true}
+	opts := container.ListOptions{All: true}
 	opts.Filters = filters.NewArgs()
 	opts.Filters.Add("name", containerName)
 	existingContainers, err := docker.ContainerList(ctx, opts)
@@ -170,7 +170,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 	}
 
 	if len(existingContainers) > 0 {
-		err = docker.ContainerRemove(ctx, existingContainers[0].ID, types.ContainerRemoveOptions{Force: true})
+		err = docker.ContainerRemove(ctx, existingContainers[0].ID, container.RemoveOptions{Force: true})
 		if err != nil {
 			log.Error().
 				Str("image_name", imageName).
@@ -211,7 +211,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 		return "", err
 	}
 
-	err = docker.ContainerStart(ctx, created.ID, types.ContainerStartOptions{})
+	err = docker.ContainerStart(ctx, created.ID, container.StartOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).
@@ -243,14 +243,14 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain
 
 	log.Debug().Int64("status", statusCode).Msg("container wait status")
 
-	out, err := docker.ContainerLogs(ctx, created.ID, types.ContainerLogsOptions{ShowStdout: true})
+	out, err := docker.ContainerLogs(ctx, created.ID, container.LogsOptions{ShowStdout: true})
 	if err != nil {
 		log.Error().Err(err).Str("image_name", imageName).Str("container_name", containerName).Msg("getting container log")
 
 		return "", err
 	}
 
-	err = docker.ContainerRemove(ctx, created.ID, types.ContainerRemoveOptions{})
+	err = docker.ContainerRemove(ctx, created.ID, container.RemoveOptions{})
 	if err != nil {
 		log.Error().
 			Str("image_name", imageName).

api/http/handler/registries/registry_delete.go

@@ -8,6 +8,7 @@ import (
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/pendingactions"
+	"github.com/portainer/portainer/api/pendingactions/actions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -91,7 +92,7 @@ func (handler *Handler) deleteKubernetesSecrets(registry *portainer.Registry) er
 			if len(failedNamespaces) > 0 {
 				handler.PendingActionsService.Create(portainer.PendingActions{
 					EndpointID: endpointId,
-					Action:     pendingactions.DeletePortainerK8sRegistrySecrets,
+					Action:     actions.DeletePortainerK8sRegistrySecrets,
 
 					// When extracting the data, this is the type we need to pull out
 					// i.e. pendingactions.DeletePortainerK8sRegistrySecretsData

api/http/handler/settings/settings_update.go

@@ -13,6 +13,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"golang.org/x/oauth2"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
@@ -95,6 +96,11 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 		}
 	}
 
+	if payload.OAuthSettings != nil {
+		if payload.OAuthSettings.AuthStyle < oauth2.AuthStyleAutoDetect || payload.OAuthSettings.AuthStyle > oauth2.AuthStyleInHeader {
+			return errors.New("Invalid OAuth AuthStyle")
+		}
+	}
 	return nil
 }
 
@@ -225,6 +231,7 @@ func (handler *Handler) updateSettings(tx dataservices.DataStoreTx, payload sett
 		settings.OAuthSettings = *payload.OAuthSettings
 		settings.OAuthSettings.ClientSecret = clientSecret
 		settings.OAuthSettings.KubeSecretKey = kubeSecret
+		settings.OAuthSettings.AuthStyle = payload.OAuthSettings.AuthStyle
 	}
 
 	if payload.EnableEdgeComputeFeatures != nil {

api/http/handler/stacks/handler.go

@@ -21,6 +21,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 )
@@ -190,7 +191,7 @@ func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoin
 		}
 	}
 
-	containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{All: true})
 	if err != nil {
 		return false, err
 	}

api/http/handler/stacks/stack_list.go

@@ -23,6 +23,7 @@ type stackListOperationFilters struct {
 // @description List all stacks based on the current user authorizations.
 // @description Will return all stacks if using an administrator account otherwise it
 // @description will only return the list of stacks the user have access to.
+// @description Limited stacks will not be returned by this endpoint.
 // @description **Access policy**: authenticated
 // @tags stacks
 // @security ApiKeyAuth
@@ -91,25 +92,55 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 	return response.JSON(w, stacks)
 }
 
+// filterStacks refines a collection of Stack instances using specified criteria.
+// This function examines the provided filters: EndpointID, SwarmID, and IncludeOrphanedStacks.
+// - If both EndpointID is zero and SwarmID is an empty string, the function directly returns the original stack list without any modifications.
+// - If either filter is specified, it proceeds to selectively include stacks that match the criteria.
+
+// Key Points on Business Logic:
+// 1. Determining Inclusion of Orphaned Stacks:
+//    - The decision to include orphaned stacks is influenced by the user's role and usually set by the client (UI).
+//    - Administrators or environment administrators can include orphaned stacks by setting IncludeOrphanedStacks to true, reflecting their broader access rights.
+//    - For non-administrative users, this is typically set to false, limiting their visibility to only stacks within their purview.
+
+// 2. Inclusion Criteria for Orphaned Stacks:
+//    - When IncludeOrphanedStacks is true and an EndpointID is specified (not zero), the function selects:
+//      a) Stacks linked to the specified EndpointID.
+//      b) Orphaned stacks that don't have a naming conflict with any stack associated with the EndpointID.
+//    - This approach is designed to avoid name conflicts within Docker Compose, which restricts the creation of multiple stacks with the same name.
+
+// 3. Type Matching for Orphaned Stacks:
+//    - The function ensures that orphaned stacks are compatible with the environment's stack type (compose or swarm).
+//    - It filters out orphaned swarm stacks in Docker standalone environments
+//    - It filters out orphaned standalone stack in Docker swarm environments
+//    - This ensures that re-association respects the constraints of the environment and stack type.
+
+// The outcome is a new list of stacks that align with these filtering and business logic criteria.
 func filterStacks(stacks []portainer.Stack, filters *stackListOperationFilters, endpoints []portainer.Endpoint) []portainer.Stack {
 	if filters.EndpointID == 0 && filters.SwarmID == "" {
 		return stacks
 	}
 
 	filteredStacks := make([]portainer.Stack, 0, len(stacks))
+	uniqueStackNames := make(map[string]struct{})
 	for _, stack := range stacks {
-		if filters.IncludeOrphanedStacks && isOrphanedStack(stack, endpoints) {
-			if (stack.Type == portainer.DockerComposeStack && filters.SwarmID == "") || (stack.Type == portainer.DockerSwarmStack && filters.SwarmID != "") {
-				filteredStacks = append(filteredStacks, stack)
-			}
-			continue
-		}
-
 		if stack.Type == portainer.DockerComposeStack && stack.EndpointID == portainer.EndpointID(filters.EndpointID) {
 			filteredStacks = append(filteredStacks, stack)
+			uniqueStackNames[stack.Name] = struct{}{}
 		}
 		if stack.Type == portainer.DockerSwarmStack && stack.SwarmID == filters.SwarmID {
 			filteredStacks = append(filteredStacks, stack)
+			uniqueStackNames[stack.Name] = struct{}{}
+		}
+	}
+
+	for _, stack := range stacks {
+		if filters.IncludeOrphanedStacks && isOrphanedStack(stack, endpoints) {
+			if (stack.Type == portainer.DockerComposeStack && filters.SwarmID == "") || (stack.Type == portainer.DockerSwarmStack && filters.SwarmID != "") {
+				if _, exists := uniqueStackNames[stack.Name]; !exists {
+					filteredStacks = append(filteredStacks, stack)
+				}
+			}
 		}
 	}
 

api/http/handler/stacks/stack_list_test.go

@@ -0,0 +1,74 @@
+package stacks
+
+import (
+	"sort"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterStacks(t *testing.T) {
+	t.Run("filter stacks against particular endpoint and all orphaned stacks", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: true}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}, {ID: 2}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+
+	t.Run("filter unique stacks against particular endpoint and all orphaned stacks and an orphaned stack has the same name with normal stack", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+			{ID: 4, EndpointID: 4, Name: "normal_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: true}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}, {ID: 2}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+
+	t.Run("only filter stacks against particular endpoint and no orphaned stacks", func(t *testing.T) {
+		stacks := []portainer.Stack{
+			{ID: 1, EndpointID: 3, Name: "normal_stack", Type: portainer.DockerComposeStack},
+			{ID: 2, EndpointID: 4, Name: "orphaned_stack", Type: portainer.DockerComposeStack},
+			{ID: 3, EndpointID: 5, Name: "other_stack", Type: portainer.DockerComposeStack},
+			{ID: 4, EndpointID: 4, Name: "normal_stack", Type: portainer.DockerComposeStack},
+		}
+		filters := &stackListOperationFilters{EndpointID: 3, IncludeOrphanedStacks: false}
+		endpoints := []portainer.Endpoint{{ID: 3}, {ID: 5}}
+
+		expectStacks := []portainer.Stack{{ID: 1}}
+		actualStacks := filterStacks(stacks, filters, endpoints)
+
+		isEqualStacks(t, expectStacks, actualStacks)
+	})
+}
+
+func isEqualStacks(t *testing.T, expectStacks, actualStacks []portainer.Stack) {
+	expectStackIDs := make([]int, len(expectStacks))
+	for i, stack := range expectStacks {
+		expectStackIDs[i] = int(stack.ID)
+	}
+	sort.Ints(expectStackIDs)
+
+	actualStackIDs := make([]int, len(actualStacks))
+	for i, stack := range actualStacks {
+		actualStackIDs[i] = int(stack.ID)
+	}
+	sort.Ints(actualStackIDs)
+
+	assert.Equal(t, expectStackIDs, actualStackIDs)
+}

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -27,6 +27,8 @@ type stackGitRedployPayload struct {
 	Prune                    bool
 	// Force a pulling to current image with the original tag though the image is already the latest
 	PullImage bool `example:"false"`
+
+	StackName string
 }
 
 func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
@@ -44,7 +46,7 @@ func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
 // @produce json
 // @param id path int true "Stack identifier"
 // @param endpointId query int false "Stacks created before version 1.18.0 might not have an associated environment(endpoint) identifier. Use this optional parameter to set the environment(endpoint) identifier used by the stack."
-// @param body body stackGitRedployPayload true "Git configs for pull and redeploy a stack"
+// @param body body stackGitRedployPayload true "Git configs for pull and redeploy of a stack. **StackName** may only be populated for Kuberenetes stacks, and if specified with a blank string, it will be set to blank"
 // @success 200 {object} portainer.Stack "Success"
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
@@ -136,6 +138,10 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
+	if stack.Type == portainer.KubernetesStack {
+		stack.Name = payload.StackName
+	}
+
 	repositoryUsername := ""
 	repositoryPassword := ""
 	if payload.RepositoryAuthentication {

api/http/handler/users/user_create_access_token.go

@@ -2,6 +2,7 @@ package users
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
@@ -20,9 +21,6 @@ type userAccessTokenCreatePayload struct {
 }
 
 func (payload *userAccessTokenCreatePayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.Password) {
-		return errors.New("invalid password: cannot be empty")
-	}
 	if govalidator.IsNull(payload.Description) {
 		return errors.New("invalid description: cannot be empty")
 	}
@@ -44,6 +42,7 @@ type accessTokenResponse struct {
 // @summary Generate an API key for a user
 // @description Generates an API key for a user.
 // @description Only the calling user can generate a token for themselves.
+// @description Password is required only for internal authentication.
 // @description **Access policy**: restricted
 // @tags users
 // @security jwt
@@ -51,7 +50,7 @@ type accessTokenResponse struct {
 // @produce json
 // @param id path int true "User identifier"
 // @param body body userAccessTokenCreatePayload true "details"
-// @success 201 {object} accessTokenResponse "Created"
+// @success 200 {object} accessTokenResponse "Created"
 // @failure 400 "Invalid request"
 // @failure 401 "Unauthorized"
 // @failure 403 "Permission denied"
@@ -60,8 +59,13 @@ type accessTokenResponse struct {
 // @router /users/{id}/tokens [post]
 func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	// specifically require Cookie auth for this endpoint since API-Key based auth is not supported
-	if jwt, _ := handler.bouncer.CookieAuthLookup(r); jwt == nil {
-		return httperror.Unauthorized("Auth not supported", errors.New("Cookie Authentication required"))
+	jwt, _ := handler.bouncer.CookieAuthLookup(r)
+	if jwt == nil {
+		jwt, _ = handler.bouncer.JWTAuthLookup(r)
+	}
+
+	if jwt == nil {
+		return httperror.Unauthorized("Auth not supported", errors.New("Authentication required"))
 	}
 
 	var payload userAccessTokenCreatePayload
@@ -89,9 +93,21 @@ func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Req
 		return httperror.InternalServerError("Unable to find a user with the specified identifier inside the database", err)
 	}
 
-	err = handler.CryptoService.CompareHashAndData(user.Password, payload.Password)
+	internalAuth, err := handler.usesInternalAuthentication(portainer.UserID(userID))
 	if err != nil {
-		return httperror.Forbidden("Current password doesn't match", errors.New("Current password does not match the password provided. Please try again"))
+		return httperror.InternalServerError("Unable to determine the authentication method", err)
+	}
+
+	if internalAuth {
+		// Internal auth requires the password field and must not be empty
+		if govalidator.IsNull(payload.Password) {
+			return httperror.BadRequest("Invalid request payload", errors.New("invalid password: cannot be empty"))
+		}
+
+		err = handler.CryptoService.CompareHashAndData(user.Password, payload.Password)
+		if err != nil {
+			return httperror.Forbidden("Current password doesn't match", errors.New("Current password does not match the password provided. Please try again"))
+		}
 	}
 
 	rawAPIKey, apiKey, err := handler.apiKeyService.GenerateApiKey(*user, payload.Description)
@@ -99,6 +115,20 @@ func (handler *Handler) userCreateAccessToken(w http.ResponseWriter, r *http.Req
 		return httperror.InternalServerError("Internal Server Error", err)
 	}
 
-	w.WriteHeader(http.StatusCreated)
-	return response.JSON(w, accessTokenResponse{rawAPIKey, *apiKey})
+	return response.JSONWithStatus(w, accessTokenResponse{rawAPIKey, *apiKey}, http.StatusCreated)
+}
+
+func (handler *Handler) usesInternalAuthentication(userid portainer.UserID) (bool, error) {
+	// userid 1 is the admin user and always uses internal auth
+	if userid == 1 {
+		return true, nil
+	}
+
+	// otherwise determine the auth method from the settings
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return false, fmt.Errorf("unable to retrieve the settings from the database: %w", err)
+	}
+
+	return settings.AuthenticationMethod == portainer.AuthenticationInternal, nil
 }

api/http/handler/users/user_create_access_token_test.go

@@ -107,7 +107,7 @@ func Test_userCreateAccessToken(t *testing.T) {
 
 		body, err := io.ReadAll(rr.Body)
 		is.NoError(err, "ReadAll should not return error")
-		is.Equal(`{"message":"Auth not supported","details":"Cookie Authentication required"}`, string(body))
+		is.Equal(`{"message":"Auth not supported","details":"Authentication required"}`, string(body))
 	})
 }
 

api/http/handler/webhooks/webhook_list.go

@@ -22,7 +22,7 @@ type webhookListOperationFilters struct {
 // @tags webhooks
 // @accept json
 // @produce json
-// @param filters query webhookListOperationFilters false "Filters"
+// @param filters query string false "Filters (json-string)" example({"EndpointID":1,"ResourceID":"abc12345-abcd-2345-ab12-58005b4a0260"})
 // @success 200 {array} portainer.Webhook
 // @failure 400
 // @failure 500

api/http/security/bouncer.go

@@ -1,6 +1,7 @@
 package security
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 	"time"
@@ -10,6 +11,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/rs/zerolog/log"
 
 	"github.com/pkg/errors"
 )
@@ -27,6 +29,7 @@ type (
 		AuthorizedEdgeEndpointOperation(*http.Request, *portainer.Endpoint) error
 		TrustedEdgeEnvironmentAccess(dataservices.DataStoreTx, *portainer.Endpoint) error
 		CookieAuthLookup(*http.Request) (*portainer.TokenData, error)
+		JWTAuthLookup(*http.Request) (*portainer.TokenData, error)
 	}
 
 	// RequestBouncer represents an entity that manages API request accesses
@@ -280,7 +283,7 @@ func (bouncer *RequestBouncer) mwAuthenticateFirst(tokenLookups []tokenLookup, n
 		for _, lookup := range tokenLookups {
 			resultToken, err := lookup(r)
 			if err != nil {
-				httperror.WriteError(w, http.StatusUnauthorized, "Invalid API key", httperrors.ErrUnauthorized)
+				httperror.WriteError(w, http.StatusUnauthorized, "Invalid JWT token", httperrors.ErrUnauthorized)
 				return
 			}
 
@@ -316,7 +319,7 @@ func (bouncer *RequestBouncer) CookieAuthLookup(r *http.Request) (*portainer.Tok
 
 	tokenData, err := bouncer.jwtService.ParseAndVerifyToken(token)
 	if err != nil {
-		return nil, ErrInvalidKey
+		return nil, err
 	}
 
 	return tokenData, nil
@@ -332,7 +335,7 @@ func (bouncer *RequestBouncer) JWTAuthLookup(r *http.Request) (*portainer.TokenD
 
 	tokenData, err := bouncer.jwtService.ParseAndVerifyToken(token)
 	if err != nil {
-		return nil, ErrInvalidKey
+		return nil, err
 	}
 
 	return tokenData, nil
@@ -366,7 +369,8 @@ func (bouncer *RequestBouncer) apiKeyLookup(r *http.Request) (*portainer.TokenDa
 		Role:     user.Role,
 	}
 	if _, _, err := bouncer.jwtService.GenerateToken(tokenData); err != nil {
-		return nil, ErrInvalidKey
+		log.Debug().Err(err).Msg("Failed to generate token")
+		return nil, fmt.Errorf("failed to generate token")
 	}
 
 	if now := time.Now().UTC().Unix(); now-apiKey.LastUsed > 60 { // [seconds]

api/http/server.go

@@ -61,7 +61,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 	edgestackservice "github.com/portainer/portainer/api/internal/edge/edgestacks"
-	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/internal/upgrade"
 	k8s "github.com/portainer/portainer/api/kubernetes"
@@ -382,7 +381,8 @@ func (server *Server) Start() error {
 
 	go shutdown(server.ShutdownCtx, httpsServer)
 
-	go snapshot.NewBackgroundSnapshotter(server.DataStore, server.ReverseTunnelService)
+	// Temporarily disable for EE-6905 until we have a solution for the snapshotter
+	// go snapshot.NewBackgroundSnapshotter(server.DataStore, server.ReverseTunnelService)
 
 	return httpsServer.ListenAndServeTLS("", "")
 }

api/internal/testhelpers/request_bouncer.go

@@ -54,6 +54,10 @@ func (testRequestBouncer) CookieAuthLookup(r *http.Request) (*portainer.TokenDat
 	return nil, nil
 }
 
+func (testRequestBouncer) JWTAuthLookup(r *http.Request) (*portainer.TokenData, error) {
+	return nil, nil
+}
+
 // AddTestSecurityCookie adds a security cookie to the request
 func AddTestSecurityCookie(r *http.Request, jwt string) {
 	r.AddCookie(&http.Cookie{

api/kubernetes.go

@@ -3,10 +3,11 @@ package portainer
 func KubernetesDefault() KubernetesData {
 	return KubernetesData{
 		Configuration: KubernetesConfiguration{
-			UseLoadBalancer:  false,
-			UseServerMetrics: false,
-			StorageClasses:   []KubernetesStorageClassConfig{},
-			IngressClasses:   []KubernetesIngressClassConfig{},
+			UseLoadBalancer:          false,
+			UseServerMetrics:         false,
+			EnableResourceOverCommit: true,
+			StorageClasses:           []KubernetesStorageClassConfig{},
+			IngressClasses:           []KubernetesIngressClassConfig{},
 		},
 		Snapshots: []KubernetesSnapshot{},
 	}

api/kubernetes/cli/client.go

@@ -10,6 +10,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 
 	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
@@ -80,22 +81,31 @@ func (factory *ClientFactory) RemoveKubeClient(endpointID portainer.EndpointID)
 // GetKubeClient checks if an existing client is already registered for the environment(endpoint) and returns it if one is found.
 // If no client is registered, it will create a new client, register it, and returns it.
 func (factory *ClientFactory) GetKubeClient(endpoint *portainer.Endpoint) (*KubeClient, error) {
+	factory.mu.Lock()
+	key := strconv.Itoa(int(endpoint.ID))
+	if client, ok := factory.endpointClients[key]; ok {
+		factory.mu.Unlock()
+		return client, nil
+	}
+	factory.mu.Unlock()
+
+	// EE-6901: Do not lock
+	client, err := factory.createCachedAdminKubeClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	factory.mu.Lock()
 	defer factory.mu.Unlock()
 
-	key := strconv.Itoa(int(endpoint.ID))
-	client, ok := factory.endpointClients[key]
-	if !ok {
-		var err error
-
-		client, err = factory.createCachedAdminKubeClient(endpoint)
-		if err != nil {
-			return nil, err
-		}
-
-		factory.endpointClients[key] = client
+	// The lock was released before the client was created,
+	// so we need to check again
+	if c, ok := factory.endpointClients[key]; ok {
+		return c, nil
 	}
 
+	factory.endpointClients[key] = client
+
 	return client, nil
 }
 
@@ -277,106 +287,111 @@ func buildLocalConfig() (*rest.Config, error) {
 	return config, nil
 }
 
-func (factory *ClientFactory) MigrateEndpointIngresses(e *portainer.Endpoint) error {
-	// classes is a list of controllers which have been manually added to the
-	// cluster setup view. These need to all be allowed globally, but then
-	// blocked in specific namespaces which they were not previously allowed in.
-	classes := e.Kubernetes.Configuration.IngressClasses
-
-	// We need a kube client to gather namespace level permissions. In pre-2.16
-	// versions of portainer, the namespace level permissions were stored by
-	// creating an actual ingress rule in the cluster with a particular
-	// annotation indicating that it's name (the class name) should be allowed.
-	cli, err := factory.GetKubeClient(e)
-	if err != nil {
-		return err
-	}
-
-	detected, err := cli.GetIngressControllers()
-	if err != nil {
-		return err
-	}
-
-	// newControllers is a set of all currently detected controllers.
-	newControllers := make(map[string]struct{})
-	for _, controller := range detected {
-		newControllers[controller.ClassName] = struct{}{}
-	}
-
-	namespaces, err := cli.GetNamespaces()
-	if err != nil {
-		return err
-	}
-
-	// Set of namespaces, if any, in which "allow none" should be true.
-	allow := make(map[string]map[string]struct{})
-	for _, c := range classes {
-		allow[c.Name] = make(map[string]struct{})
-	}
-	allow["none"] = make(map[string]struct{})
-
-	for namespace := range namespaces {
-		// Compare old annotations with currently detected controllers.
-		ingresses, err := cli.GetIngresses(namespace)
+func (factory *ClientFactory) MigrateEndpointIngresses(e *portainer.Endpoint, datastore dataservices.DataStore, cli *KubeClient) error {
+	return datastore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		environment, err := tx.Endpoint().Endpoint(e.ID)
 		if err != nil {
-			return fmt.Errorf("failure getting ingresses during migration")
+			log.Error().Err(err).Msgf("Error retrieving environment %d", e.ID)
+			return err
 		}
-		for _, ingress := range ingresses {
-			oldController, ok := ingress.Annotations["ingress.portainer.io/ingress-type"]
-			if !ok {
-				// Skip rules without our old annotation.
-				continue
-			}
 
-			if _, ok := newControllers[oldController]; ok {
-				// Skip rules which match a detected controller.
-				// TODO: Allow this particular controller.
-				allow[oldController][ingress.Namespace] = struct{}{}
-				continue
-			}
+		// classes is a list of controllers which have been manually added to the
+		// cluster setup view. These need to all be allowed globally, but then
+		// blocked in specific namespaces which they were not previously allowed in.
+		classes := environment.Kubernetes.Configuration.IngressClasses
 
-			allow["none"][ingress.Namespace] = struct{}{}
+		// In pre-2.16 versions of portainer, the namespace level permissions were stored by
+		// creating an actual ingress rule in the cluster with a particular
+		// annotation indicating that it's name (the class name) should be allowed.
+		detected, err := cli.GetIngressControllers()
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting ingress controllers in environment %d", environment.ID)
+			return err
 		}
-	}
 
-	// Locally, disable "allow none" for namespaces not inside shouldAllowNone.
-	var newClasses []portainer.KubernetesIngressClassConfig
-	for _, c := range classes {
-		var blocked []string
+		// newControllers is a set of all currently detected controllers.
+		newControllers := make(map[string]struct{})
+		for _, controller := range detected {
+			newControllers[controller.ClassName] = struct{}{}
+		}
+
+		namespaces, err := cli.GetNamespaces()
+		if err != nil {
+			log.Error().Err(err).Msgf("Error getting namespaces in environment %d", environment.ID)
+			return err
+		}
+
+		// Set of namespaces, if any, in which "allow none" should be true.
+		allow := make(map[string]map[string]struct{})
+		for _, c := range classes {
+			allow[c.Name] = make(map[string]struct{})
+		}
+		allow["none"] = make(map[string]struct{})
+
 		for namespace := range namespaces {
-			if _, ok := allow[c.Name][namespace]; ok {
-				continue
+			// Compare old annotations with currently detected controllers.
+			ingresses, err := cli.GetIngresses(namespace)
+			if err != nil {
+				log.Error().Err(err).Msgf("Error getting ingresses in environment %d", environment.ID)
+				return err
+			}
+			for _, ingress := range ingresses {
+				oldController, ok := ingress.Annotations["ingress.portainer.io/ingress-type"]
+				if !ok {
+					// Skip rules without our old annotation.
+					continue
+				}
+
+				if _, ok := newControllers[oldController]; ok {
+					// Skip rules which match a detected controller.
+					// TODO: Allow this particular controller.
+					allow[oldController][ingress.Namespace] = struct{}{}
+					continue
+				}
+
+				allow["none"][ingress.Namespace] = struct{}{}
 			}
-			blocked = append(blocked, namespace)
 		}
 
-		newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
-			Name:              c.Name,
-			Type:              c.Type,
-			GloballyBlocked:   false,
-			BlockedNamespaces: blocked,
-		})
-	}
-
-	// Handle "none".
-	if len(allow["none"]) != 0 {
-		e.Kubernetes.Configuration.AllowNoneIngressClass = true
-		var disallowNone []string
-		for namespace := range namespaces {
-			if _, ok := allow["none"][namespace]; ok {
-				continue
+		// Locally, disable "allow none" for namespaces not inside shouldAllowNone.
+		var newClasses []portainer.KubernetesIngressClassConfig
+		for _, c := range classes {
+			var blocked []string
+			for namespace := range namespaces {
+				if _, ok := allow[c.Name][namespace]; ok {
+					continue
+				}
+				blocked = append(blocked, namespace)
 			}
-			disallowNone = append(disallowNone, namespace)
-		}
-		newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
-			Name:              "none",
-			Type:              "custom",
-			GloballyBlocked:   false,
-			BlockedNamespaces: disallowNone,
-		})
-	}
 
-	e.Kubernetes.Configuration.IngressClasses = newClasses
-	e.PostInitMigrations.MigrateIngresses = false
-	return factory.dataStore.Endpoint().UpdateEndpoint(e.ID, e)
+			newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
+				Name:              c.Name,
+				Type:              c.Type,
+				GloballyBlocked:   false,
+				BlockedNamespaces: blocked,
+			})
+		}
+
+		// Handle "none".
+		if len(allow["none"]) != 0 {
+			environment.Kubernetes.Configuration.AllowNoneIngressClass = true
+			var disallowNone []string
+			for namespace := range namespaces {
+				if _, ok := allow["none"][namespace]; ok {
+					continue
+				}
+				disallowNone = append(disallowNone, namespace)
+			}
+			newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
+				Name:              "none",
+				Type:              "custom",
+				GloballyBlocked:   false,
+				BlockedNamespaces: disallowNone,
+			})
+		}
+
+		environment.Kubernetes.Configuration.IngressClasses = newClasses
+		environment.PostInitMigrations.MigrateIngresses = false
+		return tx.Endpoint().UpdateEndpoint(environment.ID, environment)
+	})
 }

api/kubernetes/cli/ingress.go

@@ -241,7 +241,10 @@ func (kcl *KubeClient) DeleteIngresses(reqs models.K8sIngressDeleteRequests) err
 // UpdateIngress updates an existing ingress in a given namespace in a k8s endpoint.
 func (kcl *KubeClient) UpdateIngress(namespace string, info models.K8sIngressInfo) error {
 	ingressClient := kcl.cli.NetworkingV1().Ingresses(namespace)
-	var ingress netv1.Ingress
+	ingress, err := ingressClient.Get(context.Background(), info.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
 
 	ingress.Name = info.Name
 	ingress.Namespace = info.Namespace
@@ -278,6 +281,7 @@ func (kcl *KubeClient) UpdateIngress(namespace string, info models.K8sIngressInf
 		})
 	}
 
+	ingress.Spec.Rules = make([]netv1.IngressRule, 0)
 	for rule, paths := range rules {
 		ingress.Spec.Rules = append(ingress.Spec.Rules, netv1.IngressRule{
 			Host: rule,
@@ -299,6 +303,6 @@ func (kcl *KubeClient) UpdateIngress(namespace string, info models.K8sIngressInf
 		}
 	}
 
-	_, err := ingressClient.Update(context.Background(), &ingress, metav1.UpdateOptions{})
+	_, err = ingressClient.Update(context.Background(), ingress, metav1.UpdateOptions{})
 	return err
 }

api/kubernetes/cli/namespace.go

@@ -73,31 +73,30 @@ func (kcl *KubeClient) CreateNamespace(info models.K8sNamespaceDetails) error {
 	ns.Annotations = info.Annotations
 	ns.Labels = portainerLabels
 
-	resourceQuota := &v1.ResourceQuota{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "portainer-rq-" + info.Name,
-			Namespace: info.Name,
-			Labels:    portainerLabels,
-		},
-		Spec: v1.ResourceQuotaSpec{
-			Hard: v1.ResourceList{},
-		},
-	}
-
 	_, err := kcl.cli.CoreV1().Namespaces().Create(context.Background(), &ns, metav1.CreateOptions{})
 	if err != nil {
 		log.Error().
 			Err(err).
 			Str("Namespace", info.Name).
-			Interface("ResourceQuota", resourceQuota).
-			Msg("Failed to create the namespace due to a resource quota issue.")
+			Msg("Failed to create the namespace")
 		return err
 	}
 
-	if info.ResourceQuota != nil {
+	if info.ResourceQuota != nil && info.ResourceQuota.Enabled {
 		log.Info().Msgf("Creating resource quota for namespace %s", info.Name)
 		log.Debug().Msgf("Creating resource quota with details: %+v", info.ResourceQuota)
 
+		resourceQuota := &v1.ResourceQuota{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "portainer-rq-" + info.Name,
+				Namespace: info.Name,
+				Labels:    portainerLabels,
+			},
+			Spec: v1.ResourceQuotaSpec{
+				Hard: v1.ResourceList{},
+			},
+		}
+
 		if info.ResourceQuota.Enabled {
 			memory := resource.MustParse(info.ResourceQuota.Memory)
 			cpu := resource.MustParse(info.ResourceQuota.CPU)

api/ldap/ldap.go

@@ -75,7 +75,14 @@ func (*Service) AuthenticateUser(username, password string, settings *portainer.
 
 	userDN, err := searchUser(username, connection, settings.SearchSettings)
 	if err != nil {
-		return err
+		if errors.Is(err, errUserNotFound) {
+			// prevent user enumeration timing attack by attempting the bind with a fake user
+			// and whatever password was provided should definately fail
+			// https://en.wikipedia.org/wiki/Timing_attack
+			userDN = "portainer-fake-ldap-username"
+		} else {
+			return err
+		}
 	}
 
 	err = connection.Bind(userDN, password)

api/oauth/oauth.go

@@ -172,8 +172,9 @@ func getResource(token string, configuration *portainer.OAuthSettings) (map[stri
 
 func buildConfig(configuration *portainer.OAuthSettings) *oauth2.Config {
 	endpoint := oauth2.Endpoint{
-		AuthURL:  configuration.AuthorizationURI,
-		TokenURL: configuration.AccessTokenURI,
+		AuthURL:   configuration.AuthorizationURI,
+		TokenURL:  configuration.AccessTokenURI,
+		AuthStyle: configuration.AuthStyle,
 	}
 
 	return &oauth2.Config{

api/pendingactions/actions/actions.go

@@ -0,0 +1,7 @@
+package actions
+
+const (
+	CleanNAPWithOverridePolicies      = "CleanNAPWithOverridePolicies"
+	DeletePortainerK8sRegistrySecrets = "DeletePortainerK8sRegistrySecrets"
+	PostInitMigrateEnvironment        = "PostInitMigrateEnvironment"
+)

api/pendingactions/delete_registry_secrets.go

@@ -17,7 +17,7 @@ func (service *PendingActionsService) DeleteKubernetesRegistrySecrets(endpoint *
 		return nil
 	}
 
-	kubeClient, err := service.clientFactory.GetKubeClient(endpoint)
+	kubeClient, err := service.kubeFactory.GetKubeClient(endpoint)
 	if err != nil {
 		return err
 	}

api/pendingactions/pendingactions.go

@@ -7,22 +7,24 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/datastore/postinit"
+	dockerClient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/pendingactions/actions"
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	CleanNAPWithOverridePolicies      = "CleanNAPWithOverridePolicies"
-	DeletePortainerK8sRegistrySecrets = "DeletePortainerK8sRegistrySecrets"
-)
-
 type (
 	PendingActionsService struct {
 		authorizationService *authorization.Service
-		clientFactory        *kubecli.ClientFactory
+		kubeFactory          *kubecli.ClientFactory
+		dockerFactory        *dockerClient.ClientFactory
 		dataStore            dataservices.DataStore
 		shutdownCtx          context.Context
+		assetsPath           string
+		kubernetesDeployer   portainer.KubernetesDeployer
 
 		mu sync.Mutex
 	}
@@ -30,15 +32,21 @@ type (
 
 func NewService(
 	dataStore dataservices.DataStore,
-	clientFactory *kubecli.ClientFactory,
+	kubeFactory *kubecli.ClientFactory,
+	dockerFactory *dockerClient.ClientFactory,
 	authorizationService *authorization.Service,
 	shutdownCtx context.Context,
+	assetsPath string,
+	kubernetesDeployer portainer.KubernetesDeployer,
 ) *PendingActionsService {
 	return &PendingActionsService{
 		dataStore:            dataStore,
 		shutdownCtx:          shutdownCtx,
 		authorizationService: authorizationService,
-		clientFactory:        clientFactory,
+		kubeFactory:          kubeFactory,
+		dockerFactory:        dockerFactory,
+		assetsPath:           assetsPath,
+		kubernetesDeployer:   kubernetesDeployer,
 		mu:                   sync.Mutex{},
 	}
 }
@@ -57,9 +65,22 @@ func (service *PendingActionsService) Execute(id portainer.EndpointID) error {
 		return fmt.Errorf("failed to retrieve environment %d: %w", id, err)
 	}
 
-	if endpoint.Status != portainer.EndpointStatusUp {
+	isKubernetesEndpoint := endpointutils.IsKubernetesEndpoint(endpoint) && !endpointutils.IsEdgeEndpoint(endpoint)
+
+	// EndpointStatusUp is only relevant for non-Kubernetes endpoints
+	// Sometimes the endpoint is UP but the status is not updated in the database
+	if !isKubernetesEndpoint && endpoint.Status != portainer.EndpointStatusUp {
 		log.Debug().Msgf("Environment %q (id: %d) is not up", endpoint.Name, id)
-		return fmt.Errorf("environment %q (id: %d) is not up: %w", endpoint.Name, id, err)
+		return fmt.Errorf("environment %q (id: %d) is not up", endpoint.Name, id)
+	}
+
+	// For Kubernetes endpoints, we need to check if the endpoint is up by creating a kube client
+	if isKubernetesEndpoint {
+		_, err := service.kubeFactory.GetKubeClient(endpoint)
+		if err != nil {
+			log.Debug().Err(err).Msgf("Environment %q (id: %d) is not up", endpoint.Name, id)
+			return fmt.Errorf("environment %q (id: %d) is not up", endpoint.Name, id)
+		}
 	}
 
 	pendingActions, err := service.dataStore.PendingActions().ReadAll()
@@ -95,7 +116,7 @@ func (service *PendingActionsService) executePendingAction(pendingAction portain
 	}()
 
 	switch pendingAction.Action {
-	case CleanNAPWithOverridePolicies:
+	case actions.CleanNAPWithOverridePolicies:
 		if (pendingAction.ActionData == nil) || (pendingAction.ActionData.(portainer.EndpointGroupID) == 0) {
 			service.authorizationService.CleanNAPWithOverridePolicies(service.dataStore, endpoint, nil)
 			return nil
@@ -114,7 +135,7 @@ func (service *PendingActionsService) executePendingAction(pendingAction portain
 		}
 
 		return nil
-	case DeletePortainerK8sRegistrySecrets:
+	case actions.DeletePortainerK8sRegistrySecrets:
 		if pendingAction.ActionData == nil {
 			return nil
 		}
@@ -130,6 +151,22 @@ func (service *PendingActionsService) executePendingAction(pendingAction portain
 			return fmt.Errorf("failed to delete kubernetes registry secrets for environment %d: %w", endpoint.ID, err)
 		}
 
+		return nil
+
+	case actions.PostInitMigrateEnvironment:
+		postInitMigrator := postinit.NewPostInitMigrator(
+			service.kubeFactory,
+			service.dockerFactory,
+			service.dataStore,
+			service.assetsPath,
+			service.kubernetesDeployer,
+		)
+		err := postInitMigrator.MigrateEnvironment(endpoint)
+		if err != nil {
+			log.Error().Err(err).Msgf("Error running post-init migrations for edge environment %d", endpoint.ID)
+			return fmt.Errorf("failed running post-init migrations for edge environment %d: %w", endpoint.ID, err)
+		}
+
 		return nil
 	}
 

api/portainer.go

@@ -6,10 +6,13 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/api/types/system"
 	"github.com/docker/docker/api/types/volume"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	models "github.com/portainer/portainer/api/http/models/kubernetes"
 	"github.com/portainer/portainer/pkg/featureflags"
+	"golang.org/x/oauth2"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -242,8 +245,8 @@ type (
 		Containers []DockerContainerSnapshot `json:"Containers" swaggerignore:"true"`
 		Volumes    volume.ListResponse       `json:"Volumes" swaggerignore:"true"`
 		Networks   []types.NetworkResource   `json:"Networks" swaggerignore:"true"`
-		Images     []types.ImageSummary      `json:"Images" swaggerignore:"true"`
-		Info       types.Info                `json:"Info" swaggerignore:"true"`
+		Images     []image.Summary           `json:"Images" swaggerignore:"true"`
+		Info       system.Info               `json:"Info" swaggerignore:"true"`
 		Version    types.Version             `json:"Version" swaggerignore:"true"`
 	}
 
@@ -756,19 +759,20 @@ type (
 
 	// OAuthSettings represents the settings used to authorize with an authorization server
 	OAuthSettings struct {
-		ClientID             string `json:"ClientID"`
-		ClientSecret         string `json:"ClientSecret,omitempty"`
-		AccessTokenURI       string `json:"AccessTokenURI"`
-		AuthorizationURI     string `json:"AuthorizationURI"`
-		ResourceURI          string `json:"ResourceURI"`
-		RedirectURI          string `json:"RedirectURI"`
-		UserIdentifier       string `json:"UserIdentifier"`
-		Scopes               string `json:"Scopes"`
-		OAuthAutoCreateUsers bool   `json:"OAuthAutoCreateUsers"`
-		DefaultTeamID        TeamID `json:"DefaultTeamID"`
-		SSO                  bool   `json:"SSO"`
-		LogoutURI            string `json:"LogoutURI"`
-		KubeSecretKey        []byte `json:"KubeSecretKey"`
+		ClientID             string           `json:"ClientID"`
+		ClientSecret         string           `json:"ClientSecret,omitempty"`
+		AccessTokenURI       string           `json:"AccessTokenURI"`
+		AuthorizationURI     string           `json:"AuthorizationURI"`
+		ResourceURI          string           `json:"ResourceURI"`
+		RedirectURI          string           `json:"RedirectURI"`
+		UserIdentifier       string           `json:"UserIdentifier"`
+		Scopes               string           `json:"Scopes"`
+		OAuthAutoCreateUsers bool             `json:"OAuthAutoCreateUsers"`
+		DefaultTeamID        TeamID           `json:"DefaultTeamID"`
+		SSO                  bool             `json:"SSO"`
+		LogoutURI            string           `json:"LogoutURI"`
+		KubeSecretKey        []byte           `json:"KubeSecretKey"`
+		AuthStyle            oauth2.AuthStyle `json:"AuthStyle"`
 	}
 
 	// Pair defines a key/value string pair
@@ -1595,7 +1599,7 @@ type (
 
 const (
 	// APIVersion is the version number of the Portainer API
-	APIVersion = "2.20.0"
+	APIVersion = "2.20.2"
 	// Edition is what this edition of Portainer is called
 	Edition = PortainerCE
 	// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax
@@ -1724,6 +1728,8 @@ const (
 	EdgeStackStatusRollingBack
 	// EdgeStackStatusRolledBack represents an Edge stack which has rolled back
 	EdgeStackStatusRolledBack
+	// EdgeStackStatusCompleted represents a completed Edge stack
+	EdgeStackStatusCompleted
 )
 
 const (

api/stacks/deployments/deployer_remote.go

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/system"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/pkg/errors"
@@ -24,7 +25,7 @@ import (
 )
 
 const (
-	defaultUnpackerImage       = "portainer/compose-unpacker:latest"
+	defaultUnpackerImage       = "portainer/compose-unpacker:" + portainer.APIVersion
 	composeUnpackerImageEnvVar = "COMPOSE_UNPACKER_IMAGE"
 	composePathPrefix          = "portainer-compose-unpacker"
 )
@@ -211,9 +212,9 @@ func (d *stackDeployer) remoteStack(stack *portainer.Stack, endpoint *portainer.
 	if err != nil {
 		return errors.Wrap(err, "unable to create unpacker container")
 	}
-	defer cli.ContainerRemove(ctx, unpackerContainer.ID, types.ContainerRemoveOptions{})
+	defer cli.ContainerRemove(ctx, unpackerContainer.ID, container.RemoveOptions{})
 
-	if err := cli.ContainerStart(ctx, unpackerContainer.ID, types.ContainerStartOptions{}); err != nil {
+	if err := cli.ContainerStart(ctx, unpackerContainer.ID, container.StartOptions{}); err != nil {
 		return errors.Wrap(err, "start unpacker container error")
 	}
 
@@ -228,7 +229,7 @@ func (d *stackDeployer) remoteStack(stack *portainer.Stack, endpoint *portainer.
 
 	stdErr := &bytes.Buffer{}
 
-	out, err := cli.ContainerLogs(ctx, unpackerContainer.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
+	out, err := cli.ContainerLogs(ctx, unpackerContainer.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true})
 	if err != nil {
 		log.Error().Err(err).Msg("unable to get logs from unpacker container")
 	} else {
@@ -335,6 +336,6 @@ func getTargetSocketBind(osType string) string {
 
 // Per https://stackoverflow.com/a/50590287 and Docker's LocalNodeState possible values
 // `LocalNodeStateInactive` means the node is not in a swarm cluster
-func isNotInASwarm(info *types.Info) bool {
+func isNotInASwarm(info *system.Info) bool {
 	return info.Swarm.LocalNodeState == swarm.LocalNodeStateInactive
 }

app/angulartics.matomo/index.js

@@ -82,6 +82,7 @@ function config($analyticsProvider, $windowProvider) {
     push('setReferrerUrl', '');
     push('setCustomUrl', basePath + path);
     push('trackPageView');
+    push('enableLinkTracking');
   });
 
   /**

app/app.js

@@ -17,6 +17,16 @@ export function onStartupAngular($rootScope, $state, cfpLoadingBar, $transitions
     HttpRequestHelper.resetAgentHeaders();
   });
 
+  // EE-6751: screens not loading  when switching quickly between side menu options
+  // Known bug of @uirouter/angularjs
+  // Fix found at https://github.com/angular-ui/ui-router/issues/3652#issuecomment-574499009
+  // This hook is cleaning the internal viewConfigs list, removing leftover data unrelated to the current transition
+  $transitions.onStart({}, (transition) => {
+    const toList = transition.treeChanges().to.map((t) => t.state.name);
+    const toConfigs = transition.router.viewService._viewConfigs.filter((vc) => toList.includes(vc.viewDecl.$context.name));
+    transition.router.viewService._viewConfigs = toConfigs;
+  });
+
   $(document).ajaxSend((event, jqXhr, jqOpts) => {
     const type = jqOpts.type === 'POST' || jqOpts.type === 'PUT' || jqOpts.type === 'PATCH';
     const hasNoContentType = jqOpts.contentType !== 'application/json' && jqOpts.headers && !jqOpts.headers['Content-Type'];

app/assets/css/app.css

@@ -17,6 +17,7 @@
 html {
   font-size: 16px;
   overflow-y: scroll;
+  scroll-behavior: smooth;
 }
 
 html[theme='dark'],

app/docker/__module.js

@@ -104,6 +104,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controllerAs: 'ctrl',
         },
       },
+      data: {
+        docs: '/user/docker/configs/add',
+      },
     };
 
     const customTemplates = {
@@ -122,7 +125,7 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
 
     const customTemplatesNew = {
       name: 'docker.templates.custom.new',
-      url: '/new?appTemplateId&type',
+      url: '/new?fileContent&appTemplateId&type',
 
       views: {
         'content@': {
@@ -165,7 +168,7 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
         },
       },
       data: {
-        docs: '/user/docker/host',
+        docs: '/user/docker/host/details',
       },
     };
 
@@ -227,6 +230,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'BuildImageController',
         },
       },
+      data: {
+        docs: '/user/docker/images/build',
+      },
     };
 
     var imageImport = {
@@ -238,6 +244,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'ImportImageController',
         },
       },
+      data: {
+        docs: '/user/docker/images/import',
+      },
     };
 
     var networks = {
@@ -273,6 +282,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'CreateNetworkController',
         },
       },
+      data: {
+        docs: '/user/docker/networks/add',
+      },
     };
 
     var nodes = {
@@ -280,7 +292,7 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
       url: '/nodes',
       abstract: true,
       data: {
-        docs: '/user/docker/swarm',
+        docs: '/user/docker/swarm/details',
       },
     };
 
@@ -338,6 +350,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'CreateSecretController',
         },
       },
+      data: {
+        docs: '/user/docker/secrets/add',
+      },
     };
 
     var services = {
@@ -374,6 +389,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'CreateServiceController',
         },
       },
+      data: {
+        docs: '/user/docker/stacks/add',
+      },
     };
 
     var serviceLogs = {
@@ -444,7 +462,7 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
         },
       },
       data: {
-        docs: '/user/docker/swarm',
+        docs: '/user/docker/swarm/details',
       },
     };
 
@@ -500,7 +518,7 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
         },
       },
       data: {
-        docs: '/user/docker/templates',
+        docs: '/user/docker/templates/application',
       },
     };
 
@@ -549,6 +567,9 @@ angular.module('portainer.docker', ['portainer.app', reactModule]).config([
           controller: 'CreateVolumeController',
         },
       },
+      data: {
+        docs: '/user/docker/volumes/add',
+      },
     };
 
     const dockerFeaturesConfiguration = {

app/docker/helpers/imageHelper.js

@@ -16,7 +16,7 @@ function ImageHelperFactory() {
 
   /**
    *
-   * @param {import('@/react/docker/images/queries/useImages').ImagesListResponse[]} images
+   * @param {Array<{tags: Array<string>; id: string;}>} images
    * @returns {{names: string[]}}}
    */
   function getImagesNamesForDownload(images) {

app/docker/models/image.js

@@ -14,7 +14,7 @@ export function ImageViewModel(data) {
     }
   }
 
-  this.VirtualSize = data.VirtualSize;
+  this.Size = data.Size;
   this.Used = data.Used;
 
   if (data.Portainer && data.Portainer.Agent && data.Portainer.Agent.NodeName) {

app/docker/models/imageDetails.js

@@ -6,15 +6,22 @@ export function ImageDetailsViewModel(data) {
   this.Created = data.Created;
   this.Checked = false;
   this.RepoTags = data.RepoTags;
-  this.VirtualSize = data.VirtualSize;
+  this.Size = data.Size;
   this.DockerVersion = data.DockerVersion;
   this.Os = data.Os;
   this.Architecture = data.Architecture;
   this.Author = data.Author;
   this.Command = data.Config.Cmd;
-  this.Entrypoint = data.ContainerConfig.Entrypoint ? data.ContainerConfig.Entrypoint : '';
-  this.ExposedPorts = data.ContainerConfig.ExposedPorts ? Object.keys(data.ContainerConfig.ExposedPorts) : [];
-  this.Volumes = data.ContainerConfig.Volumes ? Object.keys(data.ContainerConfig.Volumes) : [];
-  this.Env = data.ContainerConfig.Env ? data.ContainerConfig.Env : [];
-  this.Labels = data.ContainerConfig.Labels;
+
+  let config = {};
+  if (data.Config) {
+    config = data.Config; // this is part of OCI images-spec
+  } else if (data.ContainerConfig != null) {
+    config = data.ContainerConfig; // not OCI ; has been removed in Docker 26 (API v1.45) along with .Container
+  }
+  this.Entrypoint = config.Entrypoint ? config.Entrypoint : '';
+  this.ExposedPorts = config.ExposedPorts ? Object.keys(config.ExposedPorts) : [];
+  this.Volumes = config.Volumes ? Object.keys(config.Volumes) : [];
+  this.Env = config.Env ? config.Env : [];
+  this.Labels = config.Labels;
 }

app/docker/react/components/index.ts

@@ -79,7 +79,7 @@ const ngModule = angular
   )
   .component(
     'dockerConfigsDatatable',
-    r2a(withUIRouter(ConfigsDatatable), [
+    r2a(withUIRouter(withCurrentUser(ConfigsDatatable)), [
       'dataset',
       'onRemoveClick',
       'onRefresh',
@@ -121,7 +121,11 @@ const ngModule = angular
   .component('dockerEventsDatatable', r2a(EventsDatatable, ['dataset']))
   .component(
     'dockerSecretsDatatable',
-    r2a(withUIRouter(SecretsDatatable), ['dataset', 'onRefresh', 'onRemove'])
+    r2a(withUIRouter(withCurrentUser(SecretsDatatable)), [
+      'dataset',
+      'onRefresh',
+      'onRemove',
+    ])
   )
   .component(
     'dockerStacksDatatable',

app/docker/react/views/containers.ts

@@ -85,6 +85,9 @@ function config($stateRegistryProvider: StateRegistry) {
         component: 'createContainerView',
       },
     },
+    data: {
+      docs: '/user/docker/containers/add',
+    },
   });
 
   $stateRegistryProvider.register({

app/docker/services/imageService.js

@@ -171,6 +171,11 @@ angular.module('portainer.docker').factory('ImageService', [
       return Image.tag({ id: id, repo: image }).$promise;
     };
 
+    /**
+     *
+     * @param {Array<{tags: Array<string>; id: string;}>} images
+     * @returns {Promise<unknown>}
+     */
     service.downloadImages = function (images) {
       var names = ImageHelper.getImagesNamesForDownload(images);
       return Image.download(names).$promise;

app/docker/views/containers/console/exec.html

@@ -69,10 +69,10 @@
             </div>
           </div>
           <div ng-if="state !== states.disconnected">
-            <label class="control-label text-left"
-              >Exec into container as <code>{{ ::formValues.user || 'default user' }}</code> using command
-              <code>{{ formValues.isCustomCommand ? formValues.customCommand : formValues.command }}</code>
-              <terminal-tooltip> </terminal-tooltip>
+            <label
+              >Exec into container as <code class="!text-sm align-baseline">{{ ::formValues.user || 'default user' }}</code> using command
+              <code class="!text-sm align-baseline">{{ formValues.isCustomCommand ? formValues.customCommand : formValues.command }}</code>
+              <terminal-tooltip class="align-sub"> </terminal-tooltip>
             </label>
             <button type="button" class="btn btn-primary" ng-click="disconnect()">
               <span ng-show="state === states.connected">Disconnect</span>

app/docker/views/dashboard/dashboard.html

@@ -18,7 +18,7 @@
     <p class="text-muted" ng-if="applicationState.endpoint.mode.role === 'MANAGER'">
       <pr-icon icon="'alert-circle'" mode="'primary'"></pr-icon>
       Portainer is connected to a node that is part of a Swarm cluster. Some resources located on other nodes in the cluster might not be available for management, have a look at
-      <a href="https://docs.portainer.io/admin/environments/add/swarm/agent" target="_blank">our agent setup</a> for more details.
+      <help-link doc-link="'/admin/environments/add/swarm/agent'" target="'_blank'" children="'our agent setup'"></help-link> for more details.
     </p>
     <p class="text-muted" ng-if="applicationState.endpoint.mode.role === 'WORKER'">
       <pr-icon icon="'alert-circle'" mode="'primary'"></pr-icon>

app/docker/views/dashboard/dashboardController.js

@@ -137,5 +137,5 @@ angular.module('portainer.docker').controller('DashboardController', [
 ]);
 
 function imagesTotalSize(images) {
-  return images.reduce((acc, image) => acc + image.VirtualSize, 0);
+  return images.reduce((acc, image) => acc + image.Size, 0);
 }

app/docker/views/images/build/buildimage.html

@@ -209,7 +209,7 @@
           </uib-tab>
           <uib-tab index="1" disable="!buildLogs">
             <uib-tab-heading class="vertical-center"> <pr-icon icon="'file-text'" class="leading-none"></pr-icon> Output </uib-tab-heading>
-            <pre class="log_viewer">
+            <pre class="log_viewer" data-cy="logViewer">
               <div ng-repeat="line in buildLogs track by $index" class="line"><p class="inner_line" ng-click="active=!active" ng-class="{'line_selected': active}">{{ line }}</p></div>
               <div ng-if="!buildLogs.length" class="line"><p class="inner_line">No build output available.</p></div>
             </pre>

app/docker/views/images/edit/image.html

@@ -130,7 +130,7 @@
             </tr>
             <tr>
               <td>Size</td>
-              <td>{{ image.VirtualSize | humansize }}</td>
+              <td>{{ image.Size | humansize }}</td>
             </tr>
             <tr>
               <td>Created</td>

app/docker/views/images/edit/imageController.js

@@ -162,7 +162,7 @@ angular.module('portainer.docker').controller('ImageController', [
     function exportImage(image) {
       HttpRequestHelper.setPortainerAgentTargetHeader(image.NodeName);
       $scope.state.exportInProgress = true;
-      ImageService.downloadImages([image])
+      ImageService.downloadImages([{ tags: image.RepoTags, id: image.Id }])
         .then(function success(data) {
           var downloadData = new Blob([data.file], { type: 'application/x-tar' });
           FileSaver.saveAs(downloadData, 'images.tar');

app/edge/__module.js

@@ -1,5 +1,6 @@
 import angular from 'angular';
 
+import { AccessHeaders } from '@/portainer/authorization-guard';
 import edgeStackModule from './views/edge-stacks';
 import { reactModule } from './react';
 
@@ -12,6 +13,9 @@ angular
       url: '/edge',
       parent: 'root',
       abstract: true,
+      data: {
+        access: AccessHeaders.EdgeAdmin,
+      },
     };
 
     const groups = {
@@ -62,12 +66,15 @@ angular
 
     const stacksNew = {
       name: 'edge.stacks.new',
-      url: '/new?templateId',
+      url: '/new?templateId&templateType',
       views: {
         'content@': {
           component: 'createEdgeStackView',
         },
       },
+      data: {
+        docs: '/user/edge/stacks/add',
+      },
     };
 
     const stacksEdit = {
@@ -137,7 +144,7 @@ angular
           },
         },
         data: {
-          docs: '/user/edge/devices',
+          docs: '/user/edge/waiting-room',
         },
       });
     }
@@ -151,7 +158,7 @@ angular
         },
       },
       data: {
-        docs: '/user/edge/templates',
+        docs: '/user/edge/templates/application',
       },
     });
 

app/edge/react/components/index.ts

@@ -13,7 +13,7 @@ import { withUIRouter } from '@/react-tools/withUIRouter';
 import { EdgeGroupAssociationTable } from '@/react/edge/components/EdgeGroupAssociationTable';
 import { AssociatedEdgeEnvironmentsSelector } from '@/react/edge/components/AssociatedEdgeEnvironmentsSelector';
 import { EnvironmentsDatatable } from '@/react/edge/edge-stacks/ItemView/EnvironmentsDatatable';
-import { TemplateFieldset } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset';
+import { TemplateFieldset } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset/TemplateFieldset';
 
 const ngModule = angular
   .module('portainer.edge.react.components', [])

app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js

@@ -13,7 +13,11 @@ import { StackType } from '@/react/common/stacks/types';
 import { applySetStateAction } from '@/react-tools/apply-set-state-action';
 import { getVariablesFieldDefaultValues } from '@/react/portainer/custom-templates/components/CustomTemplatesVariablesField';
 import { renderTemplate } from '@/react/portainer/custom-templates/components/utils';
-import { getInitialTemplateValues } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset';
+import { getInitialTemplateValues } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset/TemplateFieldset';
+import { getAppTemplates } from '@/react/portainer/templates/app-templates/queries/useAppTemplates';
+import { fetchFilePreview } from '@/react/portainer/templates/app-templates/queries/useFetchTemplateFile';
+import { TemplateViewModel } from '@/react/portainer/templates/app-templates/view-model';
+import { getDefaultValues as getAppVariablesDefaultValues } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset/EnvVarsFieldset';
 
 export default class CreateEdgeStackViewController {
   /* @ngInject */
@@ -73,7 +77,7 @@ export default class CreateEdgeStackViewController {
   }
 
   /**
-   * @param {import('react').SetStateAction<import('@/react/edge/edge-stacks/CreateView/TemplateFieldset').Values>} templateAction
+   * @param {import('react').SetStateAction<import('@/react/edge/edge-stacks/CreateView/TemplateFieldset/types').Values>} templateAction
    */
   setTemplateValues(templateAction) {
     return this.$async(async () => {
@@ -82,44 +86,52 @@ export default class CreateEdgeStackViewController {
       const newTemplateId = newTemplateValues.template && newTemplateValues.template.Id;
       this.state.templateValues = newTemplateValues;
       if (newTemplateId !== oldTemplateId) {
-        await this.onChangeTemplate(newTemplateValues.template);
+        await this.onChangeTemplate(newTemplateValues.type, newTemplateValues.template);
       }
 
-      let definitions = [];
-      if (this.state.templateValues.template) {
-        definitions = this.state.templateValues.template.Variables;
-      }
-      const newFile = renderTemplate(this.state.templateValues.file, this.state.templateValues.variables, definitions);
+      if (newTemplateValues.type === 'custom') {
+        const definitions = this.state.templateValues.template.Variables;
+        const newFile = renderTemplate(this.state.templateValues.file, this.state.templateValues.variables, definitions);
 
-      this.formValues.StackFileContent = newFile;
+        this.formValues.StackFileContent = newFile;
+      }
     });
   }
 
-  onChangeTemplate(template) {
+  onChangeTemplate(type, template) {
     return this.$async(async () => {
       if (!template) {
         return;
       }
 
-      this.state.templateValues.template = template;
-      this.state.templateValues.variables = getVariablesFieldDefaultValues(template.Variables);
+      if (type === 'custom') {
+        this.formValues = {
+          ...this.formValues,
+          DeploymentType: template.Type === StackType.Kubernetes ? DeploymentType.Kubernetes : DeploymentType.Compose,
+          ...toGitFormModel(template.GitConfig),
+          ...(template.EdgeSettings
+            ? {
+                PrePullImage: template.EdgeSettings.PrePullImage || false,
+                RetryDeploy: template.EdgeSettings.RetryDeploy || false,
+                PrivateRegistryId: template.EdgeSettings.PrivateRegistryId || null,
+                ...template.EdgeSettings.RelativePathSettings,
+              }
+            : {}),
+        };
 
-      const fileContent = await getCustomTemplateFile({ id: template.Id, git: !!template.GitConfig });
-      this.state.templateValues.file = fileContent;
+        const fileContent = await getCustomTemplateFile({ id: template.Id, git: !!template.GitConfig });
+        this.state.templateValues.file = fileContent;
+      }
 
-      this.formValues = {
-        ...this.formValues,
-        DeploymentType: template.Type === StackType.Kubernetes ? DeploymentType.Kubernetes : DeploymentType.Compose,
-        ...toGitFormModel(template.GitConfig),
-        ...(template.EdgeSettings
-          ? {
-              PrePullImage: template.EdgeSettings.PrePullImage || false,
-              RetryDeploy: template.EdgeSettings.RetryDeploy || false,
-              PrivateRegistryId: template.EdgeSettings.PrivateRegistryId || null,
-              ...template.EdgeSettings.RelativePathSettings,
-            }
-          : {}),
-      };
+      if (type === 'app') {
+        this.formValues.StackFileContent = '';
+        try {
+          const fileContent = await fetchFilePreview(template.Id);
+          this.formValues.StackFileContent = fileContent;
+        } catch (err) {
+          this.Notifications.error('Failure', err, 'Unable to retrieve Template');
+        }
+      }
     });
   }
 
@@ -159,13 +171,27 @@ export default class CreateEdgeStackViewController {
     }
   }
 
-  async preSelectTemplate(templateId) {
+  /**
+   *
+   * @param {'app' | 'custom'} templateType
+   * @param {number} templateId
+   * @returns {Promise<void>}
+   */
+  async preSelectTemplate(templateType, templateId) {
     return this.$async(async () => {
       try {
         this.state.Method = 'template';
-        const template = await getCustomTemplate(templateId);
+        const template = await getTemplate(templateType, templateId);
+        if (!template) {
+          return;
+        }
 
-        this.setTemplateValues({ template });
+        this.setTemplateValues({
+          template,
+          type: templateType,
+          envVars: templateType === 'app' ? getAppVariablesDefaultValues(template.Env) : {},
+          variables: templateType === 'custom' ? getVariablesFieldDefaultValues(template.Variables) : [],
+        });
       } catch (e) {
         notifyError('Failed loading template', e);
       }
@@ -179,9 +205,10 @@ export default class CreateEdgeStackViewController {
       this.Notifications.error('Failure', err, 'Unable to retrieve Edge groups');
     }
 
-    const templateId = this.$state.params.templateId;
-    if (templateId) {
-      this.preSelectTemplate(templateId);
+    const templateId = parseInt(this.$state.params.templateId, 10);
+    const templateType = this.$state.params.templateType;
+    if (templateType && templateId && !Number.isNaN(templateId)) {
+      this.preSelectTemplate(templateType, templateId);
     }
 
     this.$window.onbeforeunload = () => {
@@ -198,6 +225,16 @@ export default class CreateEdgeStackViewController {
   createStack() {
     return this.$async(async () => {
       const name = this.formValues.Name;
+
+      if (!this.validateTemplate()) {
+        return;
+      }
+
+      let envVars = this.formValues.envVars;
+      if (this.state.Method === 'template' && this.state.templateValues.type === 'app') {
+        envVars = [...envVars, ...Object.entries(this.state.templateValues.envVars).map(([key, value]) => ({ name: key, value }))];
+      }
+
       const method = getMethod(this.state.Method, this.state.templateValues.template);
 
       if (!this.validateForm(method)) {
@@ -206,7 +243,7 @@ export default class CreateEdgeStackViewController {
 
       this.state.actionInProgress = true;
       try {
-        await this.createStackByMethod(name, method);
+        await this.createStackByMethod(name, method, envVars);
 
         this.Notifications.success('Success', 'Stack successfully deployed');
         this.state.isEditorDirty = false;
@@ -258,19 +295,19 @@ export default class CreateEdgeStackViewController {
     return true;
   }
 
-  createStackByMethod(name, method) {
+  createStackByMethod(name, method, envVars) {
     switch (method) {
       case 'editor':
-        return this.createStackFromFileContent(name);
+        return this.createStackFromFileContent(name, envVars);
       case 'upload':
-        return this.createStackFromFileUpload(name);
+        return this.createStackFromFileUpload(name, envVars);
       case 'repository':
-        return this.createStackFromGitRepository(name);
+        return this.createStackFromGitRepository(name, envVars);
     }
   }
 
-  createStackFromFileContent(name) {
-    const { StackFileContent, Groups, DeploymentType, UseManifestNamespaces, envVars } = this.formValues;
+  createStackFromFileContent(name, envVars) {
+    const { StackFileContent, Groups, DeploymentType, UseManifestNamespaces } = this.formValues;
 
     return this.EdgeStackService.createStackFromFileContent({
       name,
@@ -282,8 +319,9 @@ export default class CreateEdgeStackViewController {
     });
   }
 
-  createStackFromFileUpload(name) {
-    const { StackFile, Groups, DeploymentType, UseManifestNamespaces, envVars } = this.formValues;
+  createStackFromFileUpload(name, envVars) {
+    const { StackFile, Groups, DeploymentType, UseManifestNamespaces } = this.formValues;
+
     return this.EdgeStackService.createStackFromFileUpload(
       {
         Name: name,
@@ -296,8 +334,9 @@ export default class CreateEdgeStackViewController {
     );
   }
 
-  createStackFromGitRepository(name) {
-    const { Groups, DeploymentType, UseManifestNamespaces, envVars } = this.formValues;
+  async createStackFromGitRepository(name, envVars) {
+    const { Groups, DeploymentType, UseManifestNamespaces } = this.formValues;
+
     const repositoryOptions = {
       RepositoryURL: this.formValues.RepositoryURL,
       RepositoryReferenceName: this.formValues.RepositoryReferenceName,
@@ -306,6 +345,7 @@ export default class CreateEdgeStackViewController {
       RepositoryUsername: this.formValues.RepositoryUsername,
       RepositoryPassword: this.formValues.RepositoryPassword,
       TLSSkipVerify: this.formValues.TLSSkipVerify,
+      CreatedFromCustomTemplateID: this.state.templateValues.template && this.state.templateValues.template.Id,
     };
     return this.EdgeStackService.createStackFromGitRepository(
       {
@@ -328,12 +368,26 @@ export default class CreateEdgeStackViewController {
     });
   }
 
+  validateTemplate() {
+    if (this.state.Method === 'template' && this.state.templateValues.type === 'app') {
+      return Object.entries(this.state.templateValues.envVars).every(([, value]) => !!value);
+    }
+
+    if (this.state.Method === 'template' && this.state.templateValues.type === 'custom') {
+      return Object.entries(this.state.templateValues.variables).every(([, v]) => {
+        return !!v.value;
+      });
+    }
+    return true;
+  }
+
   formIsInvalid() {
     return (
       this.form.$invalid ||
       !this.formValues.Groups.length ||
       (['template', 'editor'].includes(this.state.Method) && !this.formValues.StackFileContent) ||
-      ('upload' === this.state.Method && !this.formValues.StackFile)
+      ('upload' === this.state.Method && !this.formValues.StackFile) ||
+      !this.validateTemplate()
     );
   }
 }
@@ -354,3 +408,25 @@ function getMethod(method, template) {
   }
   return 'editor';
 }
+
+/**
+ *
+ * @param {'app' | 'custom'} templateType
+ * @param {number} templateId
+ * @returns {Promise<import('@/react/portainer/templates/app-templates/view-model').TemplateViewModel | import('@/react/portainer/templates/custom-templates/types').CustomTemplate | undefined>}
+ */
+async function getTemplate(templateType, templateId) {
+  if (!['app', 'custom'].includes(templateType)) {
+    notifyError('Invalid template type', `Invalid template type: ${templateType}`);
+    return;
+  }
+
+  if (templateType === 'app') {
+    const templatesResponse = await getAppTemplates();
+    const template = templatesResponse.templates.find((t) => t.id === templateId);
+    return new TemplateViewModel(template, templatesResponse.version);
+  }
+
+  const template = await getCustomTemplate(templateId);
+  return template;
+}

app/edge/views/edge-stacks/createEdgeStackView/docker-compose-form/docker-compose-form.controller.js

@@ -1,4 +1,4 @@
-import { getInitialTemplateValues } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset';
+import { getInitialTemplateValues } from '@/react/edge/edge-stacks/CreateView/TemplateFieldset/TemplateFieldset';
 import { editor, git, edgeStackTemplate, upload } from '@@/BoxSelector/common-options/build-methods';
 
 class DockerComposeFormController {

app/edge/views/edge-stacks/createEdgeStackView/docker-compose-form/docker-compose-form.html

@@ -35,6 +35,7 @@
     on-change="($ctrl.onChangeFormValues)"
     base-webhook-url="{{ $ctrl.state.baseWebhookUrl }}"
     webhook-id="{{ $ctrl.state.webhookId }}"
+    created-from-custom-template-id="($ctrl.state.templateValues.type === 'custom' ? $ctrl.state.templateValues.template.Id : 0)"
     docs-links
   ></git-form>
 </div>

app/kubernetes/__module.js

@@ -140,7 +140,7 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
         },
       },
       data: {
-        docs: '/user/kubernetes/helm',
+        docs: '/user/kubernetes/inspect-helm',
       },
     };
 
@@ -153,7 +153,7 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
         },
       },
       data: {
-        docs: '/user/kubernetes/services',
+        docs: '/user/kubernetes/networking/services',
       },
     };
 
@@ -166,7 +166,7 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
         },
       },
       data: {
-        docs: '/user/kubernetes/ingresses',
+        docs: '/user/kubernetes/networking/ingresses',
       },
     };
 
@@ -178,6 +178,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesIngressesCreateView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/networking/ingresses/add',
+      },
     };
 
     const ingressesEdit = {
@@ -211,6 +214,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesCreateApplicationView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/applications/add',
+      },
     };
 
     const application = {
@@ -221,6 +227,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'applicationDetailsView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/applications/inspect',
+      },
     };
 
     const applicationEdit = {
@@ -231,6 +240,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesCreateApplicationView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/applications/edit',
+      },
     };
 
     const applicationConsole = {
@@ -317,6 +329,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesCreateConfigMapView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/configurations/add-configmap',
+      },
     };
 
     const configMap = {
@@ -346,6 +361,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesCreateSecretView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/configurations/add-secret',
+      },
     };
 
     const secret = {
@@ -367,7 +385,7 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
         },
       },
       data: {
-        docs: '/user/kubernetes/cluster',
+        docs: '/user/kubernetes/cluster/details',
       },
     };
 
@@ -379,6 +397,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesNodeView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/cluster/node',
+      },
     };
 
     const nodeStats = {
@@ -412,6 +433,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesDeployView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/applications/manifest',
+      },
     };
 
     const resourcePools = {
@@ -435,6 +459,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesCreateNamespaceView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/namespaces/add',
+      },
     };
 
     const resourcePool = {
@@ -445,6 +472,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesResourcePoolView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/namespaces/manage',
+      },
     };
 
     const resourcePoolAccess = {
@@ -455,6 +485,9 @@ angular.module('portainer.kubernetes', ['portainer.app', registriesModule, custo
           component: 'kubernetesResourcePoolAccessView',
         },
       },
+      data: {
+        docs: '/user/kubernetes/namespaces/access',
+      },
     };
 
     const volumes = {

app/kubernetes/components/datatables/applications-datatable/applicationsDatatable.html

@@ -113,7 +113,7 @@
     </div>
     <div class="toolBar !pt-0">
       <div class="w-full">
-        <div class="form-group !h-[30px] min-w-[140px] float-right">
+        <div class="form-group float-right !h-[30px] min-w-[140px] mr-2">
           <div class="input-group">
             <span class="input-group-addon">
               <div className="flex items-center gap-1">
@@ -140,7 +140,7 @@
             </span>
           </div>
           <div class="w-fit">
-            <insights-box type="'slim'" header="'From 2.18 on, you can filter this view by namespace.'" insight-close-id="'k8s-namespace-filtering'"></insights-box>
+            <helm-insights-box></helm-insights-box>
           </div>
         </div>
       </div>

app/kubernetes/components/datatables/applications-datatable/applicationsDatatableController.js

@@ -169,20 +169,22 @@ angular.module('portainer.kubernetes').controller('KubernetesApplicationsDatatab
     };
 
     this.$onChanges = function (changes) {
-      // when the table is visible, sync the show system setting with the stack show system setting
-      if (changes.isVisible && changes.isVisible.currentValue) {
-        const storedStacksSettings = DatatableService.getDataTableSettings('kubernetes.applications.stacks');
-        if (storedStacksSettings && storedStacksSettings.state) {
-          this.settings.showSystem = storedStacksSettings.state.showSystemResources;
+      if (this.settingsLoaded) {
+        // when the table is visible, sync the show system setting with the stack show system setting
+        if (changes.isVisible && changes.isVisible.currentValue) {
+          const storedStacksSettings = DatatableService.getDataTableSettings('kubernetes.applications.stacks');
+          if (storedStacksSettings && storedStacksSettings.state) {
+            this.settings.showSystem = storedStacksSettings.state.showSystemResources;
+            DatatableService.setDataTableSettings(this.settingsKey, this.settings);
+          }
+        } else if (typeof this.isSystemResources !== 'undefined') {
+          this.settings.showSystem = this.isSystemResources;
           DatatableService.setDataTableSettings(this.settingsKey, this.settings);
         }
-      } else if (typeof this.isSystemResources !== 'undefined') {
-        this.settings.showSystem = this.isSystemResources;
-        DatatableService.setDataTableSettings(this.settingsKey, this.settings);
+        this.state.namespace = this.namespace;
+        this.updateNamespace();
+        this.prepareTableFromDataset();
       }
-      this.state.namespace = this.namespace;
-      this.updateNamespace();
-      this.prepareTableFromDataset();
     };
 
     this.$onInit = function () {
@@ -226,7 +228,7 @@ angular.module('portainer.kubernetes').controller('KubernetesApplicationsDatatab
         // make show system in sync with the stack show system settings
         const storedStacksSettings = DatatableService.getDataTableSettings('kubernetes.applications.stacks');
         if (storedStacksSettings && storedStacksSettings.state) {
-          this.settings.showSystem = storedStacksSettings.state.showSystemResources;
+          this.settings.showSystem = storedStacksSettings.state.showSystemResources || this.settings.showSystem;
         }
 
         this.setSystemResources && this.setSystemResources(this.settings.showSystem);

app/kubernetes/components/datatables/resource-pools-datatable/resourcePoolsDatatable.html

@@ -21,6 +21,7 @@
               auto-focus
               ng-model-options="{ debounce: 300 }"
               data-cy="k8sNamespace-namespaceSearchInput"
+              aria-label="Search input"
             />
           </div>
           <div class="actionBar !mr-0 !gap-3" ng-if="$ctrl.isAdmin">

app/kubernetes/components/helm/helm-templates/helm-templates-list/helm-templates-list-item/helm-templates-list-item.html

@@ -1,5 +1,5 @@
 <!-- helm chart -->
-<div ng-class="{ 'blocklist-item--selected': $ctrl.model.Selected }" class="blocklist-item template-item mx-0" ng-click="$ctrl.onSelect($ctrl.model)">
+<div ng-class="{ 'blocklist-item--selected': $ctrl.model.Selected }" class="blocklist-item template-item mx-0" ng-click="$ctrl.onSelect($ctrl.model)" role="listitem">
   <div class="blocklist-item-box">
     <!-- helmchart-image -->
     <span class="shrink-0">

app/kubernetes/components/helm/helm-templates/helm-templates-list/helm-templates-list.html

@@ -1,4 +1,4 @@
-<div class="datatable">
+<section class="datatable" aria-label="Helm charts">
   <div class="toolBar vertical-center relative w-full flex-wrap !gap-x-5 !gap-y-1 !px-0">
     <div class="toolBarTitle vertical-center">
       {{ $ctrl.titleText }}
@@ -6,7 +6,7 @@
 
     <div class="searchBar vertical-center !mr-0">
       <pr-icon icon="'search'" class="searchIcon"></pr-icon>
-      <input type="text" class="searchInput" ng-model="$ctrl.state.textFilter" placeholder="Search for a chart..." auto-focus ng-model-options="{ debounce: 300 }" />
+      <input type="text" class="searchInput" ng-model="$ctrl.state.textFilter" placeholder="Search..." auto-focus ng-model-options="{ debounce: 300 }" aria-label="Search input" />
     </div>
     <div class="w-1/5">
       <por-select
@@ -21,7 +21,8 @@
   </div>
   <div class="w-full">
     <div class="mb-2 small text-muted"
-      >Select the Helm chart to use. Bring further Helm charts into your selection list via <a ui-sref="portainer.account">User settings - Helm repositories</a>.</div
+      >Select the Helm chart to use. Bring further Helm charts into your selection list via
+      <a ui-sref="portainer.account({'#': 'helm-repositories'})">User settings - Helm repositories</a>.</div
     >
     <beta-alert
       is-html="true"
@@ -29,7 +30,7 @@
     ></beta-alert>
   </div>
 
-  <div class="blocklist !px-0">
+  <div class="blocklist !px-0" role="list">
     <helm-templates-list-item
       ng-repeat="chart in allCharts = ($ctrl.charts | filter:$ctrl.state.textFilter | filter: $ctrl.state.selectedCategory)"
       model="chart"
@@ -44,4 +45,4 @@
     </div>
     <div ng-if="!$ctrl.loading && $ctrl.charts.length === 0" class="text-muted text-center"> No helm charts available. </div>
   </div>
-</div>
+</section>

app/kubernetes/components/helm/helm-templates/helm-templates.html

@@ -40,7 +40,7 @@
             <pr-icon icon="'plus'" class="vertical-center"></pr-icon>
             Show custom values
           </button>
-          <span class="small interactive vertical-center" ng-if="$ctrl.state.loadingValues">
+          <span class="small interactive vertical-center" ng-if="$ctrl.state.loadingValues" role="status">
             <pr-icon icon="'refresh-cw'" class="mr-1"></pr-icon>
             Loading values.yaml...
           </span>

app/kubernetes/converters/configMap.js

@@ -38,6 +38,7 @@ class KubernetesConfigMapConverter {
     res.ConfigurationOwner = data.metadata.labels ? data.metadata.labels[KubernetesPortainerConfigurationOwnerLabel] : '';
     res.CreationDate = data.metadata.creationTimestamp;
     res.Yaml = yaml ? yaml.data : '';
+    res.Labels = data.metadata.labels;
 
     res.Data = _.concat(
       _.map(data.data, (value, key) => {
@@ -98,6 +99,7 @@ class KubernetesConfigMapConverter {
     res.metadata.uid = data.Id;
     res.metadata.name = data.Name;
     res.metadata.namespace = data.Namespace;
+    res.metadata.labels = data.Labels || {};
     res.metadata.labels[KubernetesPortainerConfigurationOwnerLabel] = data.ConfigurationOwner;
     _.forEach(data.Data, (entry) => {
       if (entry.IsBinary) {

app/kubernetes/converters/configuration.js

@@ -21,6 +21,7 @@ class KubernetesConfigurationConverter {
     if (secret.Annotations) {
       res.ServiceAccountName = secret.Annotations['kubernetes.io/service-account.name'];
     }
+    res.Labels = secret.Labels;
     return res;
   }
 
@@ -37,6 +38,7 @@ class KubernetesConfigurationConverter {
     });
     res.data = res.Data;
     res.ConfigurationOwner = configMap.ConfigurationOwner;
+    res.Labels = configMap.Labels;
     return res;
   }
 }

app/kubernetes/converters/daemonSet.js

@@ -21,7 +21,9 @@ class KubernetesDaemonSetConverter {
     const res = new KubernetesDaemonSet();
     res.Namespace = formValues.ResourcePool.Namespace.Name;
     res.Name = formValues.Name;
-    res.StackName = formValues.StackName ? formValues.StackName : formValues.Name;
+    if (formValues.StackName) {
+      res.StackName = formValues.StackName;
+    }
     res.ApplicationOwner = formValues.ApplicationOwner;
     res.ApplicationName = formValues.Name;
     res.ImageModel = formValues.ImageModel;

app/kubernetes/converters/deployment.js

@@ -22,7 +22,9 @@ class KubernetesDeploymentConverter {
     const res = new KubernetesDeployment();
     res.Namespace = formValues.ResourcePool.Namespace.Name;
     res.Name = formValues.Name;
-    res.StackName = formValues.StackName ? formValues.StackName : formValues.Name;
+    if (formValues.StackName) {
+      res.StackName = formValues.StackName;
+    }
     res.ApplicationOwner = formValues.ApplicationOwner;
     res.ApplicationName = formValues.Name;
     res.ReplicaCount = formValues.ReplicaCount;

app/kubernetes/converters/secret.js

@@ -39,6 +39,7 @@ class KubernetesSecretConverter {
     res.metadata.name = secret.Name;
     res.metadata.namespace = secret.Namespace;
     res.type = secret.Type;
+    res.metadata.labels = secret.Labels || {};
     res.metadata.labels[KubernetesPortainerConfigurationOwnerLabel] = secret.ConfigurationOwner;
 
     let annotation = '';
@@ -67,6 +68,7 @@ class KubernetesSecretConverter {
     res.Name = payload.metadata.name;
     res.Namespace = payload.metadata.namespace;
     res.Type = payload.type;
+    res.Labels = payload.metadata.labels || {};
     res.ConfigurationOwner = payload.metadata.labels ? payload.metadata.labels[KubernetesPortainerConfigurationOwnerLabel] : '';
     res.CreationDate = payload.metadata.creationTimestamp;
     res.Annotations = payload.metadata.annotations;

app/kubernetes/converters/statefulSet.js

@@ -24,7 +24,9 @@ class KubernetesStatefulSetConverter {
     const res = new KubernetesStatefulSet();
     res.Namespace = formValues.ResourcePool.Namespace.Name;
     res.Name = formValues.Name;
-    res.StackName = formValues.StackName ? formValues.StackName : formValues.Name;
+    if (formValues.StackName) {
+      res.StackName = formValues.StackName;
+    }
     res.ApplicationOwner = formValues.ApplicationOwner;
     res.ApplicationName = formValues.Name;
     res.ReplicaCount = formValues.ReplicaCount;

app/kubernetes/custom-templates/index.js

@@ -37,6 +37,9 @@ function config($stateRegistryProvider) {
     params: {
       fileContent: '',
     },
+    data: {
+      docs: '/user/kubernetes/templates/add',
+    },
   };
 
   const customTemplatesEdit = {

app/kubernetes/models/config-map/models.js

@@ -21,6 +21,7 @@ const _KubernetesConfigMap = Object.freeze({
   Yaml: '',
   ConfigurationOwner: '',
   Data: [],
+  Labels: {},
 });
 
 export class KubernetesConfigMap {