allow nested kube error that's thrown to bubble up

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -93,9 +93,6 @@ body:
       description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
       multiple: false
       options:
-        - '2.19.4'
-        - '2.19.3'
-        - '2.19.2'
         - '2.19.1'
         - '2.19.0'
         - '2.18.4'

.github/workflows/ci.yaml

@@ -1,155 +0,0 @@
-name: ci
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'develop'
-      - '!release/*'
-  pull_request:
-    branches:
-      - 'develop'
-      - 'release/*'
-      - 'feat/*'
-      - 'fix/*'
-      - 'refactor/*'
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
-
-env:
-  DOCKER_HUB_REPO: portainerci/portainer
-  NODE_ENV: testing
-  GO_VERSION: 1.21.5
-  NODE_VERSION: 18.x
-
-jobs:
-  build_images:
-    strategy:
-      matrix:
-        config:
-          - { platform: linux, arch: amd64 }
-          - { platform: linux, arch: arm64 }
-          - { platform: windows, arch: amd64, version: 1809 }
-          - { platform: windows, arch: amd64, version: ltsc2022 }
-    runs-on: arc-runner-set
-    if: github.event.pull_request.draft == false
-    steps:
-      - name: '[preparation] checkout the current branch'
-        uses: actions/checkout@v3.5.3
-        with:
-          ref: ${{ github.event.inputs.branch }}
-      - name: '[preparation] set up golang'
-        uses: actions/setup-go@v4.0.1
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-      - name: '[preparation] cache paths'
-        id: cache-dir-path
-        run: |
-          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
-          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
-      - name: '[preparation] cache go'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache-dir-path.outputs.go-build-dir }}
-            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
-          enableCrossOsArchive: true
-      - name: '[preparation] set up node.js'
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: ''
-      - name: '[preparation] cache yarn'
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
-          enableCrossOsArchive: true
-      - name: '[preparation] set up qemu'
-        uses: docker/setup-qemu-action@v2
-      - name: '[preparation] set up docker context for buildx'
-        run: docker context create builders
-      - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
-        with:
-          endpoint: builders
-      - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-      - name: '[preparation] set the container image tag'
-        run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
-          else
-            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
-          fi
-
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
-          else 
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
-          fi
-
-          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
-      - name: '[execution] build linux & windows portainer binaries'
-        run: |
-          export YARN_VERSION=$(yarn --version)
-          export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
-          export BUILDNUMBER=${GITHUB_RUN_NUMBER}
-          make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
-        env:
-          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
-      - name: '[execution] build and push docker images'
-        run: |
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            mv dist/portainer dist/portainer.exe
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
-          else 
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
-          fi
-        env:
-          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
-  build_manifests:
-    runs-on: arc-runner-set
-    if: github.event.pull_request.draft == false
-    needs: [build_images]
-    steps:
-      - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-      - name: '[preparation] set up docker context for buildx'
-        run: docker version && docker context create builders
-      - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
-        with:
-          endpoint: builders
-      - name: '[execution] build and push manifests'
-        run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
-          else
-            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
-          fi
-
-          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"

.github/workflows/lint.yml

@@ -11,31 +11,21 @@ on:
       - master
       - develop
       - release/*
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
-
-env:
-  GO_VERSION: 1.21.5
-  NODE_VERSION: 18.x
 
 jobs:
   run-linters:
     name: Run linters
     runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: '18'
           cache: 'yarn'
       - uses: actions/setup-go@v4
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.21.0
       - run: yarn --frozen-lockfile
       - name: Run linters
         uses: wearerequired/lint-action@v1
@@ -51,5 +41,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.55.2
+          version: v1.54.1
+          working-directory: api
           args: --timeout=10m -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -5,9 +5,6 @@ on:
     - cron: '0 20 * * *'
   workflow_dispatch:
 
-env:
-  GO_VERSION: 1.21.5
-
 jobs:
   client-dependencies:
     name: Client Dependency Check
@@ -28,7 +25,7 @@ jobs:
         with:
           json: true
 
-      - name: upload scan result as develop artifact
+      - name: upload scan result as develop artifact 
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
@@ -44,7 +41,7 @@ jobs:
           name: html-js-result-${{github.run_id}}
           path: js-result.html
 
-      - name: analyse vulnerabilities
+      - name: analyse vulnerabilities 
         id: set-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
@@ -61,10 +58,10 @@ jobs:
       - name: checkout repository
         uses: actions/checkout@master
 
-      - name: install Go
+      - name: install Go 
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.21.0'
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -77,7 +74,7 @@ jobs:
           yarn global add snyk
           snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: upload scan result as develop artifact
+      - name: upload scan result as develop artifact 
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-develop-result
@@ -105,68 +102,35 @@ jobs:
     if: >-
       github.ref == 'refs/heads/develop'
     outputs:
-      image-trivy: ${{ steps.set-trivy-matrix.outputs.image_trivy_result }}
-      image-docker-scout: ${{ steps.set-docker-scout-matrix.outputs.image_docker_scout_result }}
+      image: ${{ steps.set-matrix.outputs.image_result }}
     steps:
-      - name: scan vulnerabilities by Trivy
+      - name: scan vulnerabilities by Trivy 
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
           args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
 
-      - name: upload Trivy image security scan result as artifact
+      - name: upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: develop Trivy scan report export to html
+      - name: develop scan report export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-trivy-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-result")
 
-      - name: upload html file as Trivy artifact
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-${{github.run_id}}
-          path: image-trivy-result.html
+          path: image-result.html
 
-      - name: analyse vulnerabilities from Trivy
-        id: set-trivy-matrix
+      - name: analyse vulnerabilities
+        id: set-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
-          echo "image_trivy_result=${result}" >> $GITHUB_OUTPUT
-
-      - name: scan vulnerabilities by Docker Scout
-        uses: docker/scout-action@v1
-        continue-on-error: true
-        with:
-          command: cves
-          image: portainerci/portainer:develop
-          sarif-file: image-docker-scout.json
-          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
-          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: upload Docker Scout image security scan result as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: image-security-scan-develop-result
-          path: image-docker-scout.json
-
-      - name: develop Docker Scout scan report export to html
-        run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
-
-      - name: upload html file as Docker Scout artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: html-image-result-${{github.run_id}}
-          path: image-docker-scout-result.html
-
-      - name: analyse vulnerabilities from Docker Scout
-        id: set-docker-scout-matrix
-        run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=matrix)
-          echo "image_docker_scout_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Results
@@ -178,26 +142,22 @@ jobs:
       matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
-        image-trivy: ${{fromJson(needs.image-vulnerability.outputs.image-trivy)}}
-        image-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.image-docker-scout)}}
+        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
     steps:
       - name: display the results of js, Go, and image scan
         run: |
           echo "${{ matrix.js.status }}"
           echo "${{ matrix.go.status }}"
-          echo "${{ matrix.image-trivy.status }}"
-          echo "${{ matrix.image-docker-scout.status }}"
+          echo "${{ matrix.image.status }}"
           echo "${{ matrix.js.summary }}"
           echo "${{ matrix.go.summary }}"
-          echo "${{ matrix.image-trivy.summary }}"
-          echo "${{ matrix.image-docker-scout.summary }}"
+          echo "${{ matrix.image.summary }}"
 
-      - name: send message to Slack
-        if: >-
+      - name: send message to Slack 
+        if: >- 
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
-          matrix.image-trivy.status == 'failure' || 
-          matrix.image-docker-scout.status == 'failure'
+          matrix.image.status == 'failure'
         uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |
@@ -233,14 +193,7 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "*Image Trivy vulnerability check*: *${{ matrix.image-trivy.status }}*\n${{ matrix.image-trivy.summary }}\n"
-                      }
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "*Image Docker Scout vulnerability check*: *${{ matrix.image-docker-scout.status }}*\n${{ matrix.image-docker-scout.summary }}\n"
+                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
                       }
                     }
                   ]

.github/workflows/pr-security.yml

@@ -8,23 +8,19 @@ on:
     paths:
       - 'package.json'
       - 'go.mod'
+      - 'gruntfile.js'
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
       - '.github/workflows/pr-security.yml'
 
-env:
-  GO_VERSION: 1.21.3
-  NODE_VERSION: 18.x
-
 jobs:
   client-dependencies:
     name: Client Dependency Check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan' &&
-      github.event.pull_request.draft == false
+      github.event.review.body == '/scan'
     outputs:
       jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
     steps:
@@ -78,8 +74,7 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan' &&
-      github.event.pull_request.draft == false
+      github.event.review.body == '/scan'
     outputs:
       godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
     steps:
@@ -89,7 +84,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.21.0'
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -141,24 +136,22 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan' &&
-      github.event.pull_request.draft == false
+      github.event.review.body == '/scan'
     outputs:
-      imagediff-trivy: ${{ steps.set-diff-trivy-matrix.outputs.image_diff_trivy_result }}
-      imagediff-docker-scout: ${{ steps.set-diff-docker-scout-matrix.outputs.image_diff_docker_scout_result }}
+      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
     steps:
       - name: checkout code
         uses: actions/checkout@master
 
-      - name: install Go
+      - name: install Go 1.21.0
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.21.0'
 
-      - name: install Node.js
+      - name: install Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: 18.x
 
       - name: Install packages
         run: yarn --frozen-lockfile
@@ -174,26 +167,26 @@ jobs:
         with:
           context: .
           file: build/linux/Dockerfile
-          tags: local-portainer:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/local-portainer-image.tar
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
 
       - name: load docker image
         run: |
-          docker load --input /tmp/local-portainer-image.tar
+          docker load --input /tmp/trivy-portainer-image.tar
 
       - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress local-portainer:${{ github.sha }}
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
-      - name: upload Trivy image security scan result as artifact
+      - name: upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-feature-result
           path: image-trivy.json
 
-      - name: download Trivy artifacts from develop branch built by nightly scan
+      - name: download artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -205,65 +198,21 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: pr vs develop Trivy scan report comparison export to html
+      - name: pr vs develop scan report comparison export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-trivy-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-result")
 
-      - name: upload html file as Trivy artifact
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-trivy-result.html
+          path: image-result.html
 
-      - name: analyse different vulnerabilities against develop branch by Trivy
-        id: set-diff-trivy-matrix
+      - name: analyse different vulnerabilities against develop branch
+        id: set-diff-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
-          echo "image_diff_trivy_result=${result}" >> $GITHUB_OUTPUT
-
-      - name: scan vulnerabilities by Docker Scout
-        uses: docker/scout-action@v1
-        continue-on-error: true
-        with:
-          command: cves
-          image: local-portainer:${{ github.sha }}
-          sarif-file: image-docker-scout.json
-          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
-          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: upload Docker Scout image security scan result as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: image-security-scan-feature-result
-          path: image-docker-scout.json
-
-      - name: download Docker Scout artifacts from develop branch built by nightly scan
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          mv ./image-docker-scout.json ./image-docker-scout-feature.json
-          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
-          if [[ -e ./image-docker-scout.json ]]; then
-            mv ./image-docker-scout.json ./image-docker-scout-develop.json
-          else
-            echo "null" > ./image-docker-scout-develop.json
-          fi
-
-      - name: pr vs develop Docker Scout scan report comparison export to html
-        run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
-
-      - name: upload html file as Docker Scout artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-docker-scout-result.html
-
-      - name: analyse different vulnerabilities against develop branch by Docker Scout
-        id: set-diff-docker-scout-matrix
-        run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=matrix)
-          echo "image_diff_docker_scout_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_diff_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Result Against develop Branch
@@ -271,28 +220,23 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan' &&
-      github.event.pull_request.draft == false
+      github.event.review.body == '/scan'
     strategy:
       matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
-        imagediff-trivy: ${{fromJson(needs.image-vulnerability.outputs.imagediff-trivy)}}
-        imagediff-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.imagediff-docker-scout)}}
+        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
     steps:
       - name: check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff-trivy.status == 'failure' ||
-          matrix.imagediff-docker-scout.status == 'failure'
+          matrix.imagediff.status == 'failure'
         run: |
           echo "${{ matrix.jsdiff.status }}"
           echo "${{ matrix.godiff.status }}"
-          echo "${{ matrix.imagediff-trivy.status }}"
-          echo "${{ matrix.imagediff-docker-scout.status }}"
+          echo "${{ matrix.imagediff.status }}"
           echo "${{ matrix.jsdiff.summary }}"
           echo "${{ matrix.godiff.summary }}"
-          echo "${{ matrix.imagediff-trivy.summary }}"
-          echo "${{ matrix.imagediff-docker-scout.summary }}"
+          echo "${{ matrix.imagediff.summary }}"
           exit 1

.github/workflows/test.yaml

@@ -1,48 +1,25 @@
 name: Test
-
-env:
-  GO_VERSION: 1.21.5
-  NODE_VERSION: 18.x
-
-on:
-  pull_request:
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
-  push:
-    
+on: push
 jobs:
   test-client:
     runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
         run: make test-client ARGS="--maxWorkers=2"
   test-server:
-    strategy:
-      matrix:
-        config:
-          - { platform: linux, arch: amd64 }
-          - { platform: linux, arch: arm64 }
-          - { platform: windows, arch: amd64, version: 1809 }
-          - { platform: windows, arch: amd64, version: ltsc2022 }
     runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
-
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.21.0
       - name: Run tests
         run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -6,32 +6,22 @@ on:
       - master
       - develop
       - 'release/*'
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
-
-env:
-  GO_VERSION: 1.21.5
-  NODE_VERSION: 18.x
 
 jobs:
   openapi-spec:
     runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
     steps:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.21.0'
 
       - name: Download golang modules
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

Makefile

@@ -7,7 +7,7 @@ ARCH=$(shell go env GOARCH)
 # build target, can be one of "production", "testing", "development"
 ENV=development
 WEBPACK_CONFIG=webpack/webpack.$(ENV).js
-TAG=local
+TAG=latest
 
 SWAG=go run github.com/swaggo/swag/cmd/swag@v1.8.11 
 GOTESTSUM=go run gotest.tools/gotestsum@latest
@@ -68,7 +68,7 @@ test-client: ## Run client tests
 	yarn test $(ARGS)
 
 test-server:	## Run server tests
-	$(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
+	cd api && $(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
 
 ##@ Dev
 .PHONY: dev dev-client dev-server
@@ -92,7 +92,7 @@ format-client: ## Format client code
 	yarn format
 
 format-server: ## Format server code
-	go fmt ./...
+	cd api && go fmt ./...
 
 ##@ Lint
 .PHONY: lint lint-client lint-server
@@ -102,7 +102,7 @@ lint-client: ## Lint client code
 	yarn lint
 
 lint-server: ## Lint server code
-	golangci-lint run --timeout=10m -c .golangci.yaml
+	cd api && go vet ./...
 
 
 ##@ Extension
@@ -114,7 +114,7 @@ dev-extension: build-server build-client ## Run the extension in development mod
 ##@ Docs
 .PHONY: docs-build docs-validate docs-clean docs-validate-clean
 docs-build: init-dist ## Build docs
-	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./ 
+	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
 
 docs-validate: docs-build ## Validate docs
 	yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml

api/.golangci.yaml

@@ -4,20 +4,15 @@ linters:
 
   # Enable these for now
   enable:
-    - unused
     - depguard
-    - gosimple
     - govet
     - errorlint
     - exportloopref
-
 linters-settings:
   depguard:
     rules:
       main:
         deny:
-          - pkg: 'encoding/json'
-            desc: 'use github.com/segmentio/encoding/json'
           - pkg: 'github.com/sirupsen/logrus'
             desc: 'logging is allowed only by github.com/rs/zerolog'
           - pkg: 'golang.org/x/exp'

api/chisel/service.go

@@ -21,7 +21,6 @@ const (
 	tunnelCleanupInterval = 10 * time.Second
 	requiredTimeout       = 15 * time.Second
 	activeTimeout         = 4*time.Minute + 30*time.Second
-	pingTimeout           = 3 * time.Second
 )
 
 // Service represents a service to manage the state of multiple reverse tunnels.
@@ -60,18 +59,14 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 	}
 
 	httpClient := &http.Client{
-		Timeout: pingTimeout,
+		Timeout: 3 * time.Second,
 	}
 
 	resp, err := httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-
 	io.Copy(io.Discard, resp.Body)
 	resp.Body.Close()
 
-	return nil
+	return err
 }
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done

api/chisel/service_test.go

@@ -1,39 +0,0 @@
-package chisel
-
-import (
-	"net"
-	"net/http"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestPingAgentPanic(t *testing.T) {
-	endpointID := portainer.EndpointID(1)
-
-	s := NewService(nil, nil, nil)
-
-	defer func() {
-		require.Nil(t, recover())
-	}()
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(pingTimeout + 1*time.Second)
-	})
-
-	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
-	require.NoError(t, err)
-
-	go func() {
-		require.NoError(t, http.Serve(ln, mux))
-	}()
-
-	s.getTunnelDetails(endpointID)
-	s.tunnelDetailsMap[endpointID].Port = ln.Addr().(*net.TCPAddr).Port
-
-	require.Error(t, s.pingAgent(endpointID))
-}

api/cli/cli.go

@@ -49,7 +49,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),

api/cli/confirm.go

@@ -9,7 +9,7 @@ import (
 
 // Confirm starts a rollback db cli application
 func Confirm(message string) (bool, error) {
-	fmt.Printf("%s [y/N] ", message)
+	fmt.Printf("%s [y/N]", message)
 
 	reader := bufio.NewReader(os.Stdin)
 

api/cmd/portainer/main.go

@@ -3,9 +3,11 @@ package main
 import (
 	"context"
 	"crypto/sha256"
+	"math/rand"
 	"os"
 	"path"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
@@ -198,7 +200,7 @@ func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
 	return apikey.NewAPIKeyService(datastore.APIKeyRepository(), datastore.User())
 }
 
-func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (portainer.JWTService, error) {
+func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (dataservices.JWTService, error) {
 	if userSessionTimeout == "" {
 		userSessionTimeout = portainer.DefaultUserSessionTimeout
 	}
@@ -629,6 +631,8 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 }
 
 func main() {
+	rand.Seed(time.Now().UnixNano())
+
 	configureLogger()
 	setLoggingMode("PRETTY")
 

api/database/boltdb/db.go

@@ -144,8 +144,6 @@ func (connection *DbConnection) Open() error {
 // Close closes the BoltDB database.
 // Safe to being called multiple times.
 func (connection *DbConnection) Close() error {
-	log.Info().Msg("closing PortainerDB")
-
 	if connection.DB != nil {
 		return connection.DB.Close()
 	}
@@ -257,7 +255,7 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 			return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 		}
 
-		err := connection.UnmarshalObject(data, object)
+		err := connection.UnmarshalObjectWithJsoniter(data, object)
 		if err != nil {
 			return err
 		}

api/database/boltdb/export.go

@@ -1,10 +1,10 @@
 package boltdb
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 	bolt "go.etcd.io/bbolt"
 )
 

api/database/boltdb/json.go

@@ -1,41 +1,34 @@
 package boltdb
 
 import (
-	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
-	"github.com/segmentio/encoding/json"
 )
 
 var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
 
 // MarshalObject encodes an object to binary format
-func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error) {
-	buf := &bytes.Buffer{}
-
+func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		buf.WriteString(v)
+		data = []byte(v)
 	} else {
-		enc := json.NewEncoder(buf)
-		enc.SetSortMapKeys(false)
-		enc.SetAppendNewline(false)
-
-		if err := enc.Encode(object); err != nil {
-			return nil, err
+		data, err = json.Marshal(object)
+		if err != nil {
+			return data, err
 		}
 	}
-
 	if connection.getEncryptionKey() == nil {
-		return buf.Bytes(), nil
+		return data, nil
 	}
-
-	return encrypt(buf.Bytes(), connection.getEncryptionKey())
+	return encrypt(data, connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
@@ -61,6 +54,31 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
 	return err
 }
 
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// decoding at the moment.
+func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	if connection.getEncryptionKey() != nil {
+		var err error
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return err
+		}
+	}
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	err := jsoni.Unmarshal(data, &object)
+	if err != nil {
+		if s, ok := object.(*string); ok {
+			*s = string(data)
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
+}
+
 // mmm, don't have a KMS .... aes GCM seems the most likely from
 // https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
 

api/database/boltdb/tx.go

@@ -28,7 +28,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	return tx.conn.UnmarshalObject(value, object)
+	return tx.conn.UnmarshalObjectWithJsoniter(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -134,7 +134,7 @@ func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{},
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	return bucket.ForEach(func(k []byte, v []byte) error {
-		err := tx.conn.UnmarshalObject(v, obj)
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
 		if err == nil {
 			obj, err = appendFn(obj)
 		}
@@ -147,7 +147,7 @@ func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte
 	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
 
 	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
-		err := tx.conn.UnmarshalObject(v, obj)
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
 		if err != nil {
 			return err
 		}

api/dataservices/endpoint/endpoint.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -145,23 +144,6 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
-func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
-
-	return endpoints, service.connection.GetAll(
-		BucketName,
-		&portainer.Endpoint{},
-		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
-			for t := range e.TeamAccessPolicies {
-				if t == teamID {
-					return true
-				}
-			}
-			return false
-		}),
-	)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -122,23 +122,6 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
-func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
-
-	return endpoints, service.tx.GetAll(
-		BucketName,
-		&portainer.Endpoint{},
-		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
-			for t := range e.TeamAccessPolicies {
-				if t == teamID {
-					return true
-				}
-			}
-			return false
-		}),
-	)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/endpointgroup/endpointgroup.go

@@ -5,7 +5,10 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-const BucketName = "endpoint_groups"
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_groups"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {

api/dataservices/interface.go

@@ -2,6 +2,7 @@ package dataservices
 
 import (
 	"io"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
@@ -94,7 +95,6 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
-		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)
@@ -132,6 +132,15 @@ type (
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
 	}
 
+	// JWTService represents a service for managing JWT tokens
+	JWTService interface {
+		GenerateToken(data *portainer.TokenData) (string, error)
+		GenerateTokenForOAuth(data *portainer.TokenData, expiryTime *time.Time) (string, error)
+		GenerateTokenForKubeconfig(data *portainer.TokenData) (string, error)
+		ParseAndVerifyToken(token string) (*portainer.TokenData, error)
+		SetUserSessionDuration(userSessionDuration time.Duration)
+	}
+
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
 		BaseCRUD[portainer.Registry, portainer.RegistryID]

api/dataservices/snapshot/snapshot.go

@@ -5,7 +5,9 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-const BucketName = "snapshots"
+const (
+	BucketName = "snapshots"
+)
 
 type Service struct {
 	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]

api/dataservices/stack/stack.go

@@ -106,6 +106,7 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 	}
 
 	return nil, err
+
 }
 
 // RefreshableStacks returns stacks that are configured for a periodic update

api/dataservices/tag/tag.go

@@ -5,8 +5,10 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
-const BucketName = "tags"
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "tags"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {

api/dataservices/tag/tx.go

@@ -22,7 +22,7 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTagFunc is a no-op inside a transaction.
+// UpdateTagFunc is a no-op inside a transaction
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }

api/datastore/backup.go

@@ -4,82 +4,186 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"time"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/rs/zerolog/log"
 )
 
-func (store *Store) Backup() (string, error) {
-	if err := store.createBackupPath(); err != nil {
-		return "", err
-	}
-
-	backupFilename := store.backupFilename()
-	log.Info().Str("from", store.connection.GetDatabaseFilePath()).Str("to", backupFilename).Msgf("Backing up database")
-
-	// Close the store before backing up
-	err := store.Close()
-	if err != nil {
-		return "", fmt.Errorf("failed to close store before backup: %w", err)
-	}
-
-	err = store.fileService.Copy(store.connection.GetDatabaseFilePath(), backupFilename, true)
-	if err != nil {
-		return "", fmt.Errorf("failed to create backup file: %w", err)
-	}
-
-	// reopen the store
-	_, err = store.Open()
-	if err != nil {
-		return "", fmt.Errorf("failed to reopen store after backup: %w", err)
-	}
-
-	return backupFilename, nil
+var backupDefaults = struct {
+	backupDir string
+	commonDir string
+}{
+	"backups",
+	"common",
 }
 
-func (store *Store) Restore() error {
-	backupFilename := store.backupFilename()
-	return store.RestoreFromFile(backupFilename)
-}
+//
+// Backup Helpers
+//
 
-func (store *Store) RestoreFromFile(backupFilename string) error {
-	store.Close()
-	if err := store.fileService.Copy(backupFilename, store.connection.GetDatabaseFilePath(), true); err != nil {
-		return fmt.Errorf("unable to restore backup file %q. err: %w", backupFilename, err)
-	}
-
-	log.Info().Str("from", backupFilename).Str("to", store.connection.GetDatabaseFilePath()).Msgf("database restored")
-
-	_, err := store.Open()
-	if err != nil {
-		return fmt.Errorf("unable to determine version of restored portainer backup file: %w", err)
-	}
-
-	// determine the db version
-	version, err := store.VersionService.Version()
-	if err != nil {
-		return fmt.Errorf("unable to determine restored database version. err: %w", err)
-	}
-
-	editionLabel := portainer.SoftwareEdition(version.Edition).GetEditionLabel()
-	log.Info().Msgf("Restored database version: Portainer %s %s", editionLabel, version.SchemaVersion)
-	return nil
-}
-
-func (store *Store) createBackupPath() error {
-	backupDir := path.Join(store.connection.GetStorePath(), "backups")
-	if exists, _ := store.fileService.FileExists(backupDir); !exists {
-		if err := os.MkdirAll(backupDir, 0700); err != nil {
-			return fmt.Errorf("unable to create backup folder: %w", err)
+// createBackupFolders create initial folders for backups
+func (store *Store) createBackupFolders() {
+	// create common dir
+	commonDir := store.commonBackupDir()
+	if exists, _ := store.fileService.FileExists(commonDir); !exists {
+		if err := os.MkdirAll(commonDir, 0700); err != nil {
+			log.Error().Err(err).Msg("error while creating common backup folder")
 		}
 	}
-	return nil
-}
-
-func (store *Store) backupFilename() string {
-	return path.Join(store.connection.GetStorePath(), "backups", store.connection.GetDatabaseFileName()+".bak")
 }
 
 func (store *Store) databasePath() string {
 	return store.connection.GetDatabaseFilePath()
 }
+
+func (store *Store) commonBackupDir() string {
+	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
+}
+
+func (store *Store) copyDBFile(from string, to string) error {
+	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
+
+	err := store.fileService.Copy(from, to, true)
+	if err != nil {
+		log.Error().Err(err).Msg("failed")
+	}
+
+	return err
+}
+
+// BackupOptions provide a helper to inject backup options
+type BackupOptions struct {
+	Version        string
+	BackupDir      string
+	BackupFileName string
+	BackupPath     string
+}
+
+// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
+// - db backup prior to version upgrade
+// - db rollback
+func getBackupRestoreOptions(backupDir string) *BackupOptions {
+	return &BackupOptions{
+		BackupDir:      backupDir, //connection.commonBackupDir(),
+		BackupFileName: beforePortainerVersionUpgradeBackup,
+	}
+}
+
+// Backup current database with default options
+func (store *Store) Backup(version *models.Version) (string, error) {
+	if version == nil {
+		return store.backupWithOptions(nil)
+	}
+
+	return store.backupWithOptions(&BackupOptions{
+		Version: version.SchemaVersion,
+	})
+}
+
+func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
+	if options == nil {
+		options = &BackupOptions{}
+	}
+	if options.Version == "" {
+		v, err := store.VersionService.Version()
+		if err != nil {
+			options.Version = ""
+		}
+		options.Version = v.SchemaVersion
+	}
+	if options.BackupDir == "" {
+		options.BackupDir = store.commonBackupDir()
+	}
+	if options.BackupFileName == "" {
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
+	}
+	if options.BackupPath == "" {
+		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
+	}
+	return options
+}
+
+// BackupWithOptions backup current database with options
+func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
+	log.Info().Msg("creating DB backup")
+
+	store.createBackupFolders()
+
+	options = store.setupOptions(options)
+	dbPath := store.databasePath()
+
+	if err := store.Close(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error closing datastore before creating backup: %w",
+			err,
+		)
+	}
+
+	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
+		return options.BackupPath, err
+	}
+
+	if _, err := store.Open(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error opening datastore after creating backup: %w",
+			err,
+		)
+	}
+
+	return options.BackupPath, nil
+}
+
+// RestoreWithOptions previously saved backup for the current Edition  with options
+// Restore strategies:
+// - default: restore latest from current edition
+// - restore a specific
+func (store *Store) restoreWithOptions(options *BackupOptions) error {
+	options = store.setupOptions(options)
+
+	// Check if backup file exist before restoring
+	_, err := os.Stat(options.BackupPath)
+	if os.IsNotExist(err) {
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
+
+		return err
+	}
+
+	err = store.Close()
+	if err != nil {
+		log.Error().Err(err).Msg("error while closing store before restore")
+
+		return err
+	}
+
+	log.Info().Msg("restoring DB backup")
+	err = store.copyDBFile(options.BackupPath, store.databasePath())
+	if err != nil {
+		return err
+	}
+
+	_, err = store.Open()
+	return err
+}
+
+// RemoveWithOptions removes backup database based on supplied options
+func (store *Store) removeWithOptions(options *BackupOptions) error {
+	log.Info().Msg("removing DB backup")
+
+	options = store.setupOptions(options)
+	_, err := os.Stat(options.BackupPath)
+
+	if os.IsNotExist(err) {
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
+		return err
+	}
+
+	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
+	err = os.Remove(options.BackupPath)
+	if err != nil {
+		log.Error().Err(err).Msg("failed")
+		return err
+	}
+
+	return nil
+}

api/datastore/backup_test.go

@@ -2,79 +2,106 @@ package datastore
 
 import (
 	"fmt"
+	"os"
+	"path"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
-
-	"github.com/rs/zerolog/log"
 )
 
+func TestCreateBackupFolders(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
+
+	connection := store.GetConnection()
+	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
+
+	if isFileExist(backupPath) {
+		t.Error("Expect backups folder to not exist")
+	}
+
+	store.createBackupFolders()
+	if !isFileExist(backupPath) {
+		t.Error("Expect backups folder to exist")
+	}
+}
+
 func TestStoreCreation(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
 	if store == nil {
-		t.Fatal("Expect to create a store")
+		t.Error("Expect to create a store")
 	}
 
-	v, err := store.VersionService.Version()
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
-	if portainer.SoftwareEdition(v.Edition) != portainer.PortainerCE {
+	if store.CheckCurrentEdition() != nil {
 		t.Error("Expect to get CE Edition")
 	}
-
-	if v.SchemaVersion != portainer.APIVersion {
-		t.Error("Expect to get APIVersion")
-	}
 }
 
 func TestBackup(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
-	backupFileName := store.backupFilename()
-	t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) {
+	connection := store.GetConnection()
+
+	t.Run("Backup should create default db backup", func(t *testing.T) {
 		v := models.Version{
-			Edition:       int(portainer.PortainerCE),
 			SchemaVersion: portainer.APIVersion,
 		}
 		store.VersionService.UpdateVersion(&v)
-		store.Backup()
+		store.backupWithOptions(nil)
 
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+	})
+
+	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
+		store.backupWithOptions(&BackupOptions{
+			BackupFileName: beforePortainerVersionUpgradeBackup,
+			BackupDir:      store.commonBackupDir(),
+		})
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
 	})
 }
 
-func TestRestore(t *testing.T) {
-	_, store := MustNewTestStore(t, true, false)
+func TestRemoveWithOptions(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
 
-	t.Run("Basic Restore", func(t *testing.T) {
-		// override and set initial db version and edition
-		updateEdition(store, portainer.PortainerCE)
-		updateVersion(store, "2.4")
+	t.Run("successfully removes file if existent", func(t *testing.T) {
+		store.createBackupFolders()
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
 
-		store.Backup()
-		updateVersion(store, "2.16")
-		testVersion(store, "2.16", t)
-		store.Restore()
+		filePath := path.Join(options.BackupDir, options.BackupFileName)
+		f, err := os.Create(filePath)
+		if err != nil {
+			t.Fatalf("file should be created; err=%s", err)
+		}
+		f.Close()
 
-		// check if the restore is successful and the version is correct
-		testVersion(store, "2.4", t)
+		err = store.removeWithOptions(options)
+		if err != nil {
+			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
+		}
+
+		if isFileExist(f.Name()) {
+			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
+		}
 	})
 
-	t.Run("Basic Restore After Multiple Backups", func(t *testing.T) {
-		// override and set initial db version and edition
-		updateEdition(store, portainer.PortainerCE)
-		updateVersion(store, "2.4")
-		store.Backup()
-		updateVersion(store, "2.14")
-		updateVersion(store, "2.16")
-		testVersion(store, "2.16", t)
-		store.Restore()
+	t.Run("fails to removes file if non-existent", func(t *testing.T) {
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
 
-		// check if the restore is successful and the version is correct
-		testVersion(store, "2.4", t)
+		err := store.removeWithOptions(options)
+		if err == nil {
+			t.Error("RemoveWithOptions should fail for non-existent file")
+		}
 	})
 }

api/datastore/datastore.go

@@ -31,14 +31,8 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	if encryptionReq {
-		backupFilename, err := store.Backup()
-		if err != nil {
-			return false, fmt.Errorf("failed to backup database prior to encrypting: %w", err)
-		}
-
 		err = store.encryptDB()
 		if err != nil {
-			store.RestoreFromFile(backupFilename) // restore from backup if encryption fails
 			return false, err
 		}
 	}

api/datastore/helpers_test.go

@@ -1,58 +0,0 @@
-package datastore
-
-import (
-	"path/filepath"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
-)
-
-// isFileExist is helper function to check for file existence
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}
-
-func updateVersion(store *Store, v string) {
-	version, err := store.VersionService.Version()
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
-	version.SchemaVersion = v
-
-	err = store.VersionService.UpdateVersion(version)
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-}
-
-func updateEdition(store *Store, edition portainer.SoftwareEdition) {
-	version, err := store.VersionService.Version()
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
-	version.Edition = int(edition)
-
-	err = store.VersionService.UpdateVersion(version)
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-}
-
-// testVersion is a helper which tests current store version against wanted version
-func testVersion(store *Store, versionWant string, t *testing.T) {
-	v, err := store.VersionService.Version()
-	if err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	if v.SchemaVersion != versionWant {
-		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
-	}
-}

api/datastore/init.go

@@ -53,7 +53,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			},
 			SnapshotInterval:         portainer.DefaultSnapshotInterval,
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-			TemplatesURL:             "",
+			TemplatesURL:             portainer.DefaultTemplatesURL,
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,

api/datastore/migrate_data.go

@@ -2,7 +2,6 @@ package datastore
 
 import (
 	"fmt"
-	"os"
 	"runtime/debug"
 
 	portainer "github.com/portainer/portainer/api"
@@ -16,6 +15,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
+
 func (store *Store) MigrateData() error {
 	updating, err := store.VersionService.IsUpdating()
 	if err != nil {
@@ -40,7 +41,7 @@ func (store *Store) MigrateData() error {
 	}
 
 	// before we alter anything in the DB, create a backup
-	_, err = store.Backup()
+	backupPath, err := store.Backup(version)
 	if err != nil {
 		return errors.Wrap(err, "while backing up database")
 	}
@@ -50,9 +51,9 @@ func (store *Store) MigrateData() error {
 		err = errors.Wrap(err, "failed to migrate database")
 
 		log.Warn().Err(err).Msg("migration failed, restoring database to previous version")
-		restoreErr := store.Restore()
-		if restoreErr != nil {
-			return errors.Wrap(restoreErr, "failed to restore database")
+		restorErr := store.restoreWithOptions(&BackupOptions{BackupPath: backupPath})
+		if restorErr != nil {
+			return errors.Wrap(restorErr, "failed to restore database")
 		}
 
 		log.Info().Msg("database restored to previous version")
@@ -116,11 +117,6 @@ func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version *models
 		return err
 	}
 
-	// Special test code to simulate a failure (used by migrate_data_test.go).  Do not remove...
-	if os.Getenv("PORTAINER_TEST_MIGRATE_FAIL") == "FAIL" {
-		panic("test migration failure")
-	}
-
 	err = store.VersionService.StoreIsUpdating(false)
 	if err != nil {
 		return errors.Wrap(err, "failed to update the store")
@@ -139,7 +135,9 @@ func (store *Store) connectionRollback(force bool) error {
 		}
 	}
 
-	err := store.Restore()
+	options := getBackupRestoreOptions(store.commonBackupDir())
+
+	err := store.restoreWithOptions(options)
 	if err != nil {
 		return err
 	}

api/datastore/migrate_data_test.go

@@ -7,20 +7,29 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
-	"github.com/Masterminds/semver"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
-	"github.com/portainer/portainer/api/database/models"
-	"github.com/portainer/portainer/api/datastore/migrator"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/rs/zerolog/log"
 )
 
+// testVersion is a helper which tests current store version against wanted version
+func testVersion(store *Store, versionWant string, t *testing.T) {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		t.Errorf("Expect store version to be %s but was %s with error: %s", versionWant, v.SchemaVersion, err)
+	}
+	if v.SchemaVersion != versionWant {
+		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
+	}
+}
+
 func TestMigrateData(t *testing.T) {
-	tests := []struct {
+	snapshotTests := []struct {
 		testName           string
 		srcPath            string
 		wantPath           string
@@ -33,7 +42,7 @@ func TestMigrateData(t *testing.T) {
 			overrideInstanceId: true,
 		},
 	}
-	for _, test := range tests {
+	for _, test := range snapshotTests {
 		t.Run(test.testName, func(t *testing.T) {
 			err := migrateDBTestHelper(t, test.srcPath, test.wantPath, test.overrideInstanceId)
 			if err != nil {
@@ -46,167 +55,147 @@ func TestMigrateData(t *testing.T) {
 		})
 	}
 
-	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store := MustNewTestStore(t, true, false)
-		if !newStore {
-			t.Error("Expect a new DB")
-		}
+	// t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
+	// 	newStore, store, teardown := MustNewTestStore(t, true, false)
+	// 	defer teardown()
 
-		testVersion(store, portainer.APIVersion, t)
-		store.Close()
+	// 	if !newStore {
+	// 		t.Error("Expect a new DB")
+	// 	}
 
-		newStore, _ = store.Open()
-		if newStore {
-			t.Error("Expect store to NOT be new DB")
-		}
-	})
+	// 	testVersion(store, portainer.APIVersion, t)
+	// 	store.Close()
 
-	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store := MustNewTestStore(t, true, false)
-		store.VersionService.UpdateVersion(&models.Version{SchemaVersion: "1.0", Edition: int(portainer.PortainerCE)})
-		store.MigrateData()
+	// 	newStore, _ = store.Open()
+	// 	if newStore {
+	// 		t.Error("Expect store to NOT be new DB")
+	// 	}
+	// })
 
-		backupfilename := store.backupFilename()
-		if exists, _ := store.fileService.FileExists(backupfilename); !exists {
-			t.Errorf("Expect backup file to be created %s", backupfilename)
-		}
-	})
+	// tests := []struct {
+	// 	version         string
+	// 	expectedVersion string
+	// }{
+	// 	{version: "1.24.1", expectedVersion: portainer.APIVersion},
+	// 	{version: "2.0.0", expectedVersion: portainer.APIVersion},
+	// }
+	// for _, tc := range tests {
+	// 	_, store, teardown := MustNewTestStore(t, true, true)
+	// 	defer teardown()
 
-	t.Run("MigrateData should recover and restore backup during migration critical failure", func(t *testing.T) {
-		os.Setenv("PORTAINER_TEST_MIGRATE_FAIL", "FAIL")
+	// 	// Setup data
+	// 	v := models.Version{SchemaVersion: tc.version}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		version := "2.15"
-		_, store := MustNewTestStore(t, true, false)
-		store.VersionService.UpdateVersion(&models.Version{SchemaVersion: version, Edition: int(portainer.PortainerCE)})
-		store.MigrateData()
+	// 	// Required roles by migrations 22.2
+	// 	store.RoleService.Create(&portainer.Role{ID: 1})
+	// 	store.RoleService.Create(&portainer.Role{ID: 2})
+	// 	store.RoleService.Create(&portainer.Role{ID: 3})
+	// 	store.RoleService.Create(&portainer.Role{ID: 4})
 
-		store.Open()
-		testVersion(store, version, t)
-	})
+	// 	t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
+	// 		store.MigrateData()
+	// 		testVersion(store, tc.expectedVersion, t)
+	// 	})
 
-	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store := MustNewTestStore(t, true, false)
-		store.VersionService.StoreIsUpdating(true)
-		store.MigrateData()
+	// 	t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
+	// 		store.Rollback(true)
+	// 		store.Open()
+	// 		testVersion(store, tc.version, t)
+	// 	})
+	// }
 
-		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
-		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
-		// only create a backup when the version changes.
-		backupfilename := store.backupFilename()
-		if exists, _ := store.fileService.FileExists(backupfilename); exists {
-			t.Errorf("Backup file should not exist for dirty database")
-		}
-	})
+	// t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store := MustNewTestStore(t, true, false)
+	// 	v := models.Version{SchemaVersion: "1.24.1"}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		// Set migrator the count to match our migrations array (simulate no changes).
-		// Should not create a backup
-		v, err := store.VersionService.Version()
-		if err != nil {
-			t.Errorf("Unable to read version from db: %s", err)
-			t.FailNow()
-		}
+	// 	store.MigrateData()
 
-		migratorParams := store.newMigratorParameters(v)
-		m := migrator.NewMigrator(migratorParams)
-		latestMigrations := m.LatestMigrations()
+	// 	testVersion(store, v.SchemaVersion, t)
+	// })
 
-		if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
-			v.MigratorCount = len(latestMigrations.MigrationFuncs)
-			store.VersionService.UpdateVersion(v)
-		}
+	// t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-		store.MigrateData()
+	// 	v := models.Version{SchemaVersion: "0.0.0"}
+	// 	store.VersionService.UpdateVersion(&v)
 
-		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
-		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
-		// only create a backup when the version changes.
-		backupfilename := store.backupFilename()
-		if exists, _ := store.fileService.FileExists(backupfilename); exists {
-			t.Errorf("Backup file should not exist for dirty database")
-		}
-	})
+	// 	store.MigrateData()
 
-	t.Run("MigrateData should create backup on startup if portainer version matches db and migrationFuncs counts differ", func(t *testing.T) {
-		_, store := MustNewTestStore(t, true, false)
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-		// Set migrator count very large to simulate changes
-		// Should not create a backup
-		v, err := store.VersionService.Version()
-		if err != nil {
-			t.Errorf("Unable to read version from db: %s", err)
-			t.FailNow()
-		}
+	// 	if !isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should exist; file=%s", options.BackupPath)
+	// 	}
+	// })
 
-		v.MigratorCount = 1000
-		store.VersionService.UpdateVersion(v)
-		store.MigrateData()
+	// t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
 
-		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
-		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
-		// only create a backup when the version changes.
-		backupfilename := store.backupFilename()
-		if exists, _ := store.fileService.FileExists(backupfilename); !exists {
-			t.Errorf("DB backup should exist and there should be no error")
-		}
-	})
+	// 	store.VersionService.StoreIsUpdating(true)
+
+	// 	store.MigrateData()
+
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+
+	// 	if isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+	// 	}
+	// })
+
+	// t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
+	// 	_, store, teardown := MustNewTestStore(t, false, true)
+	// 	defer teardown()
+
+	// 	store.MigrateData()
+
+	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+
+	// 	if isFileExist(options.BackupPath) {
+	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+	// 	}
+	// })
+}
+
+func Test_getBackupRestoreOptions(t *testing.T) {
+	_, store := MustNewTestStore(t, false, true)
+
+	options := getBackupRestoreOptions(store.commonBackupDir())
+
+	wantDir := store.commonBackupDir()
+	if !strings.HasSuffix(options.BackupDir, wantDir) {
+		log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir")
+	}
+
+	wantFilename := "portainer.db.bak"
+	if options.BackupFileName != wantFilename {
+		log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file")
+	}
 }
 
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := "2.11"
-
-		v := models.Version{
-			SchemaVersion: version,
-		}
-
-		_, store := MustNewTestStore(t, false, false)
-		store.VersionService.UpdateVersion(&v)
-
-		_, err := store.Backup()
-		if err != nil {
-			log.Fatal().Err(err).Msg("")
-		}
-
-		v.SchemaVersion = "2.14"
-		// Change the current edition
-		err = store.VersionService.UpdateVersion(&v)
-		if err != nil {
-			log.Fatal().Err(err).Msg("")
-		}
-
-		err = store.Rollback(true)
-		if err != nil {
-			t.Logf("Rollback failed: %s", err)
-			t.Fail()
-			return
-		}
-
-		store.Open()
-		testVersion(store, version, t)
-	})
-
-	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := "2.15"
-
-		v := models.Version{
-			SchemaVersion: version,
-			Edition:       int(portainer.PortainerCE),
-		}
-
+		version := models.Version{SchemaVersion: "2.4.0"}
 		_, store := MustNewTestStore(t, true, false)
-		store.VersionService.UpdateVersion(&v)
 
-		_, err := store.Backup()
+		err := store.VersionService.UpdateVersion(&version)
+		if err != nil {
+			t.Errorf("Failed updating version: %v", err)
+		}
+
+		_, err = store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
 
-		v.SchemaVersion = "2.14"
-		// Change the current edition
-		err = store.VersionService.UpdateVersion(&v)
+		// Change the current version
+		version2 := models.Version{SchemaVersion: "2.6.0"}
+		err = store.VersionService.UpdateVersion(&version2)
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
@@ -218,11 +207,26 @@ func TestRollback(t *testing.T) {
 			return
 		}
 
-		store.Open()
-		testVersion(store, version, t)
+		_, err = store.Open()
+		if err != nil {
+			t.Logf("Open failed: %s", err)
+			t.Fail()
+			return
+		}
+
+		testVersion(store, version.SchemaVersion, t)
 	})
 }
 
+// isFileExist is helper function to check for file existence
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
 // migrateDBTestHelper loads a json representation of a bolt database from srcPath,
 // parses it into a database, runs a migration on that database, and then
 // compares it with an expected output database.

api/datastore/migrator/migrate_dbversion100.go

@@ -10,8 +10,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func (m *Migrator) migrateDockerDesktopExtensionSetting() error {
-	log.Info().Msg("updating docker desktop extension flag in settings")
+func (m *Migrator) migrateDockerDesktopExtentionSetting() error {
+	log.Info().Msg("updating docker desktop extention flag in settings")
 
 	isDDExtension := false
 	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {

api/datastore/migrator/migrate_dbversion110.go

@@ -1,25 +0,0 @@
-package migrator
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/rs/zerolog/log"
-)
-
-// updateAppTemplatesVersionForDB110 changes the templates URL to be empty if it was never changed
-// from the default value (version 2.0 URL)
-func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
-	log.Info().Msg("updating app templates url to v3.0")
-
-	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
-
-	settings, err := migrator.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	if settings.TemplatesURL == version2URL || settings.TemplatesURL == portainer.DefaultTemplatesURL {
-		settings.TemplatesURL = ""
-	}
-
-	return migrator.settingsService.UpdateSettings(settings)
-}

api/datastore/migrator/migrate_dbversion24.go

@@ -14,10 +14,8 @@ func (m *Migrator) updateSettingsToDB25() error {
 		return err
 	}
 
-	// to keep the same migration functionality as before 2.20.0, we need to set the templates URL to v2
-	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
 	if legacySettings.TemplatesURL == "" {
-		legacySettings.TemplatesURL = version2URL
+		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
 	}
 
 	legacySettings.UserSessionTimeout = portainer.DefaultUserSessionTimeout

api/datastore/migrator/migrate_dbversion31.go

@@ -245,7 +245,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 	return nil
 }
 
-func findResourcesToUpdateForDB32(dockerID string, volumesData volume.ListResponse, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
+func findResourcesToUpdateForDB32(dockerID string, volumesData volume.VolumeListOKBody, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
 	volumes := volumesData.Volumes
 	for _, volume := range volumes {
 		volumeName := volume.Name

api/datastore/migrator/migrator.go

@@ -225,12 +225,9 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.18", m.migrateDBVersionToDB90)
 	m.addMigrations("2.19",
 		m.convertSeedToPrivateKeyForDB100,
-		m.migrateDockerDesktopExtensionSetting,
+		m.migrateDockerDesktopExtentionSetting,
 		m.updateEdgeStackStatusForDB100,
 	)
-	m.addMigrations("2.20",
-		m.updateAppTemplatesVersionForDB110,
-	)
 
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.

api/datastore/services.go

@@ -1,6 +1,7 @@
 package datastore
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 
@@ -37,7 +38,6 @@ import (
 	"github.com/portainer/portainer/api/dataservices/webhook"
 
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 // Store defines the implementation of portainer.DataStore using

api/datastore/test_data/output_24_to_latest.json

@@ -46,10 +46,12 @@
       },
       "EdgeCheckinInterval": 0,
       "EdgeKey": "",
+      "EnableGPUManagement": false,
       "Gpus": [],
       "GroupId": 1,
       "Heartbeat": false,
       "Id": 1,
+      "IsEdgeDevice": false,
       "Kubernetes": {
         "Configuration": {
           "AllowNoneIngressClass": false,
@@ -99,7 +101,8 @@
       "TeamAccessPolicies": {},
       "Type": 1,
       "URL": "unix:///var/run/docker.sock",
-      "UserAccessPolicies": {}
+      "UserAccessPolicies": {},
+      "UserTrusted": false
     }
   ],
   "registries": [
@@ -121,7 +124,8 @@
       "Name": "canister.io",
       "Password": "MjWbx8A6YK7cw7",
       "Quay": {
-        "OrganisationName": ""
+        "OrganisationName": "",
+        "UseOrganisation": false
       },
       "RegistryAccesses": {
         "1": {
@@ -580,8 +584,11 @@
     "AllowHostNamespaceForRegularUsers": true,
     "AllowPrivilegedModeForRegularUsers": true,
     "AllowStackManagementForRegularUsers": true,
+    "AllowVolumeBrowserForRegularUsers": false,
     "AuthenticationMethod": 1,
     "BlackListedLabels": [],
+    "DisplayDonationHeader": false,
+    "DisplayExternalContributors": false,
     "Edge": {
       "AsyncMode": false,
       "CommandInterval": 0,
@@ -591,6 +598,7 @@
     "EdgeAgentCheckinInterval": 5,
     "EdgePortainerUrl": "",
     "EnableEdgeComputeFeatures": false,
+    "EnableHostManagementFeatures": false,
     "EnableTelemetry": true,
     "EnforceEdgeID": false,
     "FeatureFlagSettings": null,
@@ -601,6 +609,7 @@
     "InternalAuthSettings": {
       "RequiredPasswordLength": 12
     },
+    "IsDockerDesktopExtension": false,
     "KubeconfigExpiry": "0",
     "KubectlShellImage": "portainer/kubectl-shell",
     "LDAPSettings": {
@@ -645,7 +654,7 @@
     },
     "ShowKomposeBuildOption": false,
     "SnapshotInterval": "5m",
-    "TemplatesURL": "",
+    "TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
     "TrustOnFirstConnect": false,
     "UserSessionTimeout": "8h",
     "fdoConfiguration": {
@@ -903,7 +912,7 @@
         "color": ""
       },
       "TokenIssueAt": 0,
-      "UseCache": false,
+      "UserTheme": "",
       "Username": "admin"
     },
     {
@@ -933,11 +942,11 @@
         "color": ""
       },
       "TokenIssueAt": 0,
-      "UseCache": false,
+      "UserTheme": "",
       "Username": "prabhat"
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/snapshot.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	_container "github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	portainer "github.com/portainer/portainer/api"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -174,12 +174,7 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 				if !snapshot.Swarm {
 					return err
 				} else {
-					if !strings.Contains(err.Error(), "No such container") {
-						return err
-					}
-					// It is common to have containers running on different Swarm nodes,
-					// so we just log the error in the debug level
-					log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
+					log.Info().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
 				}
 			} else {
 				var gpuOptions *_container.DeviceRequest = nil
@@ -245,7 +240,7 @@ func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) erro
 }
 
 func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
-	volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
+	volumes, err := cli.VolumeList(context.Background(), filters.Args{})
 	if err != nil {
 		return err
 	}

api/exec/compose_stack_integration_test.go

@@ -10,8 +10,8 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/pkg/libstack/compose"
-	"github.com/portainer/portainer/pkg/testhelpers"
 
 	"github.com/rs/zerolog/log"
 )

api/exec/swarm_stack.go

@@ -2,6 +2,7 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -14,9 +15,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/registryutils"
 	"github.com/portainer/portainer/api/stacks/stackutils"
-
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 // SwarmStackManager represents a service for managing stacks.

api/filesystem/filesystem.go

@@ -2,6 +2,7 @@ package filesystem
 
 import (
 	"bytes"
+	"encoding/json"
 	"encoding/pem"
 	"errors"
 	"fmt"
@@ -14,7 +15,6 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 const (
@@ -173,7 +173,7 @@ func (service *Service) GetStackProjectPathByVersion(stackIdentifier string, ver
 	}
 
 	if commitHash != "" {
-		versionStr = commitHash
+		versionStr = fmt.Sprintf("%s", commitHash)
 	}
 	return JoinPaths(service.wrapFileStore(ComposeStorePath), stackIdentifier, versionStr)
 }

api/git/azure.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -10,13 +11,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
 	gittypes "github.com/portainer/portainer/api/git/types"
 
-	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/pkg/errors"
-	"github.com/segmentio/encoding/json"
 )
 
 const (

api/hostmanagement/openamt/authorization.go

@@ -2,13 +2,12 @@ package openamt
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type authenticationResponse struct {

api/hostmanagement/openamt/configCIRA.go

@@ -2,6 +2,7 @@ package openamt
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -10,8 +11,6 @@ import (
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type CIRAConfig struct {

api/hostmanagement/openamt/configDevice.go

@@ -1,12 +1,11 @@
 package openamt
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type Device struct {

api/hostmanagement/openamt/configDomain.go

@@ -1,12 +1,11 @@
 package openamt
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type (

api/hostmanagement/openamt/configProfile.go

@@ -1,12 +1,11 @@
 package openamt
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type (

api/hostmanagement/openamt/deviceActions.go

@@ -1,13 +1,12 @@
 package openamt
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type ActionResponse struct {

api/hostmanagement/openamt/deviceFeatures.go

@@ -1,12 +1,11 @@
 package openamt
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 func (service *Service) enableDeviceFeatures(configuration portainer.OpenAMTConfiguration, deviceGUID string, features portainer.OpenAMTDeviceEnabledFeatures) error {

api/hostmanagement/openamt/openamt.go

@@ -2,6 +2,7 @@ package openamt
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -11,7 +12,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 
-	"github.com/segmentio/encoding/json"
 	"golang.org/x/sync/errgroup"
 )
 

api/http/client/client.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -13,7 +14,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 var errInvalidResponseStatus = errors.New("invalid response status (expecting 200)")

api/http/csrf/csrf.go

@@ -1,62 +0,0 @@
-package csrf
-
-import (
-	"crypto/rand"
-	"fmt"
-	"net/http"
-
-	httperror "github.com/portainer/portainer/pkg/libhttp/error"
-
-	gorillacsrf "github.com/gorilla/csrf"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/urfave/negroni"
-)
-
-func WithProtect(handler http.Handler) (http.Handler, error) {
-	handler = withSendCSRFToken(handler)
-
-	token := make([]byte, 32)
-	_, err := rand.Read(token)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
-	}
-
-	handler = gorillacsrf.Protect([]byte(token), gorillacsrf.Path("/"))(handler)
-
-	return withSkipCSRF(handler), nil
-}
-
-func withSendCSRFToken(handler http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		sw := negroni.NewResponseWriter(w)
-
-		sw.Before(func(sw negroni.ResponseWriter) {
-			statusCode := sw.Status()
-			if statusCode >= 200 && statusCode < 300 {
-				csrfToken := gorillacsrf.Token(r)
-				sw.Header().Set("X-CSRF-Token", csrfToken)
-			}
-		})
-
-		handler.ServeHTTP(sw, r)
-
-	})
-}
-
-func withSkipCSRF(handler http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-		skip, err := security.ShouldSkipCSRFCheck(r)
-		if err != nil {
-			httperror.WriteError(w, http.StatusForbidden, err.Error(), err)
-			return
-		}
-
-		if skip {
-			r = gorillacsrf.UnsafeSkipCheck(r)
-		}
-
-		handler.ServeHTTP(w, r)
-	})
-}

api/http/handler/auth/authenticate.go

@@ -6,7 +6,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
-	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -26,7 +25,7 @@ type authenticatePayload struct {
 
 type authenticateResponse struct {
 	// JWT token used to authenticate against the API
-	JWT string `json:"jwt" example:"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzAB"`
+	JWT string `json:"jwt" example:"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE"`
 }
 
 func (payload *authenticatePayload) Validate(r *http.Request) error {
@@ -143,15 +142,12 @@ func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User,
 }
 
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
-	token, expirationTime, err := handler.JWTService.GenerateToken(tokenData)
+	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
 		return httperror.InternalServerError("Unable to generate JWT token", err)
 	}
 
-	security.AddAuthCookie(w, token, expirationTime)
-
 	return response.JSON(w, &authenticateResponse{JWT: token})
-
 }
 
 func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settings *portainer.LDAPSettings) error {
@@ -200,7 +196,7 @@ func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settin
 
 func teamExists(teamName string, ldapGroups []string) bool {
 	for _, group := range ldapGroups {
-		if strings.EqualFold(group, teamName) {
+		if strings.ToLower(group) == strings.ToLower(teamName) {
 			return true
 		}
 	}

api/http/handler/auth/handler.go

@@ -18,13 +18,12 @@ type Handler struct {
 	*mux.Router
 	DataStore                   dataservices.DataStore
 	CryptoService               portainer.CryptoService
-	JWTService                  portainer.JWTService
+	JWTService                  dataservices.JWTService
 	LDAPService                 portainer.LDAPService
 	OAuthService                portainer.OAuthService
 	ProxyManager                *proxy.Manager
 	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
 	passwordStrengthChecker     security.PasswordStrengthChecker
-	bouncer                     security.BouncerService
 }
 
 // NewHandler creates a handler to manage authentication operations.
@@ -32,7 +31,6 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 	h := &Handler{
 		Router:                  mux.NewRouter(),
 		passwordStrengthChecker: passwordStrengthChecker,
-		bouncer:                 bouncer,
 	}
 
 	h.Handle("/auth/oauth/validate",
@@ -40,6 +38,7 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 	h.Handle("/auth",
 		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
 	h.Handle("/auth/logout",
-		bouncer.PublicAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
+
 	return h
 }

api/http/handler/auth/logout.go

@@ -11,7 +11,7 @@ import (
 
 // @id Logout
 // @summary Logout
-// @description **Access policy**: public
+// @description **Access policy**: authenticated
 // @security ApiKeyAuth
 // @security jwt
 // @tags auth
@@ -19,14 +19,14 @@ import (
 // @failure 500 "Server error"
 // @router /auth/logout [post]
 func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	tokenData, _ := handler.bouncer.CookieAuthLookup(r)
-
-	if tokenData != nil {
-		handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
-		logoutcontext.Cancel(tokenData.Token)
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
 	}
 
-	security.RemoveAuthCookie(w)
+	handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
+
+	logoutcontext.Cancel(tokenData.Token)
 
 	return response.Empty(w)
 }

api/http/handler/customtemplates/customtemplate_create.go

@@ -1,6 +1,7 @@
 package customtemplates
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -20,7 +21,6 @@ import (
 
 	"github.com/asaskevich/govalidator"
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 func (handler *Handler) customTemplateCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -103,8 +103,6 @@ type customTemplateFromFileContentPayload struct {
 	FileContent string `validate:"required"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
-	// EdgeTemplate indicates if this template purpose for Edge Stack
-	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) error {
@@ -152,7 +150,7 @@ func isValidNote(note string) bool {
 // @success 200 {object} portainer.CustomTemplate
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /custom_templates/create/string [post]
+// @router /custom_templates/string [post]
 func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*portainer.CustomTemplate, error) {
 	var payload customTemplateFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
@@ -162,16 +160,15 @@ func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*p
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:           portainer.CustomTemplateID(customTemplateID),
-		Title:        payload.Title,
-		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Description:  payload.Description,
-		Note:         payload.Note,
-		Platform:     (payload.Platform),
-		Type:         (payload.Type),
-		Logo:         payload.Logo,
-		Variables:    payload.Variables,
-		EdgeTemplate: payload.EdgeTemplate,
+		ID:          portainer.CustomTemplateID(customTemplateID),
+		Title:       payload.Title,
+		EntryPoint:  filesystem.ComposeFileDefaultName,
+		Description: payload.Description,
+		Note:        payload.Note,
+		Platform:    (payload.Platform),
+		Type:        (payload.Type),
+		Logo:        payload.Logo,
+		Variables:   payload.Variables,
 	}
 
 	templateFolder := strconv.Itoa(customTemplateID)
@@ -221,8 +218,6 @@ type customTemplateFromGitRepositoryPayload struct {
 	TLSSkipVerify bool `example:"false"`
 	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
 	IsComposeFormat bool `example:"false"`
-	// EdgeTemplate indicates if this template purpose for Edge Stack
-	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -269,7 +264,7 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 // @success 200 {object} portainer.CustomTemplate
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /custom_templates/create/repository [post]
+// @router /custom_templates/repository [post]
 func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (*portainer.CustomTemplate, error) {
 	var payload customTemplateFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
@@ -288,7 +283,6 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		Logo:            payload.Logo,
 		Variables:       payload.Variables,
 		IsComposeFormat: payload.IsComposeFormat,
-		EdgeTemplate:    payload.EdgeTemplate,
 	}
 
 	getProjectPath := func() string {
@@ -373,8 +367,6 @@ type customTemplateFromFileUploadPayload struct {
 	FileContent []byte
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
-	// EdgeTemplate indicates if this template purpose for Edge Stack
-	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) error {
@@ -427,15 +419,8 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 		if err != nil {
 			return errors.New("Invalid variables. Ensure that the variables are valid JSON")
 		}
-		err = validateVariablesDefinitions(payload.Variables)
-		if err != nil {
-			return err
-		}
+		return validateVariablesDefinitions(payload.Variables)
 	}
-
-	edgeTemplate, _ := request.RetrieveBooleanMultiPartFormValue(r, "EdgeTemplate", true)
-	payload.EdgeTemplate = edgeTemplate
-
 	return nil
 }
 
@@ -459,7 +444,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 // @success 200 {object} portainer.CustomTemplate
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /custom_templates/create/file [post]
+// @router /custom_templates/file [post]
 func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*portainer.CustomTemplate, error) {
 	payload := &customTemplateFromFileUploadPayload{}
 	err := payload.Validate(r)
@@ -469,16 +454,15 @@ func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*po
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:           portainer.CustomTemplateID(customTemplateID),
-		Title:        payload.Title,
-		Description:  payload.Description,
-		Note:         payload.Note,
-		Platform:     payload.Platform,
-		Type:         payload.Type,
-		Logo:         payload.Logo,
-		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Variables:    payload.Variables,
-		EdgeTemplate: payload.EdgeTemplate,
+		ID:          portainer.CustomTemplateID(customTemplateID),
+		Title:       payload.Title,
+		Description: payload.Description,
+		Note:        payload.Note,
+		Platform:    payload.Platform,
+		Type:        payload.Type,
+		Logo:        payload.Logo,
+		EntryPoint:  filesystem.ComposeFileDefaultName,
+		Variables:   payload.Variables,
 	}
 
 	templateFolder := strconv.Itoa(customTemplateID)

api/http/handler/customtemplates/customtemplate_git_fetch_test.go

@@ -2,6 +2,8 @@ package customtemplates
 
 import (
 	"bytes"
+	"encoding/json"
+	"fmt"
 	"io"
 	"io/fs"
 	"net/http"
@@ -17,10 +19,7 @@ import (
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
-
-	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -76,7 +75,7 @@ func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect stri
 	}
 
 	req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
-	testhelpers.AddTestSecurityCookie(req, jwt)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
 
 	rr := httptest.NewRecorder()
 	h.ServeHTTP(rr, req)
@@ -132,8 +131,8 @@ func Test_customTemplateGitFetch(t *testing.T) {
 	h := NewHandler(requestBouncer, store, fileService, gitService)
 
 	// generate two standard users' tokens
-	jwt1, _, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
-	jwt2, _, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
+	jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
+	jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
 
 	t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
 		singleAPIRequest(h, jwt1, is, "abcdefg")

api/http/handler/customtemplates/customtemplate_update.go

@@ -59,8 +59,6 @@ type customTemplateUpdatePayload struct {
 	TLSSkipVerify bool `example:"false"`
 	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
 	IsComposeFormat bool `example:"false"`
-	// EdgeTemplate indicates if this template purpose for Edge Stack
-	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
@@ -163,7 +161,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.Type = payload.Type
 	customTemplate.Variables = payload.Variables
 	customTemplate.IsComposeFormat = payload.IsComposeFormat
-	customTemplate.EdgeTemplate = payload.EdgeTemplate
 
 	if payload.RepositoryURL != "" {
 		if !govalidator.IsURL(payload.RepositoryURL) {

api/http/handler/docker/images/images_list.go

@@ -2,7 +2,6 @@ package images
 
 import (
 	"net/http"
-	"strings"
 
 	"github.com/docker/docker/api/types"
 	"github.com/portainer/portainer/api/http/handler/docker/utils"
@@ -67,12 +66,6 @@ func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *http
 
 	imagesList := make([]ImageResponse, len(images))
 	for i, image := range images {
-		if (image.RepoTags == nil || len(image.RepoTags) == 0) && (image.RepoDigests != nil && len(image.RepoDigests) > 0) {
-			for _, repoDigest := range image.RepoDigests {
-				image.RepoTags = append(image.RepoTags, repoDigest[0:strings.Index(repoDigest, "@")]+":<none>")
-			}
-		}
-
 		imagesList[i] = ImageResponse{
 			Created: image.Created,
 			ID:      image.ID,

api/http/handler/edgestacks/edgestack_create_git.go

@@ -31,7 +31,7 @@ type edgeStackFromGitRepositoryPayload struct {
 	// Path to the Stack file inside the Git repository
 	FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// List of identifiers of EdgeGroups
-	EdgeGroups []portainer.EdgeGroupID `example:"1" validate:"required"`
+	EdgeGroups []portainer.EdgeGroupID `example:"1"`
 	// Deployment type to deploy this stack
 	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
 	// compose is enabled only for docker environments
@@ -85,6 +85,7 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
+// @param method query string true "Creation Method" Enums(file,string,repository)
 // @param body body edgeStackFromGitRepositoryPayload true "stack config"
 // @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
 // @success 200 {object} portainer.EdgeStack

api/http/handler/edgestacks/edgestack_create_test.go

@@ -2,14 +2,13 @@ package edgestacks
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 // Create

api/http/handler/edgestacks/edgestack_delete_test.go

@@ -1,14 +1,13 @@
 package edgestacks
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 // Delete

api/http/handler/edgestacks/edgestack_status_update_test.go

@@ -2,14 +2,13 @@ package edgestacks
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 // Update Status

api/http/handler/edgestacks/edgestack_update.go

@@ -2,6 +2,7 @@ package edgestacks
 
 import (
 	"net/http"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
@@ -189,3 +190,26 @@ func (handler *Handler) handleChangeEdgeGroups(tx dataservices.DataStoreTx, edge
 
 	return newRelatedEnvironmentIDs, endpointsToAdd, nil
 }
+
+func newStatus(oldStatus map[portainer.EndpointID]portainer.EdgeStackStatus, relatedEnvironmentIds []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeStackStatus {
+	newStatus := make(map[portainer.EndpointID]portainer.EdgeStackStatus)
+	for _, endpointID := range relatedEnvironmentIds {
+		newEnvStatus := portainer.EdgeStackStatus{}
+
+		oldEnvStatus, ok := oldStatus[endpointID]
+		if ok {
+			newEnvStatus = oldEnvStatus
+		}
+
+		newEnvStatus.Status = []portainer.EdgeStackDeploymentStatus{
+			{
+				Time: time.Now().Unix(),
+				Type: portainer.EdgeStackStatusPending,
+			},
+		}
+
+		newStatus[endpointID] = newEnvStatus
+	}
+
+	return newStatus
+}

api/http/handler/edgestacks/edgestack_update_test.go

@@ -2,6 +2,7 @@ package edgestacks
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -9,8 +10,6 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/segmentio/encoding/json"
 )
 
 // Update

api/http/handler/edgestacks/handler.go

@@ -1,10 +1,12 @@
 package edgestacks
 
 import (
+	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	edgestackservice "github.com/portainer/portainer/api/internal/edge/edgestacks"
@@ -24,6 +26,8 @@ type Handler struct {
 	KubernetesDeployer portainer.KubernetesDeployer
 }
 
+const contextKey = "edgeStack_item"
+
 // NewHandler creates a handler to manage environment(endpoint) group operations.
 func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service) *Handler {
 	h := &Handler{
@@ -58,6 +62,35 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 	return h
 }
 
+func (handler *Handler) convertAndStoreKubeManifestIfNeeded(stackFolder string, projectPath, composePath string, relatedEndpointIds []portainer.EndpointID) (manifestPath string, err error) {
+	hasKubeEndpoint, err := hasKubeEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
+	if err != nil {
+		return "", fmt.Errorf("unable to check if edge stack has kube environments: %w", err)
+	}
+
+	if !hasKubeEndpoint {
+		return "", nil
+	}
+
+	composeConfig, err := handler.FileService.GetFileContent(projectPath, composePath)
+	if err != nil {
+		return "", fmt.Errorf("unable to retrieve Compose file from disk: %w", err)
+	}
+
+	kompose, err := handler.KubernetesDeployer.ConvertCompose(composeConfig)
+	if err != nil {
+		return "", fmt.Errorf("failed converting compose file to kubernetes manifest: %w", err)
+	}
+
+	komposeFileName := filesystem.ManifestFileDefaultName
+	_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, komposeFileName, kompose)
+	if err != nil {
+		return "", fmt.Errorf("failed to store kube manifest file: %w", err)
+	}
+
+	return komposeFileName, nil
+}
+
 func (handler *Handler) handlerDBErr(err error, msg string) *httperror.HandlerError {
 	httpErr := httperror.InternalServerError(msg, err)
 

api/http/handler/edgetemplates/edgetemplate_list.go

@@ -1,15 +1,13 @@
 package edgetemplates
 
 import (
+	"encoding/json"
 	"net/http"
-	"slices"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/client"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type templateFileFormat struct {
@@ -18,7 +16,6 @@ type templateFileFormat struct {
 }
 
 // @id EdgeTemplateList
-// @deprecated
 // @summary Fetches the list of Edge Templates
 // @description **Access policy**: administrator
 // @tags edge_templates
@@ -53,16 +50,10 @@ func (handler *Handler) edgeTemplateList(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to parse template file", err)
 	}
 
-	// We only support version 3 of the template format
-	// this is only a temporary fix until we have custom edge templates
-	if templateFile.Version != "3" {
-		return httperror.InternalServerError("Unsupported template version", nil)
-	}
-
 	filteredTemplates := make([]portainer.Template, 0)
 
 	for _, template := range templateFile.Templates {
-		if slices.Contains(template.Categories, "edge") && slices.Contains([]portainer.TemplateType{portainer.ComposeStackTemplate, portainer.SwarmStackTemplate}, template.Type) {
+		if template.Type == portainer.EdgeStackTemplate {
 			filteredTemplates = append(filteredTemplates, template)
 		}
 	}

api/http/handler/edgetemplates/handler.go

@@ -4,7 +4,6 @@ import (
 	"net/http"
 
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
@@ -26,7 +25,7 @@ func NewHandler(bouncer security.BouncerService) *Handler {
 	}
 
 	h.Handle("/edge_templates",
-		bouncer.AdminAccess(middlewares.Deprecated(httperror.LoggerHandler(h.edgeTemplateList), func(w http.ResponseWriter, r *http.Request) (string, *httperror.HandlerError) { return "", nil }))).Methods(http.MethodGet)
+		bouncer.AdminAccess(httperror.LoggerHandler(h.edgeTemplateList))).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -2,6 +2,7 @@ package endpointedge
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -16,7 +17,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/jwt"
 
-	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 

api/http/handler/endpoints/endpoint_agent_browse_docs.go

@@ -19,8 +19,6 @@ package endpoints
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
 // @router /endpoints/{id}/docker/v2/browse/put [post]
-//
-//lint:ignore U1000 Ignore unused code, for documentation purposes
 func _fileBrowseFileUploadV2() {
 	// dummy function to make swag pick up the above docs for the following REST call
 	// POST request on /browse/put?volumeID=:id

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -1,6 +1,7 @@
 package endpoints
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,8 +15,6 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type dockerhubStatusResponse struct {

api/http/handler/endpoints/endpoint_force_update_service.go

@@ -1,108 +0,0 @@
-package endpoints
-
-import (
-	"context"
-	"net/http"
-	"strings"
-
-	portaineree "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/docker/consts"
-	"github.com/portainer/portainer/api/docker/images"
-	httperror "github.com/portainer/portainer/pkg/libhttp/error"
-	"github.com/portainer/portainer/pkg/libhttp/request"
-	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/docker/docker/api/types"
-	dockertypes "github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-)
-
-type forceUpdateServicePayload struct {
-	// ServiceId to update
-	ServiceID string
-	// PullImage if true will pull the image
-	PullImage bool
-}
-
-func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
-	return nil
-}
-
-// @id endpointForceUpdateService
-// @summary force update a docker service
-// @description force update a docker service
-// @description **Access policy**: authenticated
-// @tags endpoints
-// @security ApiKeyAuth
-// @security jwt
-// @accept json
-// @produce json
-// @param id path int true "endpoint identifier"
-// @param body body forceUpdateServicePayload true "details"
-// @success 200 {object} dockertypes.ServiceUpdateResponse "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "endpoint not found"
-// @failure 500 "Server error"
-// @router /endpoints/{id}/forceupdateservice [put]
-func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest("Invalid environment identifier route variable", err)
-	}
-
-	var payload forceUpdateServicePayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
-	}
-
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portaineree.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
-	}
-
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
-		return httperror.Forbidden("Permission denied to force update service", err)
-	}
-
-	dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "", nil)
-	if err != nil {
-		return httperror.InternalServerError("Error creating docker client", err)
-	}
-	defer dockerClient.Close()
-
-	service, _, err := dockerClient.ServiceInspectWithRaw(context.Background(), payload.ServiceID, dockertypes.ServiceInspectOptions{InsertDefaults: true})
-	if err != nil {
-		return httperror.InternalServerError("Error looking up service", err)
-	}
-
-	service.Spec.TaskTemplate.ForceUpdate++
-
-	if payload.PullImage {
-		service.Spec.TaskTemplate.ContainerSpec.Image = strings.Split(service.Spec.TaskTemplate.ContainerSpec.Image, "@sha")[0]
-	}
-
-	newService, err := dockerClient.ServiceUpdate(context.Background(), payload.ServiceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{QueryRegistry: true})
-	if err != nil {
-		return httperror.InternalServerError("Error force update service", err)
-	}
-
-	go func() {
-		images.EvictImageStatus(payload.ServiceID)
-		images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
-		containers, _ := dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{
-			All:     true,
-			Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+payload.ServiceID)),
-		})
-
-		for _, container := range containers {
-			images.EvictImageStatus(container.ID)
-		}
-	}()
-
-	return response.JSON(w, newService)
-}

api/http/handler/endpoints/endpoint_list.go

@@ -2,6 +2,7 @@ package endpoints
 
 import (
 	"net/http"
+	"sort"
 	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
@@ -29,7 +30,7 @@ const (
 // @produce json
 // @param start query int false "Start searching from"
 // @param limit query int false "Limit results to this value"
-// @param sort query sortKey false "Sort results by this value" Enum("Name", "Group", "Status", "LastCheckIn", "EdgeID")
+// @param sort query int false "Sort results by this value"
 // @param order query int false "Order sorted results by desc/asc" Enum("asc", "desc")
 // @param search query string false "Search query"
 // @param groupIds query []int false "List environments(endpoints) of these groups"
@@ -97,7 +98,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 		return httperror.InternalServerError("Unable to filter endpoints", err)
 	}
 
-	sortEnvironmentsByField(filteredEndpoints, endpointGroups, getSortKey(sortField), sortOrder == "desc")
+	sortEndpointsByField(filteredEndpoints, endpointGroups, sortField, sortOrder == "desc")
 
 	filteredEndpointCount := len(filteredEndpoints)
 
@@ -146,6 +147,46 @@ func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []porta
 	return endpoints[start:end]
 }
 
+func sortEndpointsByField(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, sortField string, isSortDesc bool) {
+
+	switch sortField {
+	case "Name":
+		if isSortDesc {
+			sort.Stable(sort.Reverse(EndpointsByName(endpoints)))
+		} else {
+			sort.Stable(EndpointsByName(endpoints))
+		}
+
+	case "Group":
+		endpointGroupNames := make(map[portainer.EndpointGroupID]string, 0)
+		for _, group := range endpointGroups {
+			endpointGroupNames[group.ID] = group.Name
+		}
+
+		endpointsByGroup := EndpointsByGroup{
+			endpointGroupNames: endpointGroupNames,
+			endpoints:          endpoints,
+		}
+
+		if isSortDesc {
+			sort.Stable(sort.Reverse(endpointsByGroup))
+		} else {
+			sort.Stable(endpointsByGroup)
+		}
+
+	case "Status":
+		if isSortDesc {
+			sort.Slice(endpoints, func(i, j int) bool {
+				return endpoints[i].Status > endpoints[j].Status
+			})
+		} else {
+			sort.Slice(endpoints, func(i, j int) bool {
+				return endpoints[i].Status < endpoints[j].Status
+			})
+		}
+	}
+}
+
 func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.EndpointGroup) portainer.EndpointGroup {
 	var endpointGroup portainer.EndpointGroup
 	for _, group := range groups {

api/http/handler/endpoints/endpoint_list_test.go

@@ -1,6 +1,7 @@
 package endpoints
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -12,8 +13,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/testhelpers"
-
-	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,13 +27,13 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
 		GroupID: 1,
 		Type:    portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string `example:"1.0.0"`
+			Version string "example:\"1.0.0\""
 		}{
 			Version: "1.0.0",
 		},
 	}
 	version2Endpoint := portainer.Endpoint{ID: 2, GroupID: 1, Type: portainer.AgentOnDockerEnvironment, Agent: struct {
-		Version string `example:"1.0.0"`
+		Version string "example:\"1.0.0\""
 	}{Version: "2.0.0"}}
 	noVersionEndpoint := portainer.Endpoint{ID: 3, Type: portainer.AgentOnDockerEnvironment, GroupID: 1}
 	notAgentEnvironments := portainer.Endpoint{ID: 4, Type: portainer.DockerEnvironment, GroupID: 1}
@@ -211,7 +210,7 @@ func buildEndpointListRequest(query string) *http.Request {
 	restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
 	req = req.WithContext(restrictedCtx)
 
-	testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+	req.Header.Add("Authorization", "Bearer dummytoken")
 
 	return req
 }

api/http/handler/endpoints/endpoint_registry_access.go

@@ -140,7 +140,7 @@ func (handler *Handler) updateKubeAccess(endpoint *portainer.Endpoint, registry
 	}
 
 	for namespace := range namespacesToRemove {
-		err := cli.DeleteRegistrySecret(registry.ID, namespace)
+		err := cli.DeleteRegistrySecret(registry, namespace)
 		if err != nil {
 			return err
 		}

api/http/handler/endpoints/filter_test.go

@@ -22,12 +22,12 @@ func Test_Filter_AgentVersion(t *testing.T) {
 	version1Endpoint := portainer.Endpoint{ID: 1, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string `example:"1.0.0"`
+			Version string "example:\"1.0.0\""
 		}{Version: "1.0.0"}}
 	version2Endpoint := portainer.Endpoint{ID: 2, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string `example:"1.0.0"`
+			Version string "example:\"1.0.0\""
 		}{Version: "2.0.0"}}
 	noVersionEndpoint := portainer.Endpoint{ID: 3, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,

api/http/handler/endpoints/handler.go

@@ -6,7 +6,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/demo"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
@@ -37,7 +36,6 @@ type Handler struct {
 	K8sClientFactory      *cli.ClientFactory
 	ComposeStackManager   portainer.ComposeStackManager
 	AuthorizationService  *authorization.Service
-	DockerClientFactory   *dockerclient.ClientFactory
 	BindAddress           string
 	BindAddressHTTPS      string
 	PendingActionsService *pendingactions.PendingActionsService
@@ -81,8 +79,6 @@ func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Han
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryAccess))).Methods(http.MethodPut)
 
 	h.Handle("/endpoints/global-key", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointCreateGlobalKey))).Methods(http.MethodPost)
-	h.Handle("/endpoints/{id}/forceupdateservice",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointForceUpdateService))).Methods(http.MethodPut)
 
 	// DEPRECATED
 	h.Handle("/endpoints/{id}/status", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)

api/http/handler/endpoints/sort.go

@@ -1,94 +1,46 @@
 package endpoints
 
 import (
-	"slices"
+	"strings"
 
 	"github.com/fvbommel/sortorder"
 	portainer "github.com/portainer/portainer/api"
 )
 
-type comp[T any] func(a, b T) int
+type EndpointsByName []portainer.Endpoint
 
-func stringComp(a, b string) int {
-	if sortorder.NaturalLess(a, b) {
-		return -1
-	} else if sortorder.NaturalLess(b, a) {
-		return 1
-	} else {
-		return 0
-	}
+func (e EndpointsByName) Len() int {
+	return len(e)
 }
 
-func sortEnvironmentsByField(environments []portainer.Endpoint, environmentGroups []portainer.EndpointGroup, sortField sortKey, isSortDesc bool) {
-	if sortField == "" {
-		return
-	}
-
-	var less comp[portainer.Endpoint]
-	switch sortField {
-	case sortKeyName:
-		less = func(a, b portainer.Endpoint) int {
-			return stringComp(a.Name, b.Name)
-		}
-
-	case sortKeyGroup:
-		environmentGroupNames := make(map[portainer.EndpointGroupID]string, 0)
-		for _, group := range environmentGroups {
-			environmentGroupNames[group.ID] = group.Name
-		}
-
-		// set the "unassigned" group name to be empty string
-		environmentGroupNames[1] = ""
-
-		less = func(a, b portainer.Endpoint) int {
-			aGroup := environmentGroupNames[a.GroupID]
-			bGroup := environmentGroupNames[b.GroupID]
-
-			return stringComp(aGroup, bGroup)
-		}
-
-	case sortKeyStatus:
-		less = func(a, b portainer.Endpoint) int {
-			return int(a.Status - b.Status)
-		}
-
-	case sortKeyLastCheckInDate:
-		less = func(a, b portainer.Endpoint) int {
-			return int(a.LastCheckInDate - b.LastCheckInDate)
-		}
-	case sortKeyEdgeID:
-		less = func(a, b portainer.Endpoint) int {
-			return stringComp(a.EdgeID, b.EdgeID)
-		}
-
-	}
-
-	slices.SortStableFunc(environments, func(a, b portainer.Endpoint) int {
-		mul := 1
-		if isSortDesc {
-			mul = -1
-		}
-
-		return less(a, b) * mul
-	})
-
+func (e EndpointsByName) Swap(i, j int) {
+	e[i], e[j] = e[j], e[i]
 }
 
-type sortKey string
+func (e EndpointsByName) Less(i, j int) bool {
+	return sortorder.NaturalLess(strings.ToLower(e[i].Name), strings.ToLower(e[j].Name))
+}
 
-const (
-	sortKeyName            sortKey = "Name"
-	sortKeyGroup           sortKey = "Group"
-	sortKeyStatus          sortKey = "Status"
-	sortKeyLastCheckInDate sortKey = "LastCheckIn"
-	sortKeyEdgeID          sortKey = "EdgeID"
-)
+type EndpointsByGroup struct {
+	endpointGroupNames map[portainer.EndpointGroupID]string
+	endpoints          []portainer.Endpoint
+}
 
-func getSortKey(sortField string) sortKey {
-	fieldAsSortKey := sortKey(sortField)
-	if slices.Contains([]sortKey{sortKeyName, sortKeyGroup, sortKeyStatus, sortKeyLastCheckInDate, sortKeyEdgeID}, fieldAsSortKey) {
-		return fieldAsSortKey
+func (e EndpointsByGroup) Len() int {
+	return len(e.endpoints)
+}
+
+func (e EndpointsByGroup) Swap(i, j int) {
+	e.endpoints[i], e.endpoints[j] = e.endpoints[j], e.endpoints[i]
+}
+
+func (e EndpointsByGroup) Less(i, j int) bool {
+	if e.endpoints[i].GroupID == e.endpoints[j].GroupID {
+		return false
 	}
 
-	return ""
+	groupA := e.endpointGroupNames[e.endpoints[i].GroupID]
+	groupB := e.endpointGroupNames[e.endpoints[j].GroupID]
+
+	return sortorder.NaturalLess(strings.ToLower(groupA), strings.ToLower(groupB))
 }

api/http/handler/endpoints/sort_test.go

@@ -1,168 +0,0 @@
-package endpoints
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/slices"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSortEndpointsByField(t *testing.T) {
-	environments := []portainer.Endpoint{
-		{ID: 0, Name: "Environment 1", GroupID: 1, Status: 1, LastCheckInDate: 3, EdgeID: "edge32"},
-		{ID: 1, Name: "Environment 2", GroupID: 2, Status: 2, LastCheckInDate: 6, EdgeID: "edge57"},
-		{ID: 2, Name: "Environment 3", GroupID: 1, Status: 3, LastCheckInDate: 2, EdgeID: "test87"},
-		{ID: 3, Name: "Environment 4", GroupID: 2, Status: 4, LastCheckInDate: 1, EdgeID: "abc123"},
-	}
-
-	environmentGroups := []portainer.EndpointGroup{
-		{ID: 1, Name: "Group 1"},
-		{ID: 2, Name: "Group 2"},
-	}
-
-	tests := []struct {
-		name       string
-		sortField  sortKey
-		isSortDesc bool
-		expected   []portainer.EndpointID
-	}{
-		{
-			name:      "sort without value",
-			sortField: "",
-			expected: []portainer.EndpointID{
-				environments[0].ID,
-				environments[1].ID,
-				environments[2].ID,
-				environments[3].ID,
-			},
-		},
-		{
-			name:       "sort by name ascending",
-			sortField:  "Name",
-			isSortDesc: false,
-			expected: []portainer.EndpointID{
-				environments[0].ID,
-				environments[1].ID,
-				environments[2].ID,
-				environments[3].ID,
-			},
-		},
-		{
-			name:       "sort by name descending",
-			sortField:  "Name",
-			isSortDesc: true,
-			expected: []portainer.EndpointID{
-				environments[3].ID,
-				environments[2].ID,
-				environments[1].ID,
-				environments[0].ID,
-			},
-		},
-		{
-			name:       "sort by group name ascending",
-			sortField:  "Group",
-			isSortDesc: false,
-			expected: []portainer.EndpointID{
-				environments[0].ID,
-				environments[2].ID,
-				environments[1].ID,
-				environments[3].ID,
-			},
-		},
-		{
-			name:       "sort by group name descending",
-			sortField:  "Group",
-			isSortDesc: true,
-			expected: []portainer.EndpointID{
-				environments[1].ID,
-				environments[3].ID,
-				environments[0].ID,
-				environments[2].ID,
-			},
-		},
-
-		{
-			name:       "sort by status ascending",
-			sortField:  "Status",
-			isSortDesc: false,
-			expected: []portainer.EndpointID{
-				environments[0].ID,
-				environments[1].ID,
-				environments[2].ID,
-				environments[3].ID,
-			},
-		},
-		{
-			name:       "sort by status descending",
-			sortField:  "Status",
-			isSortDesc: true,
-			expected: []portainer.EndpointID{
-				environments[3].ID,
-				environments[2].ID,
-				environments[1].ID,
-				environments[0].ID,
-			},
-		},
-		{
-			name:       "sort by last check-in ascending",
-			sortField:  "LastCheckIn",
-			isSortDesc: false,
-			expected: []portainer.EndpointID{
-				environments[3].ID,
-				environments[2].ID,
-				environments[0].ID,
-				environments[1].ID,
-			},
-		},
-		{
-			name:       "sort by last check-in descending",
-			sortField:  "LastCheckIn",
-			isSortDesc: true,
-			expected: []portainer.EndpointID{
-				environments[1].ID,
-				environments[0].ID,
-				environments[2].ID,
-				environments[3].ID,
-			},
-		},
-		{
-			name:      "sort by edge ID ascending",
-			sortField: "EdgeID",
-			expected: []portainer.EndpointID{
-				environments[3].ID,
-				environments[0].ID,
-				environments[1].ID,
-				environments[2].ID,
-			},
-		},
-		{
-			name:       "sort by edge ID descending",
-			sortField:  "EdgeID",
-			isSortDesc: true,
-			expected: []portainer.EndpointID{
-				environments[2].ID,
-				environments[1].ID,
-				environments[0].ID,
-				environments[3].ID,
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			is := assert.New(t)
-			sortEnvironmentsByField(environments, environmentGroups, "Name", false) // reset to default sort order
-
-			sortEnvironmentsByField(environments, environmentGroups, tt.sortField, tt.isSortDesc)
-
-			is.Equal(tt.expected, getEndpointIDs(environments))
-		})
-	}
-}
-
-func getEndpointIDs(environments []portainer.Endpoint) []portainer.EndpointID {
-	return slices.Map(environments, func(environment portainer.Endpoint) portainer.EndpointID {
-		return environment.ID
-	})
-}

api/http/handler/helm/handler.go

@@ -20,14 +20,14 @@ type Handler struct {
 	*mux.Router
 	requestBouncer           security.BouncerService
 	dataStore                dataservices.DataStore
-	jwtService               portainer.JWTService
+	jwtService               dataservices.JWTService
 	kubeClusterAccessService kubernetes.KubeClusterAccessService
 	kubernetesDeployer       portainer.KubernetesDeployer
 	helmPackageManager       libhelm.HelmPackageManager
 }
 
 // NewHandler creates a handler to manage endpoint group operations.
-func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, jwtService portainer.JWTService, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeClusterAccessService kubernetes.KubeClusterAccessService) *Handler {
+func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, jwtService dataservices.JWTService, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeClusterAccessService kubernetes.KubeClusterAccessService) *Handler {
 	h := &Handler{
 		Router:                   mux.NewRouter(),
 		requestBouncer:           bouncer,
@@ -93,7 +93,7 @@ func (handler *Handler) getHelmClusterAccess(r *http.Request) (*options.Kubernet
 		return nil, httperror.InternalServerError("Unable to retrieve user authentication token", err)
 	}
 
-	bearerToken, _, err := handler.jwtService.GenerateToken(tokenData)
+	bearerToken, err := handler.jwtService.GenerateToken(tokenData)
 	if err != nil {
 		return nil, httperror.Unauthorized("Unauthorized", err)
 	}

api/http/handler/helm/helm_delete_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/portainer/portainer/api/internal/testhelpers"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -49,7 +48,7 @@ func Test_helmDelete(t *testing.T) {
 		req := httptest.NewRequest(http.MethodDelete, fmt.Sprintf("/1/kubernetes/helm/%s", options.Name), nil)
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+		req.Header.Add("Authorization", "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/helm/helm_install_test.go

@@ -2,24 +2,23 @@ package helm
 
 import (
 	"bytes"
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
+
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/testhelpers"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/pkg/libhelm/binary/test"
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/portainer/portainer/pkg/libhelm/release"
-
-	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -53,7 +52,7 @@ func Test_helmInstall(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/1/kubernetes/helm", bytes.NewBuffer(optdata))
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+		req.Header.Add("Authorization", "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/helm/helm_list_test.go

@@ -1,6 +1,7 @@
 package helm
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -10,16 +11,14 @@ import (
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/testhelpers"
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/pkg/libhelm/binary/test"
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/portainer/portainer/pkg/libhelm/release"
-
-	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
+
+	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
 func Test_helmList(t *testing.T) {
@@ -49,7 +48,7 @@ func Test_helmList(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/1/kubernetes/helm", nil)
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+		req.Header.Add("Authorization", "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/hostmanagement/openamt/amtrpc.go

@@ -2,6 +2,7 @@ package openamt
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -19,7 +20,6 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 type HostInfo struct {

api/http/handler/kubernetes/configmaps_and_secrets.go

@@ -8,22 +8,6 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
-// @id getKubernetesConfigMapsAndSecrets
-// @summary Get ConfigMaps and Secrets
-// @description Get all ConfigMaps and Secrets for a given namespace
-// @description **Access policy**: authenticated
-// @tags kubernetes
-// @security ApiKeyAuth
-// @security jwt
-// @accept json
-// @produce json
-// @param id path int true "Environment (Endpoint) identifier"
-// @param namespace path string true "Namespace name"
-// @success 200 {array} []kubernetes.K8sConfigMapOrSecret "Success"
-// @failure 400 "Invalid request"
-// @failure 500 "Server error"
-// @deprecated
-// @router /kubernetes/{id}/namespaces/{namespace}/configuration [get]
 func (handler *Handler) getKubernetesConfigMapsAndSecrets(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {

api/http/handler/kubernetes/handler.go

@@ -26,12 +26,12 @@ type Handler struct {
 	authorizationService     *authorization.Service
 	DataStore                dataservices.DataStore
 	KubernetesClientFactory  *cli.ClientFactory
-	JwtService               portainer.JWTService
+	JwtService               dataservices.JWTService
 	kubeClusterAccessService kubernetes.KubeClusterAccessService
 }
 
 // NewHandler creates a handler to process pre-proxied requests to external APIs.
-func NewHandler(bouncer security.BouncerService, authorizationService *authorization.Service, dataStore dataservices.DataStore, jwtService portainer.JWTService, kubeClusterAccessService kubernetes.KubeClusterAccessService, kubernetesClientFactory *cli.ClientFactory, kubernetesClient portainer.KubeClient) *Handler {
+func NewHandler(bouncer security.BouncerService, authorizationService *authorization.Service, dataStore dataservices.DataStore, jwtService dataservices.JWTService, kubeClusterAccessService kubernetes.KubeClusterAccessService, kubernetesClientFactory *cli.ClientFactory, kubernetesClient portainer.KubeClient) *Handler {
 	h := &Handler{
 		Router:                   mux.NewRouter(),
 		authorizationService:     authorizationService,
@@ -120,12 +120,7 @@ func (h *Handler) getProxyKubeClient(r *http.Request) (*cli.KubeClient, *httperr
 		return nil, httperror.BadRequest("Invalid environment identifier route variable", err)
 	}
 
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		return nil, httperror.Forbidden("Permission denied to access environment", err)
-	}
-
-	cli, ok := h.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), tokenData.Username)
+	cli, ok := h.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), r.Header.Get("Authorization"))
 	if !ok {
 		return nil, httperror.InternalServerError("Failed to lookup KubeClient", nil)
 	}
@@ -146,13 +141,8 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
-		tokenData, err := security.RetrieveTokenData(r)
-		if err != nil {
-			httperror.WriteError(w, http.StatusForbidden, "Permission denied to access environment", err)
-		}
-
 		// Check if we have a kubeclient against this auth token already, otherwise generate a new one
-		_, ok := handler.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), tokenData.Username)
+		_, ok := handler.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), r.Header.Get("Authorization"))
 		if ok {
 			next.ServeHTTP(w, r)
 			return
@@ -174,6 +164,12 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
+		// Generate a proxied kubeconfig, then create a kubeclient using it.
+		tokenData, err := security.RetrieveTokenData(r)
+		if err != nil {
+			httperror.WriteError(w, http.StatusForbidden, "Permission denied to access environment", err)
+			return
+		}
 		bearerToken, err := handler.JwtService.GenerateTokenForKubeconfig(tokenData)
 		if err != nil {
 			httperror.WriteError(w, http.StatusInternalServerError, "Unable to create JWT token", err)
@@ -212,7 +208,7 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
-		handler.KubernetesClientFactory.SetProxyKubeClient(strconv.Itoa(int(endpoint.ID)), tokenData.Username, kubeCli)
+		handler.KubernetesClientFactory.SetProxyKubeClient(strconv.Itoa(int(endpoint.ID)), r.Header.Get("Authorization"), kubeCli)
 		next.ServeHTTP(w, r)
 	})
 }

api/http/handler/kubernetes/ingresses.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"net/http"
+	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/middlewares"
@@ -177,9 +178,14 @@ func (handler *Handler) getKubernetesIngressControllersByNamespace(w http.Respon
 		)
 	}
 
-	cli, handlerErr := handler.getProxyKubeClient(r)
-	if handlerErr != nil {
-		return handlerErr
+	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
+		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
+	)
+	if !ok {
+		return httperror.InternalServerError(
+			"Failed to lookup KubeClient",
+			nil,
+		)
 	}
 
 	currentControllers, err := cli.GetIngressControllers()

api/http/handler/motd/motd.go

@@ -1,6 +1,7 @@
 package motd
 
 import (
+	"encoding/json"
 	"net/http"
 	"strings"
 
@@ -8,8 +9,6 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/pkg/libcrypto"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/segmentio/encoding/json"
 )
 
 type motdResponse struct {

api/http/handler/registries/handler.go

@@ -8,7 +8,6 @@ import (
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes/cli"
-	"github.com/portainer/portainer/api/pendingactions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 
@@ -26,12 +25,11 @@ func hideFields(registry *portainer.Registry, hideAccesses bool) {
 // Handler is the HTTP handler used to handle registry operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer        security.BouncerService
-	DataStore             dataservices.DataStore
-	FileService           portainer.FileService
-	ProxyManager          *proxy.Manager
-	K8sClientFactory      *cli.ClientFactory
-	PendingActionsService *pendingactions.PendingActionsService
+	requestBouncer   security.BouncerService
+	DataStore        dataservices.DataStore
+	FileService      portainer.FileService
+	ProxyManager     *proxy.Manager
+	K8sClientFactory *cli.ClientFactory
 }
 
 // NewHandler creates a handler to manage registry operations.

api/http/handler/registries/registry_delete.go

@@ -1,18 +1,14 @@
 package registries
 
 import (
-	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/pendingactions"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/rs/zerolog/log"
 )
 
 // @id RegistryDelete
@@ -42,9 +38,11 @@ func (handler *Handler) registryDelete(w http.ResponseWriter, r *http.Request) *
 		return httperror.BadRequest("Invalid registry identifier route variable", err)
 	}
 
-	registry, err := handler.DataStore.Registry().Read(portainer.RegistryID(registryID))
-	if err != nil {
-		return httperror.InternalServerError(fmt.Sprintf("Unable to load registry %q from the database", registry.Name), err)
+	_, err = handler.DataStore.Registry().Read(portainer.RegistryID(registryID))
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return httperror.NotFound("Unable to find a registry with the specified identifier inside the database", err)
+	} else if err != nil {
+		return httperror.InternalServerError("Unable to find a registry with the specified identifier inside the database", err)
 	}
 
 	err = handler.DataStore.Registry().Delete(portainer.RegistryID(registryID))
@@ -52,57 +50,5 @@ func (handler *Handler) registryDelete(w http.ResponseWriter, r *http.Request) *
 		return httperror.InternalServerError("Unable to remove the registry from the database", err)
 	}
 
-	err = handler.deleteKubernetesSecrets(registry)
-	if err != nil {
-		return httperror.InternalServerError("Unable to delete registry secrets", err)
-	}
-
 	return response.Empty(w)
 }
-
-func (handler *Handler) deleteKubernetesSecrets(registry *portainer.Registry) error {
-
-	for endpointId, access := range registry.RegistryAccesses {
-		if access.Namespaces != nil {
-			// Obtain a kubeclient for the endpoint
-			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointId)
-			if err != nil {
-				// Skip environments that can't be loaded from the DB
-				log.Warn().Err(err).Msgf("Unable to load the environment with id %d from the database", endpointId)
-				continue
-			}
-
-			cli, err := handler.K8sClientFactory.GetKubeClient(endpoint)
-			if err != nil {
-				// Skip environments that can't get a kubeclient from
-				log.Warn().Err(err).Msgf("Unable to get kubernetes client for environment %d", endpointId)
-				continue
-			}
-
-			failedNamespaces := make([]string, 0)
-			for _, ns := range access.Namespaces {
-				err = cli.DeleteRegistrySecret(registry.ID, ns)
-				if err != nil {
-					failedNamespaces = append(failedNamespaces, ns)
-					log.Warn().Err(err).Msgf("Unable to delete registry secret %q from namespace %q for environment %d. Retrying offline", cli.RegistrySecretName(registry.ID), ns, endpointId)
-				}
-			}
-
-			if len(failedNamespaces) > 0 {
-				handler.PendingActionsService.Create(portainer.PendingActions{
-					EndpointID: endpointId,
-					Action:     pendingactions.DeletePortainerK8sRegistrySecrets,
-
-					// When extracting the data, this is the type we need to pull out
-					// i.e. pendingactions.DeletePortainerK8sRegistrySecretsData
-					ActionData: pendingactions.DeletePortainerK8sRegistrySecretsData{
-						RegistryID: registry.ID,
-						Namespaces: failedNamespaces,
-					},
-				})
-			}
-		}
-	}
-
-	return nil
-}

api/http/handler/registries/registry_update.go

@@ -207,7 +207,7 @@ func (handler *Handler) updateEndpointRegistryAccess(endpoint *portainer.Endpoin
 	}
 
 	for _, namespace := range endpointAccess.Namespaces {
-		err := cli.DeleteRegistrySecret(registry.ID, namespace)
+		err := cli.DeleteRegistrySecret(registry, namespace)
 		if err != nil {
 			return err
 		}

api/http/handler/settings/handler.go

@@ -23,7 +23,7 @@ type Handler struct {
 	*mux.Router
 	DataStore       dataservices.DataStore
 	FileService     portainer.FileService
-	JWTService      portainer.JWTService
+	JWTService      dataservices.JWTService
 	LDAPService     portainer.LDAPService
 	SnapshotService portainer.SnapshotService
 	demoService     *demo.Service

api/http/handler/stacks/create_kubernetes_stack.go

@@ -153,6 +153,13 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 	if err != nil {
 		return httperror.InternalServerError("Unable to load user information from the database", err)
 	}
+	isUnique, err := handler.checkUniqueStackNameInKubernetes(endpoint, payload.StackName, 0, payload.Namespace)
+	if err != nil {
+		return httperror.InternalServerError("Unable to check for name collision", err)
+	}
+	if !isUnique {
+		return httperror.Conflict(fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), stackutils.ErrStackAlreadyExists)
+	}
 
 	stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate)
 
@@ -211,6 +218,13 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	if err != nil {
 		return httperror.InternalServerError("Unable to load user information from the database", err)
 	}
+	isUnique, err := handler.checkUniqueStackNameInKubernetes(endpoint, payload.StackName, 0, payload.Namespace)
+	if err != nil {
+		return httperror.InternalServerError("Unable to check for name collision", err)
+	}
+	if !isUnique {
+		return httperror.Conflict(fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), stackutils.ErrStackAlreadyExists)
+	}
 
 	//make sure the webhook ID is unique
 	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
@@ -282,6 +296,13 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	if err != nil {
 		return httperror.InternalServerError("Unable to load user information from the database", err)
 	}
+	isUnique, err := handler.checkUniqueStackNameInKubernetes(endpoint, payload.StackName, 0, payload.Namespace)
+	if err != nil {
+		return httperror.InternalServerError("Unable to check for name collision", err)
+	}
+	if !isUnique {
+		return httperror.Conflict(fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), stackutils.ErrStackAlreadyExists)
+	}
 
 	stackPayload := createStackPayloadFromK8sUrlPayload(payload.StackName,
 		payload.Namespace,

api/http/handler/stacks/handler.go

@@ -70,8 +70,6 @@ func NewHandler(bouncer security.BouncerService) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackDelete))).Methods(http.MethodDelete)
 	h.Handle("/stacks/{id}/associate",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.stackAssociate))).Methods(http.MethodPut)
-	h.Handle("/stacks/name/{name}",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackDeleteKubernetesByName))).Methods(http.MethodDelete)
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/git",
@@ -165,6 +163,31 @@ func (handler *Handler) checkUniqueStackName(endpoint *portainer.Endpoint, name
 	return true, nil
 }
 
+func (handler *Handler) checkUniqueStackNameInKubernetes(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, namespace string) (bool, error) {
+	isUniqueStackName, err := handler.checkUniqueStackName(endpoint, name, stackID)
+	if err != nil {
+		return false, err
+	}
+
+	if !isUniqueStackName {
+		// Check if this stack name is really used in the kubernetes.
+		// Because the stack with this name could be removed via kubectl cli outside and the datastore does not be informed of this action.
+		if namespace == "" {
+			namespace = "default"
+		}
+
+		kubeCli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+		if err != nil {
+			return false, err
+		}
+		isUniqueStackName, err = kubeCli.HasStackName(namespace, name)
+		if err != nil {
+			return false, err
+		}
+	}
+	return isUniqueStackName, nil
+}
+
 func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) {
 	isUniqueStackName, err := handler.checkUniqueStackName(endpoint, name, stackID)
 	if err != nil {

api/http/handler/stacks/stack_delete.go

@@ -251,137 +251,3 @@ func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.St
 
 	return fmt.Errorf("unsupported stack type: %v", stack.Type)
 }
-
-// @id StackDeleteKubernetesByName
-// @summary Remove Kubernetes stacks by name
-// @description Remove a stack.
-// @description **Access policy**: restricted
-// @tags stacks
-// @security ApiKeyAuth
-// @security jwt
-// @param name path string true "Stack name"
-// @param external query boolean false "Set to true to delete an external stack. Only external Swarm stacks are supported"
-// @param endpointId query int true "Environment identifier"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "Not found"
-// @failure 500 "Server error"
-// @router /stacks/name/{name} [delete]
-func (handler *Handler) stackDeleteKubernetesByName(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	stackName, err := request.RetrieveRouteVariableValue(r, "name")
-	if err != nil {
-		return httperror.BadRequest("Invalid stack identifier route variable", err)
-	}
-
-	log.Debug().Msgf("Trying to delete Kubernetes stack %q", stackName)
-
-	securityContext, err := security.RetrieveRestrictedRequestContext(r)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve info from request context", err)
-	}
-
-	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
-	if err != nil {
-		return httperror.BadRequest("Invalid query parameter: endpointId", err)
-	}
-
-	namespace, err := request.RetrieveQueryParameter(r, "namespace", false)
-	if err != nil {
-		return httperror.BadRequest("Invalid query parameter: namespace", err)
-	}
-
-	stacks, err := handler.DataStore.Stack().StacksByName(stackName)
-	if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.InternalServerError("Unable to check for stack existence inside the database", err)
-	}
-	if stacks == nil {
-		return httperror.InternalServerError("Unable to find a stacks with the specified identifier name the database", err)
-	}
-
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find the endpoint associated to the stack inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find the endpoint associated to the stack inside the database", err)
-	}
-
-	log.Debug().Msgf("Trying to delete Kubernetes stack %q for endpoint `%d`", stackName, endpointID)
-
-	// check authorizations on all the stacks one by one
-	stacksToDelete := make([]portainer.Stack, 0)
-	for _, stack := range stacks {
-		// only delete stacks for the specified namespace
-		if stack.Namespace != namespace {
-			continue
-		}
-
-		isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID
-		if stack.Type != portainer.KubernetesStack {
-			return httperror.BadRequest("Only Kubernetes stacks can be deleted by name", errors.New("Only Kubernetes stacks can be deleted by name"))
-		}
-
-		if isOrphaned && !securityContext.IsAdmin {
-			return httperror.Forbidden("Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack"))
-		}
-
-		if !isOrphaned {
-			err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-			if err != nil {
-				return httperror.Forbidden("Permission denied to access endpoint", err)
-			}
-		}
-
-		canManage, err := handler.userCanManageStacks(securityContext, endpoint)
-		if err != nil {
-			return httperror.InternalServerError("Unable to verify user authorizations to validate stack deletion", err)
-		}
-		if !canManage {
-			errMsg := "stack deletion is disabled for non-admin users"
-			return httperror.Forbidden(errMsg, fmt.Errorf(errMsg))
-		}
-
-		stacksToDelete = append(stacksToDelete, stack)
-	}
-
-	log.Debug().Msgf("Trying to delete Kubernetes stacks `%v` for endpoint `%d`", stacksToDelete, endpointID)
-
-	errors := make([]error, 0)
-	// Delete all the stacks one by one
-	for _, stack := range stacksToDelete {
-		log.Debug().Msgf("Trying to delete Kubernetes stack id `%d`", stack.ID)
-
-		// stop scheduler updates of the stack before removal
-		if stack.AutoUpdate != nil {
-			deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
-		}
-
-		err = handler.deleteStack(securityContext.UserID, &stack, endpoint)
-		if err != nil {
-			log.Err(err).Msgf("Unable to delete Kubernetes stack `%d`", stack.ID)
-			errors = append(errors, err)
-			continue
-		}
-
-		err = handler.DataStore.Stack().Delete(stack.ID)
-		if err != nil {
-			errors = append(errors, err)
-			log.Err(err).Msgf("Unable to remove the stack `%d` from the database", stack.ID)
-			continue
-		}
-
-		err = handler.FileService.RemoveDirectory(stack.ProjectPath)
-		if err != nil {
-			errors = append(errors, err)
-			log.Warn().Err(err).Msg("Unable to remove stack files from disk")
-		}
-
-		log.Debug().Msgf("Kubernetes stack `%d` deleted", stack.ID)
-	}
-
-	if len(errors) > 0 {
-		return httperror.InternalServerError("Unable to delete some Kubernetes stack(s). Check Portainer logs for more details", nil)
-	}
-
-	return response.Empty(w)
-}

api/http/handler/stacks/update_kubernetes_stack.go

@@ -41,6 +41,9 @@ func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error
 	if govalidator.IsNull(payload.StackFileContent) {
 		return errors.New("Invalid stack file content")
 	}
+	if govalidator.IsNull(payload.StackName) {
+		return errors.New("Invalid stack name")
+	}
 	return nil
 }
 

api/http/handler/system/version.go

@@ -1,6 +1,7 @@
 package system
 
 import (
+	"encoding/json"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
@@ -10,7 +11,6 @@ import (
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
 type versionResponse struct {