chore(fs): change to a more meaningful function name

Co-authored-by: testa113 <testa113>

.eslintrc.yml

@@ -23,7 +23,7 @@ parserOptions:
     modules: true
 
 rules:
-  no-console: error
+  no-console: warn
   no-alert: error
   no-control-regex: 'off'
   no-empty: warn

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -93,6 +93,8 @@ body:
       description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
       multiple: false
       options:
+        - '2.19.3'
+        - '2.19.2'
         - '2.19.1'
         - '2.19.0'
         - '2.18.4'

.github/workflows/ci.yaml

@@ -0,0 +1,148 @@
+name: ci
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'develop'
+      - '!release/*'
+  pull_request:
+    branches:
+      - 'develop'
+      - 'release/*'
+      - 'feat/*'
+      - 'fix/*'
+      - 'refactor/*'
+
+env:
+  DOCKER_HUB_REPO: portainerci/portainer
+  NODE_ENV: testing
+  GO_VERSION: 1.21.3
+  NODE_VERSION: 18.x
+
+jobs:
+  build_images:
+    strategy:
+      matrix:
+        config:
+          - { platform: linux, arch: amd64 }
+          - { platform: linux, arch: arm64 }
+          - { platform: windows, arch: amd64, version: 1809 }
+          - { platform: windows, arch: amd64, version: ltsc2022 }
+    runs-on: arc-runner-set
+    steps:
+      - name: '[preparation] checkout the current branch'
+        uses: actions/checkout@v3.5.3
+        with:
+          ref: ${{ github.event.inputs.branch }}
+      - name: '[preparation] set up golang'
+        uses: actions/setup-go@v4.0.1
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: '[preparation] cache paths'
+        id: cache-dir-path
+        run: |
+          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
+          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
+          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
+      - name: '[preparation] cache go'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ steps.cache-dir-path.outputs.go-build-dir }}
+            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
+          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
+          enableCrossOsArchive: true
+      - name: '[preparation] set up node.js'
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: ''
+      - name: '[preparation] cache yarn'
+        uses: actions/cache@v3
+        with:
+          path: |
+            **/node_modules
+            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
+          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
+          enableCrossOsArchive: true
+      - name: '[preparation] set up qemu'
+        uses: docker/setup-qemu-action@v2
+      - name: '[preparation] set up docker context for buildx'
+        run: docker context create builders
+      - name: '[preparation] set up docker buildx'
+        uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+      - name: '[preparation] docker login'
+        uses: docker/login-action@v2.2.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: '[preparation] set the container image tag'
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+
+          if [ "${{ matrix.config.platform }}" == "windows" ]; then
+            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
+          else 
+            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
+          fi
+
+          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
+      - name: '[execution] build linux & windows portainer binaries'
+        run: |
+          export YARN_VERSION=$(yarn --version)
+          export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
+          export BUILDNUMBER=${GITHUB_RUN_NUMBER}
+          make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+      - name: '[execution] build and push docker images'
+        run: |
+          if [ "${{ matrix.config.platform }}" == "windows" ]; then
+            mv dist/portainer dist/portainer.exe
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+          else 
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+          fi
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+  build_manifests:
+    runs-on: arc-runner-set
+    needs: [build_images]
+    steps:
+      - name: '[preparation] docker login'
+        uses: docker/login-action@v2.2.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: '[preparation] set up docker context for buildx'
+        run: docker version && docker context create builders
+      - name: '[preparation] set up docker buildx'
+        uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+      - name: '[execution] build and push manifests'
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"

.github/workflows/lint.yml

@@ -12,6 +12,9 @@ on:
       - develop
       - release/*
 
+env:
+  GO_VERSION: 1.21.3
+
 jobs:
   run-linters:
     name: Run linters
@@ -25,7 +28,7 @@ jobs:
           cache: 'yarn'
       - uses: actions/setup-go@v4
         with:
-          go-version: 1.21.0
+          go-version: ${{ env.GO_VERSION }}
       - run: yarn --frozen-lockfile
       - name: Run linters
         uses: wearerequired/lint-action@v1
@@ -41,6 +44,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
+          version: v1.54.1
           working-directory: api
           args: --timeout=10m -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -5,6 +5,9 @@ on:
     - cron: '0 20 * * *'
   workflow_dispatch:
 
+env:
+  GO_VERSION: 1.21.3
+
 jobs:
   client-dependencies:
     name: Client Dependency Check
@@ -25,7 +28,7 @@ jobs:
         with:
           json: true
 
-      - name: upload scan result as develop artifact 
+      - name: upload scan result as develop artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
@@ -41,7 +44,7 @@ jobs:
           name: html-js-result-${{github.run_id}}
           path: js-result.html
 
-      - name: analyse vulnerabilities 
+      - name: analyse vulnerabilities
         id: set-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
@@ -58,10 +61,10 @@ jobs:
       - name: checkout repository
         uses: actions/checkout@master
 
-      - name: install Go 
+      - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21.0'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -74,7 +77,7 @@ jobs:
           yarn global add snyk
           snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: upload scan result as develop artifact 
+      - name: upload scan result as develop artifact
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-develop-result
@@ -102,35 +105,68 @@ jobs:
     if: >-
       github.ref == 'refs/heads/develop'
     outputs:
-      image: ${{ steps.set-matrix.outputs.image_result }}
+      image-trivy: ${{ steps.set-trivy-matrix.outputs.image_trivy_result }}
+      image-docker-scout: ${{ steps.set-docker-scout-matrix.outputs.image_docker_scout_result }}
     steps:
-      - name: scan vulnerabilities by Trivy 
+      - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
           args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
 
-      - name: upload image security scan result as artifact
+      - name: upload Trivy image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: develop scan report export to html
+      - name: develop Trivy scan report export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-trivy-result")
 
-      - name: upload html file as artifact
+      - name: upload html file as Trivy artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-${{github.run_id}}
-          path: image-result.html
+          path: image-trivy-result.html
 
-      - name: analyse vulnerabilities
-        id: set-matrix
+      - name: analyse vulnerabilities from Trivy
+        id: set-trivy-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
-          echo "image_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_trivy_result=${result}" >> $GITHUB_OUTPUT
+
+      - name: scan vulnerabilities by Docker Scout
+        uses: docker/scout-action@v1
+        continue-on-error: true
+        with:
+          command: cves
+          image: portainerci/portainer:develop
+          sarif-file: image-docker-scout.json
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} 
+
+      - name: upload Docker Scout image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-develop-result
+          path: image-docker-scout.json
+
+      - name: develop Docker Scout scan report export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
+
+      - name: upload html file as Docker Scout artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-${{github.run_id}}
+          path: image-docker-scout-result.html
+
+      - name: analyse vulnerabilities from Docker Scout
+        id: set-docker-scout-matrix
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=matrix)
+          echo "image_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Results
@@ -142,22 +178,26 @@ jobs:
       matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
-        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
+        image-trivy: ${{fromJson(needs.image-vulnerability.outputs.image-trivy)}}
+        image-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.image-docker-scout)}}
     steps:
       - name: display the results of js, Go, and image scan
         run: |
           echo "${{ matrix.js.status }}"
           echo "${{ matrix.go.status }}"
-          echo "${{ matrix.image.status }}"
+          echo "${{ matrix.image-trivy.status }}"
+          echo "${{ matrix.image-docker-scout.status }}"
           echo "${{ matrix.js.summary }}"
           echo "${{ matrix.go.summary }}"
-          echo "${{ matrix.image.summary }}"
+          echo "${{ matrix.image-trivy.summary }}"
+          echo "${{ matrix.image-docker-scout.summary }}"
 
-      - name: send message to Slack 
-        if: >- 
+      - name: send message to Slack
+        if: >-
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
-          matrix.image.status == 'failure'
+          matrix.image-trivy.status == 'failure' || 
+          matrix.image-docker-scout.status == 'failure' 
         uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |
@@ -193,7 +233,14 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
+                        "text": "*Image Trivy vulnerability check*: *${{ matrix.image-trivy.status }}*\n${{ matrix.image-trivy.summary }}\n"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Image Docker Scout vulnerability check*: *${{ matrix.image-docker-scout.status }}*\n${{ matrix.image-docker-scout.summary }}\n"
                       }
                     }
                   ]

.github/workflows/pr-security.yml

@@ -8,12 +8,15 @@ on:
     paths:
       - 'package.json'
       - 'go.mod'
-      - 'gruntfile.js'
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
       - '.github/workflows/pr-security.yml'
 
+env:
+  GO_VERSION: 1.21.3
+  NODE_VERSION: 18.x
+
 jobs:
   client-dependencies:
     name: Client Dependency Check
@@ -84,7 +87,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21.0'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -138,20 +141,21 @@ jobs:
       github.event.pull_request &&
       github.event.review.body == '/scan'
     outputs:
-      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
+      imagediff-trivy: ${{ steps.set-diff-trivy-matrix.outputs.image_diff_trivy_result }}
+      imagediff-docker-scout: ${{ steps.set-diff-docker-scout-matrix.outputs.image_diff_docker_scout_result }}
     steps:
       - name: checkout code
         uses: actions/checkout@master
 
-      - name: install Go 1.21.0
+      - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21.0'
+          go-version: ${{ env.GO_VERSION }}
 
-      - name: install Node.js 18.x
+      - name: install Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 18.x
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Install packages
         run: yarn --frozen-lockfile
@@ -167,26 +171,26 @@ jobs:
         with:
           context: .
           file: build/linux/Dockerfile
-          tags: trivy-portainer:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+          tags: local-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/local-portainer-image.tar
 
       - name: load docker image
         run: |
-          docker load --input /tmp/trivy-portainer-image.tar
+          docker load --input /tmp/local-portainer-image.tar
 
       - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress local-portainer:${{ github.sha }}
 
-      - name: upload image security scan result as artifact
+      - name: upload Trivy image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-feature-result
           path: image-trivy.json
 
-      - name: download artifacts from develop branch built by nightly scan
+      - name: download Trivy artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -198,21 +202,65 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: pr vs develop scan report comparison export to html
+      - name: pr vs develop Trivy scan report comparison export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-trivy-result")
 
-      - name: upload html file as artifact
+      - name: upload html file as Trivy artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-result.html
+          path: image-trivy-result.html
 
-      - name: analyse different vulnerabilities against develop branch
-        id: set-diff-matrix
+      - name: analyse different vulnerabilities against develop branch by Trivy
+        id: set-diff-trivy-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
-          echo "image_diff_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_diff_trivy_result=${result}" >> $GITHUB_OUTPUT
+
+      - name: scan vulnerabilities by Docker Scout
+        uses: docker/scout-action@v1
+        continue-on-error: true
+        with:
+          command: cves
+          image: local-portainer:${{ github.sha }}
+          sarif-file: image-docker-scout.json
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: upload Docker Scout image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-feature-result
+          path: image-docker-scout.json
+
+      - name: download Docker Scout artifacts from develop branch built by nightly scan
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./image-docker-scout.json ./image-docker-scout-feature.json
+          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./image-docker-scout.json ]]; then
+            mv ./image-docker-scout.json ./image-docker-scout-develop.json
+          else
+            echo "null" > ./image-docker-scout-develop.json
+          fi
+
+      - name: pr vs develop Docker Scout scan report comparison export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
+
+      - name: upload html file as Docker Scout artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-compare-to-develop-${{github.run_id}}
+          path: image-docker-scout-result.html
+
+      - name: analyse different vulnerabilities against develop branch by Docker Scout
+        id: set-diff-docker-scout-matrix
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=matrix)
+          echo "image_diff_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Result Against develop Branch
@@ -225,18 +273,22 @@ jobs:
       matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
-        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
+        imagediff-trivy: ${{fromJson(needs.image-vulnerability.outputs.imagediff-trivy)}}
+        imagediff-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.imagediff-docker-scout)}}
     steps:
       - name: check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff.status == 'failure'
+          matrix.imagediff-trivy.status == 'failure' ||
+          matrix.imagediff-docker-scout.status == 'failure'
         run: |
           echo "${{ matrix.jsdiff.status }}"
           echo "${{ matrix.godiff.status }}"
-          echo "${{ matrix.imagediff.status }}"
+          echo "${{ matrix.imagediff-trivy.status }}"
+          echo "${{ matrix.imagediff-docker-scout.status }}"
           echo "${{ matrix.jsdiff.summary }}"
           echo "${{ matrix.godiff.summary }}"
-          echo "${{ matrix.imagediff.summary }}"
+          echo "${{ matrix.imagediff-trivy.summary }}"
+          echo "${{ matrix.imagediff-docker-scout.summary }}"
           exit 1

.github/workflows/test.yaml

@@ -1,5 +1,11 @@
 name: Test
+
 on: push
+
+env:
+  GO_VERSION: 1.21.3
+  NODE_VERSION: 18.x
+
 jobs:
   test-client:
     runs-on: ubuntu-latest
@@ -8,18 +14,25 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '18'
+          node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
         run: make test-client ARGS="--maxWorkers=2"
   test-server:
+    strategy:
+      matrix:
+        config:
+          - { platform: linux, arch: amd64 }
+          - { platform: linux, arch: arm64 }
+          - { platform: windows, arch: amd64, version: 1809 }
+          - { platform: windows, arch: amd64, version: ltsc2022 }
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21.0
+          go-version: ${{ env.GO_VERSION }}
       - name: Run tests
         run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -7,6 +7,10 @@ on:
       - develop
       - 'release/*'
 
+env:
+  GO_VERSION: 1.21.3
+  NODE_VERSION: 18.x
+
 jobs:
   openapi-spec:
     runs-on: ubuntu-latest
@@ -15,13 +19,13 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.21.0'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: Download golang modules
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: '18'
+          node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

api/.golangci.yaml

@@ -10,19 +10,23 @@ linters:
     - exportloopref
 linters-settings:
   depguard:
-    list-type: denylist
-    include-go-root: true
-    packages:
-      - github.com/sirupsen/logrus
-      - golang.org/x/exp
-      - github.com/portainer/libcrypto
-      - github.com/portainer/libhttp
-    packages-with-error-message:
-      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
-    ignore-file-rules:
-      - '**/*_test.go'
-      - '**/base.go'
-      - '**/base_tx.go'
+    rules:
+      main:
+        deny:
+          - pkg: 'encoding/json'
+            desc: 'use github.com/segmentio/encoding/json'
+          - pkg: 'github.com/sirupsen/logrus'
+            desc: 'logging is allowed only by github.com/rs/zerolog'
+          - pkg: 'golang.org/x/exp'
+            desc: 'exp is not allowed'
+          - pkg: 'github.com/portainer/libcrypto'
+            desc: 'use github.com/portainer/portainer/pkg/libcrypto'
+          - pkg: 'github.com/portainer/libhttp'
+            desc: 'use github.com/portainer/portainer/pkg/libhttp'
+        files:
+          - '!**/*_test.go'
+          - '!**/base.go'
+          - '!**/base_tx.go'
 
 # errorlint is causing a typecheck error for some reason. The go compiler will report these
 # anyway, so ignore them from the linter

api/chisel/service.go

@@ -75,10 +75,11 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
 			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("start")
+			Msg("KeepTunnelAlive: start")
 
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
+
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
 		defer pingTicker.Stop()
 
@@ -91,13 +92,13 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 					log.Debug().
 						Int("endpoint_id", int(endpointID)).
 						Err(err).
-						Msg("ping agent")
+						Msg("KeepTunnelAlive: ping agent")
 				}
 			case <-maxAliveTicker.C:
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("tunnel keep alive timeout")
+					Msg("KeepTunnelAlive: tunnel keep alive timeout")
 
 				return
 			case <-ctx.Done():
@@ -105,7 +106,7 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("tunnel stop")
+					Msg("KeepTunnelAlive: tunnel stop")
 
 				return
 			}

api/cli/cli.go

@@ -49,7 +49,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),

api/cmd/portainer/main.go

@@ -200,7 +200,7 @@ func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
 	return apikey.NewAPIKeyService(datastore.APIKeyRepository(), datastore.User())
 }
 
-func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (dataservices.JWTService, error) {
+func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (portainer.JWTService, error) {
 	if userSessionTimeout == "" {
 		userSessionTimeout = portainer.DefaultUserSessionTimeout
 	}

api/database/boltdb/db.go

@@ -255,7 +255,7 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 			return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 		}
 
-		err := connection.UnmarshalObjectWithJsoniter(data, object)
+		err := connection.UnmarshalObject(data, object)
 		if err != nil {
 			return err
 		}

api/database/boltdb/export.go

@@ -1,10 +1,10 @@
 package boltdb
 
 import (
-	"encoding/json"
 	"time"
 
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 	bolt "go.etcd.io/bbolt"
 )
 

api/database/boltdb/json.go

@@ -1,34 +1,41 @@
 package boltdb
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
-	"encoding/json"
 	"fmt"
 	"io"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
+	"github.com/segmentio/encoding/json"
 )
 
 var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
 
 // MarshalObject encodes an object to binary format
-func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
+func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error) {
+	buf := &bytes.Buffer{}
+
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		data = []byte(v)
+		buf.WriteString(v)
 	} else {
-		data, err = json.Marshal(object)
-		if err != nil {
-			return data, err
+		enc := json.NewEncoder(buf)
+		enc.SetSortMapKeys(false)
+		enc.SetAppendNewline(false)
+
+		if err := enc.Encode(object); err != nil {
+			return nil, err
 		}
 	}
+
 	if connection.getEncryptionKey() == nil {
-		return data, nil
+		return buf.Bytes(), nil
 	}
-	return encrypt(data, connection.getEncryptionKey())
+
+	return encrypt(buf.Bytes(), connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
@@ -54,31 +61,6 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
 	return err
 }
 
-// UnmarshalObjectWithJsoniter decodes an object from binary data
-// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
-// decoding at the moment.
-func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
-	if connection.getEncryptionKey() != nil {
-		var err error
-		data, err = decrypt(data, connection.getEncryptionKey())
-		if err != nil {
-			return err
-		}
-	}
-	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
-	err := jsoni.Unmarshal(data, &object)
-	if err != nil {
-		if s, ok := object.(*string); ok {
-			*s = string(data)
-			return nil
-		}
-
-		return err
-	}
-
-	return nil
-}
-
 // mmm, don't have a KMS .... aes GCM seems the most likely from
 // https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
 

api/database/boltdb/tx.go

@@ -28,7 +28,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	return tx.conn.UnmarshalObjectWithJsoniter(value, object)
+	return tx.conn.UnmarshalObject(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -134,7 +134,7 @@ func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{},
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	return bucket.ForEach(func(k []byte, v []byte) error {
-		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		err := tx.conn.UnmarshalObject(v, obj)
 		if err == nil {
 			obj, err = appendFn(obj)
 		}
@@ -147,7 +147,7 @@ func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte
 	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
 
 	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
-		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		err := tx.conn.UnmarshalObject(v, obj)
 		if err != nil {
 			return err
 		}

api/dataservices/endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -144,6 +145,23 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
+func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.connection.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -122,6 +122,23 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
+func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.tx.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/endpointgroup/endpointgroup.go

@@ -5,10 +5,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "endpoint_groups"
-)
+const BucketName = "endpoint_groups"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {

api/dataservices/interface.go

@@ -2,7 +2,6 @@ package dataservices
 
 import (
 	"io"
-	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
@@ -95,6 +94,7 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
+		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)
@@ -132,15 +132,6 @@ type (
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
 	}
 
-	// JWTService represents a service for managing JWT tokens
-	JWTService interface {
-		GenerateToken(data *portainer.TokenData) (string, error)
-		GenerateTokenForOAuth(data *portainer.TokenData, expiryTime *time.Time) (string, error)
-		GenerateTokenForKubeconfig(data *portainer.TokenData) (string, error)
-		ParseAndVerifyToken(token string) (*portainer.TokenData, error)
-		SetUserSessionDuration(userSessionDuration time.Duration)
-	}
-
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
 		BaseCRUD[portainer.Registry, portainer.RegistryID]

api/dataservices/snapshot/snapshot.go

@@ -5,9 +5,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	BucketName = "snapshots"
-)
+const BucketName = "snapshots"
 
 type Service struct {
 	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]

api/dataservices/stack/stack.go

@@ -106,7 +106,6 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 	}
 
 	return nil, err
-
 }
 
 // RefreshableStacks returns stacks that are configured for a periodic update

api/dataservices/tag/tag.go

@@ -5,10 +5,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "tags"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "tags"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {

api/dataservices/tag/tx.go

@@ -22,7 +22,7 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTagFunc is a no-op inside a transaction
+// UpdateTagFunc is a no-op inside a transaction.
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }

api/dataservices/version/version.go

@@ -73,6 +73,10 @@ func (service *Service) IsUpdating() (bool, error) {
 
 // StoreIsUpdating store the database updating status.
 func (service *Service) StoreIsUpdating(isUpdating bool) error {
+	if isUpdating {
+		return service.connection.UpdateObject(BucketName, []byte(updatingKey), isUpdating)
+	}
+
 	return service.connection.DeleteObject(BucketName, []byte(updatingKey))
 }
 

api/datastore/backup.go

@@ -65,7 +65,7 @@ type BackupOptions struct {
 // - db rollback
 func getBackupRestoreOptions(backupDir string) *BackupOptions {
 	return &BackupOptions{
-		BackupDir:      backupDir, //connection.commonBackupDir(),
+		BackupDir:      backupDir,
 		BackupFileName: beforePortainerVersionUpgradeBackup,
 	}
 }
@@ -76,12 +76,12 @@ func (store *Store) Backup(version *models.Version) (string, error) {
 		return store.backupWithOptions(nil)
 	}
 
-	return store.backupWithOptions(&BackupOptions{
-		Version: version.SchemaVersion,
-	})
+	backupOptions := getBackupRestoreOptions(store.commonBackupDir())
+	backupOptions.Version = version.SchemaVersion
+	return store.backupWithOptions(backupOptions)
 }
 
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
+func (store *Store) setDefaultBackupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}
 	}
@@ -110,7 +110,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 
 	store.createBackupFolders()
 
-	options = store.setupOptions(options)
+	options = store.setDefaultBackupOptions(options)
 	dbPath := store.databasePath()
 
 	if err := store.Close(); err != nil {
@@ -139,20 +139,18 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 // - default: restore latest from current edition
 // - restore a specific
 func (store *Store) restoreWithOptions(options *BackupOptions) error {
-	options = store.setupOptions(options)
+	options = store.setDefaultBackupOptions(options)
 
 	// Check if backup file exist before restoring
 	_, err := os.Stat(options.BackupPath)
 	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
-
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist")
 		return err
 	}
 
 	err = store.Close()
 	if err != nil {
 		log.Error().Err(err).Msg("error while closing store before restore")
-
 		return err
 	}
 
@@ -170,7 +168,7 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 func (store *Store) removeWithOptions(options *BackupOptions) error {
 	log.Info().Msg("removing DB backup")
 
-	options = store.setupOptions(options)
+	options = store.setDefaultBackupOptions(options)
 	_, err := os.Stat(options.BackupPath)
 
 	if os.IsNotExist(err) {

api/datastore/init.go

@@ -53,7 +53,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			},
 			SnapshotInterval:         portainer.DefaultSnapshotInterval,
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-			TemplatesURL:             portainer.DefaultTemplatesURL,
+			TemplatesURL:             "",
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,

api/datastore/migrate_data.go

@@ -2,6 +2,7 @@ package datastore
 
 import (
 	"fmt"
+	"os"
 	"runtime/debug"
 
 	portainer "github.com/portainer/portainer/api"
@@ -117,6 +118,11 @@ func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version *models
 		return err
 	}
 
+	// Special test code to simulate a failure (used by migrate_data_test.go).  Do not remove...
+	if os.Getenv("PORTAINER_TEST_MIGRATE_FAIL") == "FAIL" {
+		panic("test migration failure")
+	}
+
 	err = store.VersionService.StoreIsUpdating(false)
 	if err != nil {
 		return errors.Wrap(err, "failed to update the store")

api/datastore/migrate_data_test.go

@@ -2,7 +2,6 @@ package datastore
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -10,11 +9,13 @@ import (
 	"strings"
 	"testing"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
+	"github.com/portainer/portainer/api/database/models"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/portainer/portainer/api/database/models"
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 // testVersion is a helper which tests current store version against wanted version
@@ -55,111 +56,108 @@ func TestMigrateData(t *testing.T) {
 		})
 	}
 
-	// t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-	// 	newStore, store, teardown := MustNewTestStore(t, true, false)
-	// 	defer teardown()
+	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
+		newStore, store := MustNewTestStore(t, true, false)
 
-	// 	if !newStore {
-	// 		t.Error("Expect a new DB")
-	// 	}
+		if !newStore {
+			t.Error("Expect a new DB")
+		}
 
-	// 	testVersion(store, portainer.APIVersion, t)
-	// 	store.Close()
+		testVersion(store, portainer.APIVersion, t)
+		store.Close()
 
-	// 	newStore, _ = store.Open()
-	// 	if newStore {
-	// 		t.Error("Expect store to NOT be new DB")
-	// 	}
-	// })
+		newStore, _ = store.Open()
+		if newStore {
+			t.Error("Expect store to NOT be new DB")
+		}
+	})
 
-	// tests := []struct {
-	// 	version         string
-	// 	expectedVersion string
-	// }{
-	// 	{version: "1.24.1", expectedVersion: portainer.APIVersion},
-	// 	{version: "2.0.0", expectedVersion: portainer.APIVersion},
-	// }
-	// for _, tc := range tests {
-	// 	_, store, teardown := MustNewTestStore(t, true, true)
-	// 	defer teardown()
+	tests := []struct {
+		version         string
+		expectedVersion string
+	}{
+		{version: "1.24.1", expectedVersion: portainer.APIVersion},
+		{version: "2.0.0", expectedVersion: portainer.APIVersion},
+	}
+	for _, tc := range tests {
+		_, store := MustNewTestStore(t, true, true)
 
-	// 	// Setup data
-	// 	v := models.Version{SchemaVersion: tc.version}
-	// 	store.VersionService.UpdateVersion(&v)
+		// Setup data
+		v := models.Version{SchemaVersion: tc.version, Edition: int(portainer.PortainerCE)}
+		store.VersionService.UpdateVersion(&v)
 
-	// 	// Required roles by migrations 22.2
-	// 	store.RoleService.Create(&portainer.Role{ID: 1})
-	// 	store.RoleService.Create(&portainer.Role{ID: 2})
-	// 	store.RoleService.Create(&portainer.Role{ID: 3})
-	// 	store.RoleService.Create(&portainer.Role{ID: 4})
+		// Required roles by migrations 22.2
+		store.RoleService.Create(&portainer.Role{ID: 1})
+		store.RoleService.Create(&portainer.Role{ID: 2})
+		store.RoleService.Create(&portainer.Role{ID: 3})
+		store.RoleService.Create(&portainer.Role{ID: 4})
 
-	// 	t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
-	// 		store.MigrateData()
-	// 		testVersion(store, tc.expectedVersion, t)
-	// 	})
+		t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
+			store.MigrateData()
+			testVersion(store, tc.expectedVersion, t)
+		})
 
-	// 	t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
-	// 		store.Rollback(true)
-	// 		store.Open()
-	// 		testVersion(store, tc.version, t)
-	// 	})
-	// }
+		t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
+			store.Rollback(true)
+			store.Open()
+			testVersion(store, tc.version, t)
+		})
+	}
 
-	// t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
+		_, store := MustNewTestStore(t, false, false)
 
-	// 	v := models.Version{SchemaVersion: "1.24.1"}
-	// 	store.VersionService.UpdateVersion(&v)
+		v := models.Version{SchemaVersion: "1.24.1", Edition: int(portainer.PortainerCE)}
+		store.VersionService.UpdateVersion(&v)
 
-	// 	store.MigrateData()
+		store.MigrateData()
 
-	// 	testVersion(store, v.SchemaVersion, t)
-	// })
+		testVersion(store, v.SchemaVersion, t)
+	})
 
-	// t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
+		_, store := MustNewTestStore(t, false, false)
 
-	// 	v := models.Version{SchemaVersion: "0.0.0"}
-	// 	store.VersionService.UpdateVersion(&v)
+		v := models.Version{SchemaVersion: "0.0.0", Edition: int(portainer.PortainerCE)}
+		store.VersionService.UpdateVersion(&v)
 
-	// 	store.MigrateData()
+		store.MigrateData()
 
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		options := store.setDefaultBackupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-	// 	if !isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should exist; file=%s", options.BackupPath)
-	// 	}
-	// })
+		if !isFileExist(options.BackupPath) {
+			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
+		}
+	})
 
-	// t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
+		_, store := MustNewTestStore(t, false, false)
 
-	// 	store.VersionService.StoreIsUpdating(true)
+		store.VersionService.StoreIsUpdating(true)
 
-	// 	store.MigrateData()
+		store.MigrateData()
 
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		options := store.setDefaultBackupOptions(getBackupRestoreOptions(store.commonBackupDir()))
 
-	// 	if isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-	// 	}
-	// })
+		if isFileExist(options.BackupPath) {
+			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+		}
+	})
 
-	// t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+	t.Run("MigrateData should recover and restore backup during migration critical failure", func(t *testing.T) {
+		os.Setenv("PORTAINER_TEST_MIGRATE_FAIL", "FAIL")
 
-	// 	store.MigrateData()
+		version := "2.15"
+		_, store := MustNewTestStore(t, true, false)
+		store.VersionService.UpdateVersion(&models.Version{SchemaVersion: version, Edition: int(portainer.PortainerCE)})
+		err := store.MigrateData()
+		if err == nil {
+			t.Errorf("Expect migration to fail")
+		}
 
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
-
-	// 	if isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-	// 	}
-	// })
+		store.Open()
+		testVersion(store, version, t)
+	})
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {

api/datastore/migrator/migrate_dbversion100.go

@@ -10,8 +10,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func (m *Migrator) migrateDockerDesktopExtentionSetting() error {
-	log.Info().Msg("updating docker desktop extention flag in settings")
+func (m *Migrator) migrateDockerDesktopExtensionSetting() error {
+	log.Info().Msg("updating docker desktop extension flag in settings")
 
 	isDDExtension := false
 	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {

api/datastore/migrator/migrate_dbversion110.go

@@ -0,0 +1,43 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
+)
+
+// updateAppTemplatesVersionForDB110 changes the templates URL to be empty if it was never changed
+// from the default value (version 2.0 URL)
+func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
+	log.Info().Msg("updating app templates url to v3.0")
+
+	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
+
+	settings, err := migrator.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if settings.TemplatesURL == version2URL || settings.TemplatesURL == portainer.DefaultTemplatesURL {
+		settings.TemplatesURL = ""
+	}
+
+	return migrator.settingsService.UpdateSettings(settings)
+}
+
+// setUseCacheForDB110 sets the user cache to true for all users
+func (migrator *Migrator) setUserCacheForDB110() error {
+	users, err := migrator.userService.ReadAll()
+	if err != nil {
+		return err
+	}
+
+	for i := range users {
+		user := &users[i]
+		user.UseCache = true
+		if err := migrator.userService.Update(user.ID, user); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion24.go

@@ -14,8 +14,10 @@ func (m *Migrator) updateSettingsToDB25() error {
 		return err
 	}
 
+	// to keep the same migration functionality as before 2.20.0, we need to set the templates URL to v2
+	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
 	if legacySettings.TemplatesURL == "" {
-		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
+		legacySettings.TemplatesURL = version2URL
 	}
 
 	legacySettings.UserSessionTimeout = portainer.DefaultUserSessionTimeout

api/datastore/migrator/migrate_dbversion31.go

@@ -245,7 +245,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 	return nil
 }
 
-func findResourcesToUpdateForDB32(dockerID string, volumesData volume.VolumeListOKBody, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
+func findResourcesToUpdateForDB32(dockerID string, volumesData volume.ListResponse, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
 	volumes := volumesData.Volumes
 	for _, volume := range volumes {
 		volumeName := volume.Name

api/datastore/migrator/migrator.go

@@ -225,9 +225,13 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.18", m.migrateDBVersionToDB90)
 	m.addMigrations("2.19",
 		m.convertSeedToPrivateKeyForDB100,
-		m.migrateDockerDesktopExtentionSetting,
+		m.migrateDockerDesktopExtensionSetting,
 		m.updateEdgeStackStatusForDB100,
 	)
+	m.addMigrations("2.20",
+		m.updateAppTemplatesVersionForDB110,
+		m.setUserCacheForDB110,
+	)
 
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.

api/datastore/services.go

@@ -1,7 +1,6 @@
 package datastore
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 
@@ -38,6 +37,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/webhook"
 
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 // Store defines the implementation of portainer.DataStore using

api/datastore/test_data/output_24_to_latest.json

@@ -46,12 +46,10 @@
       },
       "EdgeCheckinInterval": 0,
       "EdgeKey": "",
-      "EnableGPUManagement": false,
       "Gpus": [],
       "GroupId": 1,
       "Heartbeat": false,
       "Id": 1,
-      "IsEdgeDevice": false,
       "Kubernetes": {
         "Configuration": {
           "AllowNoneIngressClass": false,
@@ -101,8 +99,7 @@
       "TeamAccessPolicies": {},
       "Type": 1,
       "URL": "unix:///var/run/docker.sock",
-      "UserAccessPolicies": {},
-      "UserTrusted": false
+      "UserAccessPolicies": {}
     }
   ],
   "registries": [
@@ -124,8 +121,7 @@
       "Name": "canister.io",
       "Password": "MjWbx8A6YK7cw7",
       "Quay": {
-        "OrganisationName": "",
-        "UseOrganisation": false
+        "OrganisationName": ""
       },
       "RegistryAccesses": {
         "1": {
@@ -584,11 +580,8 @@
     "AllowHostNamespaceForRegularUsers": true,
     "AllowPrivilegedModeForRegularUsers": true,
     "AllowStackManagementForRegularUsers": true,
-    "AllowVolumeBrowserForRegularUsers": false,
     "AuthenticationMethod": 1,
     "BlackListedLabels": [],
-    "DisplayDonationHeader": false,
-    "DisplayExternalContributors": false,
     "Edge": {
       "AsyncMode": false,
       "CommandInterval": 0,
@@ -598,15 +591,16 @@
     "EdgeAgentCheckinInterval": 5,
     "EdgePortainerUrl": "",
     "EnableEdgeComputeFeatures": false,
-    "EnableHostManagementFeatures": false,
     "EnableTelemetry": true,
     "EnforceEdgeID": false,
     "FeatureFlagSettings": null,
+    "GlobalDeploymentOptions": {
+      "hideStacksFunctionality": false
+    },
     "HelmRepositoryURL": "https://charts.bitnami.com/bitnami",
     "InternalAuthSettings": {
       "RequiredPasswordLength": 12
     },
-    "IsDockerDesktopExtension": false,
     "KubeconfigExpiry": "0",
     "KubectlShellImage": "portainer/kubectl-shell",
     "LDAPSettings": {
@@ -651,7 +645,7 @@
     },
     "ShowKomposeBuildOption": false,
     "SnapshotInterval": "5m",
-    "TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
+    "TemplatesURL": "",
     "TrustOnFirstConnect": false,
     "UserSessionTimeout": "8h",
     "fdoConfiguration": {
@@ -909,7 +903,7 @@
         "color": ""
       },
       "TokenIssueAt": 0,
-      "UserTheme": "",
+      "UseCache": true,
       "Username": "admin"
     },
     {
@@ -939,11 +933,11 @@
         "color": ""
       },
       "TokenIssueAt": 0,
-      "UserTheme": "",
+      "UseCache": true,
       "Username": "prabhat"
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":2,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/snapshot.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	_container "github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
 	portainer "github.com/portainer/portainer/api"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -174,7 +174,12 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 				if !snapshot.Swarm {
 					return err
 				} else {
-					log.Info().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
+					if !strings.Contains(err.Error(), "No such container") {
+						return err
+					}
+					// It is common to have containers running on different Swarm nodes,
+					// so we just log the error in the debug level
+					log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
 				}
 			} else {
 				var gpuOptions *_container.DeviceRequest = nil
@@ -240,7 +245,7 @@ func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) erro
 }
 
 func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
-	volumes, err := cli.VolumeList(context.Background(), filters.Args{})
+	volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
 	if err != nil {
 		return err
 	}

api/exec/swarm_stack.go

@@ -2,7 +2,6 @@ package exec
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -15,7 +14,9 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/registryutils"
 	"github.com/portainer/portainer/api/stacks/stackutils"
+
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 // SwarmStackManager represents a service for managing stacks.

api/filesystem/filesystem.go

@@ -2,7 +2,6 @@ package filesystem
 
 import (
 	"bytes"
-	"encoding/json"
 	"encoding/pem"
 	"errors"
 	"fmt"
@@ -15,6 +14,7 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 const (
@@ -302,6 +302,38 @@ func (service *Service) UpdateStoreStackFileFromBytes(stackIdentifier, fileName
 	return service.wrapFileStore(stackStorePath), nil
 }
 
+// UpdateStoreStackFileFromBytesByVersion makes stack file backup and updates a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) UpdateStoreStackFileFromBytesByVersion(stackIdentifier, fileName string, version int, commitHash string, data []byte) (string, error) {
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	if commitHash != "" {
+		versionStr = commitHash
+	}
+
+	if versionStr != "" {
+		stackStorePath = JoinPaths(stackStorePath, versionStr)
+	}
+
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	err := service.createBackupFileInStore(composeFilePath)
+	if err != nil {
+		return "", err
+	}
+
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return service.wrapFileStore(stackStorePath), nil
+}
+
 // RemoveStackFileBackup removes the stack file backup in the ComposeStorePath.
 func (service *Service) RemoveStackFileBackup(stackIdentifier, fileName string) error {
 	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
@@ -900,9 +932,13 @@ func FileExists(filePath string) (bool, error) {
 
 // SafeCopyDirectory copies a directory from src to dst in a safe way.
 func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
+	return safeMoveDirectory(originalPath, newPath, CopyDir)
+}
+
+func safeMoveDirectory(src, dst string, copyDirFunc func(string, string, bool) error) error {
 	// 1. Backup the source directory to a different folder
-	backupDir := fmt.Sprintf("%s-%s", filepath.Dir(originalPath), "backup")
-	err := MoveDirectory(originalPath, backupDir)
+	backupDir := fmt.Sprintf("%s-%s", src, "backup")
+	err := moveDirectoryAndOverwrite(src, backupDir)
 	if err != nil {
 		return fmt.Errorf("failed to backup source directory: %w", err)
 	}
@@ -910,25 +946,25 @@ func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
 	defer func() {
 		if err != nil {
 			// If an error occurred, rollback the backup directory
-			restoreErr := restoreBackup(originalPath, backupDir)
+			restoreErr := restoreBackup(src, backupDir)
 			if restoreErr != nil {
 				log.Warn().Err(restoreErr).Msg("failed to restore backup during creating versioning folder")
 			}
 		}
+
+		// 3. Delete the backup directory
+		err = os.RemoveAll(backupDir)
+		if err != nil {
+			log.Warn().Err(err).Msg("failed to remove backup directory")
+		}
 	}()
 
 	// 2. Copy the backup directory to the destination directory
-	err = CopyDir(backupDir, newPath, false)
+	err = copyDirFunc(backupDir, dst, false)
 	if err != nil {
 		return fmt.Errorf("failed to copy backup directory to destination directory: %w", err)
 	}
 
-	// 3. Delete the backup directory
-	err = os.RemoveAll(backupDir)
-	if err != nil {
-		return fmt.Errorf("failed to delete backup directory: %w", err)
-	}
-
 	return nil
 }
 
@@ -965,6 +1001,27 @@ func MoveDirectory(originalPath, newPath string) error {
 	return os.Rename(originalPath, newPath)
 }
 
+func moveDirectoryAndOverwrite(originalPath, newPath string) error {
+	if _, err := os.Stat(originalPath); err != nil {
+		return err
+	}
+
+	alreadyExists, err := FileExists(newPath)
+	if err != nil {
+		return err
+	}
+
+	if alreadyExists {
+		log.Info().Msgf("Target path already exists, removing it: %s", newPath)
+		err := os.RemoveAll(newPath)
+		if err != nil {
+			log.Warn().Err(err).Msgf("Failed to remove existing target path: %s", newPath)
+		}
+	}
+
+	return os.Rename(originalPath, newPath)
+}
+
 // StoreFDOProfileFileFromBytes creates a subfolder in the FDOProfileStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) StoreFDOProfileFileFromBytes(fdoProfileIdentifier string, data []byte) (string, error) {

api/filesystem/filesystem_test.go

@@ -1,6 +1,7 @@
 package filesystem
 
 import (
+	"errors"
 	"os"
 	"path"
 	"testing"
@@ -20,3 +21,62 @@ func createService(t *testing.T) *Service {
 
 	return service
 }
+
+func prepareDir(t *testing.T, dir string) error {
+	err := os.MkdirAll(path.Join(dir, "data", "compose", "1"), 0755)
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Create(path.Join(dir, "data", "compose", "1", "docker-compose.yml"))
+	if err != nil {
+		return err
+	}
+
+	file.Close()
+	return nil
+}
+
+func TestSafeSafeMoveDirectory(t *testing.T) {
+	t.Run("able to migrate original folder when the backup folder already exists", func(t *testing.T) {
+		testDir := path.Join(t.TempDir(), t.Name())
+		err := prepareDir(t, testDir)
+		assert.NoError(t, err, "prepareDir should not fail")
+		defer os.RemoveAll(testDir)
+
+		err = os.MkdirAll(path.Join(testDir, "data", "compose", "1-backup"), 0755)
+		assert.NoError(t, err, "create backupdir should not fail")
+		assert.DirExists(t, path.Join(testDir, "data", "compose", "1-backup"), "backupdir should exist")
+
+		src := path.Join(testDir, "data", "compose", "1")
+		dst := path.Join(testDir, "data", "compose", "1", "v1")
+		err = safeMoveDirectory(src, dst, CopyDir)
+		assert.NoError(t, err, "safeMoveDirectory should not fail")
+
+		assert.FileExists(t, path.Join(testDir, "data", "compose", "1", "v1", "docker-compose.yml"), "docker-compose.yml should be migrated")
+		assert.NoDirExists(t, path.Join(testDir, "data", "compose", "1-backup"), "backupdir should not exist")
+
+	})
+
+	t.Run("original folder can be restored if error occurs", func(t *testing.T) {
+		testDir := path.Join(t.TempDir(), t.Name())
+		err := prepareDir(t, testDir)
+		assert.NoError(t, err, "prepareDir should not fail")
+		defer os.RemoveAll(testDir)
+
+		src := path.Join(testDir, "data", "compose", "1")
+		dst := path.Join(testDir, "data", "compose", "1", "v1")
+
+		copyDirFunc := func(string, string, bool) error {
+			return errors.New("mock copy dir error")
+		}
+		err = safeMoveDirectory(src, dst, copyDirFunc)
+		assert.Error(t, err, "safeMoveDirectory should fail")
+
+		assert.FileExists(t, path.Join(testDir, "data", "compose", "1", "docker-compose.yml"), "original folder should be restored")
+		assert.NoDirExists(t, path.Join(testDir, "data", "compose", "1", "v1"), "the target folder should not exist")
+		assert.NoFileExists(t, path.Join(testDir, "data", "compose", "1", "v1", "docker-compose.yml"), "docker-compose.yml should not be migrated")
+		assert.NoDirExists(t, path.Join(testDir, "data", "compose", "1-backup"), "backupdir should not exist")
+	})
+
+}

api/filesystem/serialize_per_dev_configs.go

@@ -80,10 +80,14 @@ func shouldIncludeEntry(dirEntry DirEntry, deviceName, configPath string, filter
 		return shouldIncludeFile(dirEntry, deviceName, configPath)
 	}
 
-	// Include:
-	// dir entry A/B/C/<deviceName>
-	// all entries A/B/C/<deviceName>/*
-	return shouldIncludeDir(dirEntry, deviceName, configPath)
+	if filterType == portainer.PerDevConfigsTypeDir {
+		// Include:
+		// dir entry A/B/C/<deviceName>
+		// all entries A/B/C/<deviceName>/*
+		return shouldIncludeDir(dirEntry, deviceName, configPath)
+	}
+
+	return false
 }
 
 func isInConfigRootDir(dirEntry DirEntry, configPath string) bool {

api/git/azure.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,12 +10,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
 	gittypes "github.com/portainer/portainer/api/git/types"
 
+	"github.com/go-git/go-git/v5/plumbing/filemode"
 	"github.com/pkg/errors"
+	"github.com/segmentio/encoding/json"
 )
 
 const (

api/hostmanagement/openamt/authorization.go

@@ -2,12 +2,13 @@ package openamt
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type authenticationResponse struct {

api/hostmanagement/openamt/configCIRA.go

@@ -2,7 +2,6 @@ package openamt
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -11,6 +10,8 @@ import (
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type CIRAConfig struct {

api/hostmanagement/openamt/configDevice.go

@@ -1,11 +1,12 @@
 package openamt
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type Device struct {

api/hostmanagement/openamt/configDomain.go

@@ -1,11 +1,12 @@
 package openamt
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type (

api/hostmanagement/openamt/configProfile.go

@@ -1,11 +1,12 @@
 package openamt
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type (

api/hostmanagement/openamt/deviceActions.go

@@ -1,12 +1,13 @@
 package openamt
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type ActionResponse struct {

api/hostmanagement/openamt/deviceFeatures.go

@@ -1,11 +1,12 @@
 package openamt
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 func (service *Service) enableDeviceFeatures(configuration portainer.OpenAMTConfiguration, deviceGUID string, features portainer.OpenAMTDeviceEnabledFeatures) error {

api/hostmanagement/openamt/openamt.go

@@ -2,7 +2,6 @@ package openamt
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -12,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 
+	"github.com/segmentio/encoding/json"
 	"golang.org/x/sync/errgroup"
 )
 

api/http/client/client.go

@@ -2,7 +2,6 @@ package client
 
 import (
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,6 +13,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 var errInvalidResponseStatus = errors.New("invalid response status (expecting 200)")

api/http/csrf/csrf.go

@@ -0,0 +1,62 @@
+package csrf
+
+import (
+	"crypto/rand"
+	"fmt"
+	"net/http"
+
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+
+	gorillacsrf "github.com/gorilla/csrf"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/urfave/negroni"
+)
+
+func WithProtect(handler http.Handler) (http.Handler, error) {
+	handler = withSendCSRFToken(handler)
+
+	token := make([]byte, 32)
+	_, err := rand.Read(token)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
+	}
+
+	handler = gorillacsrf.Protect([]byte(token), gorillacsrf.Path("/"))(handler)
+
+	return withSkipCSRF(handler), nil
+}
+
+func withSendCSRFToken(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		sw := negroni.NewResponseWriter(w)
+
+		sw.Before(func(sw negroni.ResponseWriter) {
+			statusCode := sw.Status()
+			if statusCode >= 200 && statusCode < 300 {
+				csrfToken := gorillacsrf.Token(r)
+				sw.Header().Set("X-CSRF-Token", csrfToken)
+			}
+		})
+
+		handler.ServeHTTP(sw, r)
+
+	})
+}
+
+func withSkipCSRF(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		skip, err := security.ShouldSkipCSRFCheck(r)
+		if err != nil {
+			httperror.WriteError(w, http.StatusForbidden, err.Error(), err)
+			return
+		}
+
+		if skip {
+			r = gorillacsrf.UnsafeSkipCheck(r)
+		}
+
+		handler.ServeHTTP(w, r)
+	})
+}

api/http/handler/auth/authenticate.go

@@ -6,6 +6,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -74,7 +75,7 @@ func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *h
 		if settings.AuthenticationMethod == portainer.AuthenticationInternal ||
 			settings.AuthenticationMethod == portainer.AuthenticationOAuth ||
 			(settings.AuthenticationMethod == portainer.AuthenticationLDAP && !settings.LDAPSettings.AutoCreateUsers) {
-			return &httperror.HandlerError{StatusCode: http.StatusUnprocessableEntity, Message: "Invalid credentials", Err: httperrors.ErrUnauthorized}
+			return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
 		}
 	}
 
@@ -83,14 +84,14 @@ func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *h
 	}
 
 	if settings.AuthenticationMethod == portainer.AuthenticationOAuth {
-		return &httperror.HandlerError{StatusCode: http.StatusUnprocessableEntity, Message: "Only initial admin is allowed to login without oauth", Err: httperrors.ErrUnauthorized}
+		return httperror.NewError(http.StatusUnprocessableEntity, "Only initial admin is allowed to login without oauth", httperrors.ErrUnauthorized)
 	}
 
 	if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
 		return handler.authenticateLDAP(rw, user, payload.Username, payload.Password, &settings.LDAPSettings)
 	}
 
-	return &httperror.HandlerError{StatusCode: http.StatusUnprocessableEntity, Message: "Login method is not supported", Err: httperrors.ErrUnauthorized}
+	return httperror.NewError(http.StatusUnprocessableEntity, "Login method is not supported", httperrors.ErrUnauthorized)
 }
 
 func isUserInitialAdmin(user *portainer.User) bool {
@@ -100,7 +101,7 @@ func isUserInitialAdmin(user *portainer.User) bool {
 func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portainer.User, password string) *httperror.HandlerError {
 	err := handler.CryptoService.CompareHashAndData(user.Password, password)
 	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusUnprocessableEntity, Message: "Invalid credentials", Err: httperrors.ErrUnauthorized}
+		return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
 	}
 
 	forceChangePassword := !handler.passwordStrengthChecker.Check(password)
@@ -142,12 +143,15 @@ func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User,
 }
 
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
-	token, err := handler.JWTService.GenerateToken(tokenData)
+	token, expirationTime, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
 		return httperror.InternalServerError("Unable to generate JWT token", err)
 	}
 
+	security.AddAuthCookie(w, token, expirationTime)
+
 	return response.JSON(w, &authenticateResponse{JWT: token})
+
 }
 
 func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settings *portainer.LDAPSettings) error {

api/http/handler/auth/handler.go

@@ -18,12 +18,13 @@ type Handler struct {
 	*mux.Router
 	DataStore                   dataservices.DataStore
 	CryptoService               portainer.CryptoService
-	JWTService                  dataservices.JWTService
+	JWTService                  portainer.JWTService
 	LDAPService                 portainer.LDAPService
 	OAuthService                portainer.OAuthService
 	ProxyManager                *proxy.Manager
 	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
 	passwordStrengthChecker     security.PasswordStrengthChecker
+	bouncer                     security.BouncerService
 }
 
 // NewHandler creates a handler to manage authentication operations.
@@ -31,6 +32,7 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 	h := &Handler{
 		Router:                  mux.NewRouter(),
 		passwordStrengthChecker: passwordStrengthChecker,
+		bouncer:                 bouncer,
 	}
 
 	h.Handle("/auth/oauth/validate",
@@ -38,7 +40,6 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
 	h.Handle("/auth",
 		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
 	h.Handle("/auth/logout",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
-
+		bouncer.PublicAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
 	return h
 }

api/http/handler/auth/logout.go

@@ -4,13 +4,14 @@ import (
 	"net/http"
 
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/logoutcontext"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
 // @id Logout
 // @summary Logout
-// @description **Access policy**: authenticated
+// @description **Access policy**: public
 // @security ApiKeyAuth
 // @security jwt
 // @tags auth
@@ -18,12 +19,14 @@ import (
 // @failure 500 "Server error"
 // @router /auth/logout [post]
 func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
+	tokenData, _ := handler.bouncer.CookieAuthLookup(r)
+
+	if tokenData != nil {
+		handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
+		logoutcontext.Cancel(tokenData.Token)
 	}
 
-	handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
+	security.RemoveAuthCookie(w)
 
 	return response.Empty(w)
 }

api/http/handler/customtemplates/customtemplate_create.go

@@ -1,7 +1,6 @@
 package customtemplates
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -21,6 +20,7 @@ import (
 
 	"github.com/asaskevich/govalidator"
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 func (handler *Handler) customTemplateCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -103,6 +103,8 @@ type customTemplateFromFileContentPayload struct {
 	FileContent string `validate:"required"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// EdgeTemplate indicates if this template purpose for Edge Stack
+	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) error {
@@ -160,15 +162,16 @@ func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*p
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:          portainer.CustomTemplateID(customTemplateID),
-		Title:       payload.Title,
-		EntryPoint:  filesystem.ComposeFileDefaultName,
-		Description: payload.Description,
-		Note:        payload.Note,
-		Platform:    (payload.Platform),
-		Type:        (payload.Type),
-		Logo:        payload.Logo,
-		Variables:   payload.Variables,
+		ID:           portainer.CustomTemplateID(customTemplateID),
+		Title:        payload.Title,
+		EntryPoint:   filesystem.ComposeFileDefaultName,
+		Description:  payload.Description,
+		Note:         payload.Note,
+		Platform:     (payload.Platform),
+		Type:         (payload.Type),
+		Logo:         payload.Logo,
+		Variables:    payload.Variables,
+		EdgeTemplate: payload.EdgeTemplate,
 	}
 
 	templateFolder := strconv.Itoa(customTemplateID)
@@ -218,6 +221,8 @@ type customTemplateFromGitRepositoryPayload struct {
 	TLSSkipVerify bool `example:"false"`
 	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
 	IsComposeFormat bool `example:"false"`
+	// EdgeTemplate indicates if this template purpose for Edge Stack
+	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -264,7 +269,7 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 // @success 200 {object} portainer.CustomTemplate
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /custom_templates/repository [post]
+// @router /custom_templates/create/repository [post]
 func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (*portainer.CustomTemplate, error) {
 	var payload customTemplateFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
@@ -283,6 +288,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		Logo:            payload.Logo,
 		Variables:       payload.Variables,
 		IsComposeFormat: payload.IsComposeFormat,
+		EdgeTemplate:    payload.EdgeTemplate,
 	}
 
 	getProjectPath := func() string {
@@ -367,6 +373,8 @@ type customTemplateFromFileUploadPayload struct {
 	FileContent []byte
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// EdgeTemplate indicates if this template purpose for Edge Stack
+	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) error {
@@ -419,8 +427,15 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 		if err != nil {
 			return errors.New("Invalid variables. Ensure that the variables are valid JSON")
 		}
-		return validateVariablesDefinitions(payload.Variables)
+		err = validateVariablesDefinitions(payload.Variables)
+		if err != nil {
+			return err
+		}
 	}
+
+	edgeTemplate, _ := request.RetrieveBooleanMultiPartFormValue(r, "EdgeTemplate", true)
+	payload.EdgeTemplate = edgeTemplate
+
 	return nil
 }
 
@@ -444,7 +459,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 // @success 200 {object} portainer.CustomTemplate
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /custom_templates/file [post]
+// @router /custom_templates/create/file [post]
 func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*portainer.CustomTemplate, error) {
 	payload := &customTemplateFromFileUploadPayload{}
 	err := payload.Validate(r)
@@ -454,15 +469,16 @@ func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*po
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:          portainer.CustomTemplateID(customTemplateID),
-		Title:       payload.Title,
-		Description: payload.Description,
-		Note:        payload.Note,
-		Platform:    payload.Platform,
-		Type:        payload.Type,
-		Logo:        payload.Logo,
-		EntryPoint:  filesystem.ComposeFileDefaultName,
-		Variables:   payload.Variables,
+		ID:           portainer.CustomTemplateID(customTemplateID),
+		Title:        payload.Title,
+		Description:  payload.Description,
+		Note:         payload.Note,
+		Platform:     payload.Platform,
+		Type:         payload.Type,
+		Logo:         payload.Logo,
+		EntryPoint:   filesystem.ComposeFileDefaultName,
+		Variables:    payload.Variables,
+		EdgeTemplate: payload.EdgeTemplate,
 	}
 
 	templateFolder := strconv.Itoa(customTemplateID)

api/http/handler/customtemplates/customtemplate_git_fetch_test.go

@@ -2,8 +2,6 @@ package customtemplates
 
 import (
 	"bytes"
-	"encoding/json"
-	"fmt"
 	"io"
 	"io/fs"
 	"net/http"
@@ -19,7 +17,10 @@ import (
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
+
+	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -75,7 +76,7 @@ func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect stri
 	}
 
 	req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
-	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
+	testhelpers.AddTestSecurityCookie(req, jwt)
 
 	rr := httptest.NewRecorder()
 	h.ServeHTTP(rr, req)
@@ -131,8 +132,8 @@ func Test_customTemplateGitFetch(t *testing.T) {
 	h := NewHandler(requestBouncer, store, fileService, gitService)
 
 	// generate two standard users' tokens
-	jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
-	jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
+	jwt1, _, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
+	jwt2, _, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
 
 	t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
 		singleAPIRequest(h, jwt1, is, "abcdefg")

api/http/handler/customtemplates/customtemplate_update.go

@@ -59,6 +59,8 @@ type customTemplateUpdatePayload struct {
 	TLSSkipVerify bool `example:"false"`
 	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
 	IsComposeFormat bool `example:"false"`
+	// EdgeTemplate indicates if this template purpose for Edge Stack
+	EdgeTemplate bool `example:"false"`
 }
 
 func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
@@ -161,6 +163,7 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.Type = payload.Type
 	customTemplate.Variables = payload.Variables
 	customTemplate.IsComposeFormat = payload.IsComposeFormat
+	customTemplate.EdgeTemplate = payload.EdgeTemplate
 
 	if payload.RepositoryURL != "" {
 		if !govalidator.IsURL(payload.RepositoryURL) {

api/http/handler/docker/handler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/http/handler/docker/containers"
+	"github.com/portainer/portainer/api/http/handler/docker/images"
 	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
@@ -45,6 +46,9 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 
 	containersHandler := containers.NewHandler("/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
 	endpointRouter.PathPrefix("/containers").Handler(containersHandler)
+
+	imagesHandler := images.NewHandler("/{id}/images", bouncer, dockerClientFactory)
+	endpointRouter.PathPrefix("/images").Handler(imagesHandler)
 	return h
 }
 

api/http/handler/docker/images/handler.go

@@ -0,0 +1,32 @@
+package images
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/security"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+
+	"github.com/gorilla/mux"
+)
+
+type Handler struct {
+	*mux.Router
+	dockerClientFactory *client.ClientFactory
+	bouncer             security.BouncerService
+}
+
+// NewHandler creates a handler to process non-proxied requests to docker APIs directly.
+func NewHandler(routePrefix string, bouncer security.BouncerService, dockerClientFactory *client.ClientFactory) *Handler {
+	h := &Handler{
+		Router:              mux.NewRouter(),
+		dockerClientFactory: dockerClientFactory,
+		bouncer:             bouncer,
+	}
+
+	router := h.PathPrefix(routePrefix).Subrouter()
+	router.Use(bouncer.AuthenticatedAccess)
+
+	router.Handle("", httperror.LoggerHandler(h.imagesList)).Methods(http.MethodGet)
+	return h
+}

api/http/handler/docker/images/images_list.go

@@ -0,0 +1,86 @@
+package images
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/portainer/portainer/api/http/handler/docker/utils"
+	"github.com/portainer/portainer/api/internal/set"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+type ImageResponse struct {
+	Created  int64    `json:"created"`
+	NodeName string   `json:"nodeName"`
+	ID       string   `json:"id"`
+	Size     int64    `json:"size"`
+	Tags     []string `json:"tags"`
+
+	// Used is true if the image is used by at least one container
+	// supplied only when withUsage is true
+	Used bool `json:"used"`
+}
+
+// @id dockerImagesList
+// @summary Fetch images
+// @description
+// @description **Access policy**:
+// @tags docker
+// @security jwt
+// @param environmentId path int true "Environment identifier"
+// @param withUsage query boolean false "Include image usage information"
+// @produce json
+// @success 200 {array} ImageResponse "Success"
+// @failure 400 "Bad request"
+// @failure 500 "Internal server error"
+// @router /docker/{environmentId}/images [get]
+func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	cli, httpErr := utils.GetClient(r, handler.dockerClientFactory)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	images, err := cli.ImageList(r.Context(), types.ImageListOptions{})
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve Docker images", err)
+	}
+
+	withUsage, err := request.RetrieveBooleanQueryParameter(r, "withUsage", true)
+	if err != nil {
+		return httperror.BadRequest("Invalid query parameter: withUsage", err)
+	}
+
+	imageUsageSet := set.Set[string]{}
+	if withUsage {
+		containers, err := cli.ContainerList(r.Context(), types.ContainerListOptions{})
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Docker containers", err)
+		}
+
+		for _, container := range containers {
+			imageUsageSet.Add(container.ImageID)
+		}
+	}
+
+	imagesList := make([]ImageResponse, len(images))
+	for i, image := range images {
+		if (image.RepoTags == nil || len(image.RepoTags) == 0) && (image.RepoDigests != nil && len(image.RepoDigests) > 0) {
+			for _, repoDigest := range image.RepoDigests {
+				image.RepoTags = append(image.RepoTags, repoDigest[0:strings.Index(repoDigest, "@")]+":<none>")
+			}
+		}
+
+		imagesList[i] = ImageResponse{
+			Created: image.Created,
+			ID:      image.ID,
+			Size:    image.Size,
+			Tags:    image.RepoTags,
+			Used:    imageUsageSet.Contains(image.ID),
+		}
+	}
+
+	return response.JSON(w, imagesList)
+}

api/http/handler/docker/utils/get_client.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	"net/http"
+
+	dockerclient "github.com/docker/docker/client"
+	portainer "github.com/portainer/portainer/api"
+	prclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/http/middlewares"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+)
+
+// GetClient returns a Docker client based on the request context
+func GetClient(r *http.Request, dockerClientFactory *prclient.ClientFactory) (*dockerclient.Client, *httperror.HandlerError) {
+	endpoint, err := middlewares.FetchEndpoint(r)
+	if err != nil {
+		return nil, httperror.NotFound("Unable to find an environment on request context", err)
+	}
+
+	agentTargetHeader := r.Header.Get(portainer.PortainerAgentTargetHeader)
+
+	cli, err := dockerClientFactory.CreateClient(endpoint, agentTargetHeader, nil)
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to connect to the Docker daemon", err)
+	}
+
+	return cli, nil
+}

api/http/handler/edgegroups/associated_endpoints.go

@@ -49,6 +49,24 @@ func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, p
 	return results, nil
 }
 
+func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs []portainer.EndpointID) ([]portainer.EndpointID, error) {
+	results := []portainer.EndpointID{}
+	for _, endpointID := range endpointIDs {
+		endpoint, err := tx.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return nil, err
+		}
+
+		if !endpoint.UserTrusted {
+			continue
+		}
+
+		results = append(results, endpoint.ID)
+	}
+
+	return results, nil
+}
+
 func mapEndpointGroupToEndpoints(endpoints []portainer.Endpoint) map[portainer.EndpointGroupID]endpointSetType {
 	groupEndpoints := map[portainer.EndpointGroupID]endpointSetType{}
 

api/http/handler/edgegroups/edgegroup_delete.go

@@ -59,7 +59,7 @@ func deleteEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) erro
 	for _, edgeStack := range edgeStacks {
 		for _, groupID := range edgeStack.EdgeGroups {
 			if groupID == ID {
-				return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge stack", errors.New("edge group is used by an Edge stack"))
+				return httperror.Conflict("Edge group is used by an Edge stack", errors.New("edge group is used by an Edge stack"))
 			}
 		}
 	}
@@ -72,7 +72,7 @@ func deleteEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) erro
 	for _, edgeJob := range edgeJobs {
 		for _, groupID := range edgeJob.EdgeGroups {
 			if groupID == ID {
-				return httperror.NewError(http.StatusConflict, "Edge group is used by an Edge job", errors.New("edge group is used by an Edge job"))
+				return httperror.Conflict("Edge group is used by an Edge job", errors.New("edge group is used by an Edge job"))
 			}
 		}
 	}

api/http/handler/edgegroups/edgegroup_list.go

@@ -12,9 +12,10 @@ import (
 
 type decoratedEdgeGroup struct {
 	portainer.EdgeGroup
-	HasEdgeStack  bool `json:"HasEdgeStack"`
-	HasEdgeJob    bool `json:"HasEdgeJob"`
-	EndpointTypes []portainer.EndpointType
+	HasEdgeStack     bool `json:"HasEdgeStack"`
+	HasEdgeJob       bool `json:"HasEdgeJob"`
+	EndpointTypes    []portainer.EndpointType
+	TrustedEndpoints []portainer.EndpointID `json:"TrustedEndpoints"`
 }
 
 // @id EdgeGroupList
@@ -85,6 +86,14 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
 			}
 
 			edgeGroup.Endpoints = endpointIDs
+			edgeGroup.TrustedEndpoints = endpointIDs
+		} else {
+			trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.Endpoints)
+			if err != nil {
+				return nil, httperror.InternalServerError("Unable to retrieve environments for Edge group", err)
+			}
+
+			edgeGroup.TrustedEndpoints = trustedEndpoints
 		}
 
 		endpointTypes, err := getEndpointTypes(tx, edgeGroup.Endpoints)

api/http/handler/edgestacks/edgestack_create.go

@@ -36,7 +36,7 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 		case httperrors.IsInvalidPayloadError(err):
 			return httperror.BadRequest("Invalid payload", err)
 		case httperrors.IsConflictError(err):
-			return httperror.NewError(http.StatusConflict, err.Error(), err)
+			return httperror.Conflict(err.Error(), err)
 		default:
 			return httperror.InternalServerError("Unable to create Edge stack", err)
 		}

api/http/handler/edgestacks/edgestack_create_git.go

@@ -31,7 +31,7 @@ type edgeStackFromGitRepositoryPayload struct {
 	// Path to the Stack file inside the Git repository
 	FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// List of identifiers of EdgeGroups
-	EdgeGroups []portainer.EdgeGroupID `example:"1"`
+	EdgeGroups []portainer.EdgeGroupID `example:"1" validate:"required"`
 	// Deployment type to deploy this stack
 	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
 	// compose is enabled only for docker environments
@@ -85,7 +85,6 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param method query string true "Creation Method" Enums(file,string,repository)
 // @param body body edgeStackFromGitRepositoryPayload true "stack config"
 // @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
 // @success 200 {object} portainer.EdgeStack

api/http/handler/edgestacks/edgestack_create_test.go

@@ -2,13 +2,14 @@ package edgestacks
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 // Create

api/http/handler/edgestacks/edgestack_delete_test.go

@@ -1,13 +1,14 @@
 package edgestacks
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 // Delete

api/http/handler/edgestacks/edgestack_status_update_test.go

@@ -2,13 +2,14 @@ package edgestacks
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 // Update Status

api/http/handler/edgestacks/edgestack_update_test.go

@@ -2,7 +2,6 @@ package edgestacks
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -10,6 +9,8 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/segmentio/encoding/json"
 )
 
 // Update

api/http/handler/edgetemplates/edgetemplate_list.go

@@ -1,13 +1,15 @@
 package edgetemplates
 
 import (
-	"encoding/json"
 	"net/http"
+	"slices"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/client"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type templateFileFormat struct {
@@ -16,6 +18,7 @@ type templateFileFormat struct {
 }
 
 // @id EdgeTemplateList
+// @deprecated
 // @summary Fetches the list of Edge Templates
 // @description **Access policy**: administrator
 // @tags edge_templates
@@ -50,10 +53,16 @@ func (handler *Handler) edgeTemplateList(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to parse template file", err)
 	}
 
+	// We only support version 3 of the template format
+	// this is only a temporary fix until we have custom edge templates
+	if templateFile.Version != "3" {
+		return httperror.InternalServerError("Unsupported template version", nil)
+	}
+
 	filteredTemplates := make([]portainer.Template, 0)
 
 	for _, template := range templateFile.Templates {
-		if template.Type == portainer.EdgeStackTemplate {
+		if slices.Contains(template.Categories, "edge") && slices.Contains([]portainer.TemplateType{portainer.ComposeStackTemplate, portainer.SwarmStackTemplate}, template.Type) {
 			filteredTemplates = append(filteredTemplates, template)
 		}
 	}

api/http/handler/edgetemplates/handler.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
@@ -25,7 +26,7 @@ func NewHandler(bouncer security.BouncerService) *Handler {
 	}
 
 	h.Handle("/edge_templates",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.edgeTemplateList))).Methods(http.MethodGet)
+		bouncer.AdminAccess(middlewares.Deprecated(httperror.LoggerHandler(h.edgeTemplateList), func(w http.ResponseWriter, r *http.Request) (string, *httperror.HandlerError) { return "", nil }))).Methods(http.MethodGet)
 
 	return h
 }

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -2,7 +2,6 @@ package endpointedge
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -17,6 +16,7 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/jwt"
 
+	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 

api/http/handler/endpoints/endpoint_create.go

@@ -216,7 +216,7 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 	}
 
 	if !isUnique {
-		return httperror.NewError(http.StatusConflict, "Name is not unique", nil)
+		return httperror.Conflict("Name is not unique", nil)
 	}
 
 	endpoint, endpointCreationError := handler.createEndpoint(handler.DataStore, payload)

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -1,7 +1,6 @@
 package endpoints
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -15,6 +14,8 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type dockerhubStatusResponse struct {

api/http/handler/endpoints/endpoint_force_update_service.go

@@ -0,0 +1,108 @@
+package endpoints
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	portaineree "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/docker/consts"
+	"github.com/portainer/portainer/api/docker/images"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+
+	"github.com/docker/docker/api/types"
+	dockertypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+)
+
+type forceUpdateServicePayload struct {
+	// ServiceId to update
+	ServiceID string
+	// PullImage if true will pull the image
+	PullImage bool
+}
+
+func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// @id endpointForceUpdateService
+// @summary force update a docker service
+// @description force update a docker service
+// @description **Access policy**: authenticated
+// @tags endpoints
+// @security ApiKeyAuth
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "endpoint identifier"
+// @param body body forceUpdateServicePayload true "details"
+// @success 200 {object} dockertypes.ServiceUpdateResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied"
+// @failure 404 "endpoint not found"
+// @failure 500 "Server error"
+// @router /endpoints/{id}/forceupdateservice [put]
+func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return httperror.BadRequest("Invalid environment identifier route variable", err)
+	}
+
+	var payload forceUpdateServicePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portaineree.EndpointID(endpointID))
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
+	} else if err != nil {
+		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	if err != nil {
+		return httperror.Forbidden("Permission denied to force update service", err)
+	}
+
+	dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "", nil)
+	if err != nil {
+		return httperror.InternalServerError("Error creating docker client", err)
+	}
+	defer dockerClient.Close()
+
+	service, _, err := dockerClient.ServiceInspectWithRaw(context.Background(), payload.ServiceID, dockertypes.ServiceInspectOptions{InsertDefaults: true})
+	if err != nil {
+		return httperror.InternalServerError("Error looking up service", err)
+	}
+
+	service.Spec.TaskTemplate.ForceUpdate++
+
+	if payload.PullImage {
+		service.Spec.TaskTemplate.ContainerSpec.Image = strings.Split(service.Spec.TaskTemplate.ContainerSpec.Image, "@sha")[0]
+	}
+
+	newService, err := dockerClient.ServiceUpdate(context.Background(), payload.ServiceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{QueryRegistry: true})
+	if err != nil {
+		return httperror.InternalServerError("Error force update service", err)
+	}
+
+	go func() {
+		images.EvictImageStatus(payload.ServiceID)
+		images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
+		containers, _ := dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{
+			All:     true,
+			Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+payload.ServiceID)),
+		})
+
+		for _, container := range containers {
+			images.EvictImageStatus(container.ID)
+		}
+	}()
+
+	return response.JSON(w, newService)
+}

api/http/handler/endpoints/endpoint_list_test.go

@@ -1,7 +1,6 @@
 package endpoints
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,6 +12,8 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+
+	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -27,13 +28,13 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
 		GroupID: 1,
 		Type:    portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string "example:\"1.0.0\""
+			Version string `example:"1.0.0"`
 		}{
 			Version: "1.0.0",
 		},
 	}
 	version2Endpoint := portainer.Endpoint{ID: 2, GroupID: 1, Type: portainer.AgentOnDockerEnvironment, Agent: struct {
-		Version string "example:\"1.0.0\""
+		Version string `example:"1.0.0"`
 	}{Version: "2.0.0"}}
 	noVersionEndpoint := portainer.Endpoint{ID: 3, Type: portainer.AgentOnDockerEnvironment, GroupID: 1}
 	notAgentEnvironments := portainer.Endpoint{ID: 4, Type: portainer.DockerEnvironment, GroupID: 1}
@@ -210,7 +211,7 @@ func buildEndpointListRequest(query string) *http.Request {
 	restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
 	req = req.WithContext(restrictedCtx)
 
-	req.Header.Add("Authorization", "Bearer dummytoken")
+	testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
 
 	return req
 }

api/http/handler/endpoints/endpoint_registry_access.go

@@ -140,7 +140,7 @@ func (handler *Handler) updateKubeAccess(endpoint *portainer.Endpoint, registry
 	}
 
 	for namespace := range namespacesToRemove {
-		err := cli.DeleteRegistrySecret(registry, namespace)
+		err := cli.DeleteRegistrySecret(registry.ID, namespace)
 		if err != nil {
 			return err
 		}

api/http/handler/endpoints/endpoint_update.go

@@ -100,7 +100,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 
 		if !isUnique {
-			return httperror.NewError(http.StatusConflict, "Name is not unique", nil)
+			return httperror.Conflict("Name is not unique", nil)
 		}
 
 		endpoint.Name = name

api/http/handler/endpoints/filter_test.go

@@ -22,12 +22,12 @@ func Test_Filter_AgentVersion(t *testing.T) {
 	version1Endpoint := portainer.Endpoint{ID: 1, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string "example:\"1.0.0\""
+			Version string `example:"1.0.0"`
 		}{Version: "1.0.0"}}
 	version2Endpoint := portainer.Endpoint{ID: 2, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,
 		Agent: struct {
-			Version string "example:\"1.0.0\""
+			Version string `example:"1.0.0"`
 		}{Version: "2.0.0"}}
 	noVersionEndpoint := portainer.Endpoint{ID: 3, GroupID: 1,
 		Type: portainer.AgentOnDockerEnvironment,

api/http/handler/endpoints/handler.go

@@ -6,6 +6,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/demo"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
@@ -36,6 +37,7 @@ type Handler struct {
 	K8sClientFactory      *cli.ClientFactory
 	ComposeStackManager   portainer.ComposeStackManager
 	AuthorizationService  *authorization.Service
+	DockerClientFactory   *dockerclient.ClientFactory
 	BindAddress           string
 	BindAddressHTTPS      string
 	PendingActionsService *pendingactions.PendingActionsService
@@ -79,6 +81,8 @@ func NewHandler(bouncer security.BouncerService, demoService *demo.Service) *Han
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryAccess))).Methods(http.MethodPut)
 
 	h.Handle("/endpoints/global-key", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointCreateGlobalKey))).Methods(http.MethodPost)
+	h.Handle("/endpoints/{id}/forceupdateservice",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointForceUpdateService))).Methods(http.MethodPut)
 
 	// DEPRECATED
 	h.Handle("/endpoints/{id}/status", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)

api/http/handler/handler.go

@@ -81,6 +81,7 @@ type Handler struct {
 	UserHandler            *users.Handler
 	WebSocketHandler       *websocket.Handler
 	WebhookHandler         *webhooks.Handler
+	UserHelmHandler        *helm.Handler
 }
 
 // @title PortainerCE API

api/http/handler/helm/handler.go

@@ -20,14 +20,14 @@ type Handler struct {
 	*mux.Router
 	requestBouncer           security.BouncerService
 	dataStore                dataservices.DataStore
-	jwtService               dataservices.JWTService
+	jwtService               portainer.JWTService
 	kubeClusterAccessService kubernetes.KubeClusterAccessService
 	kubernetesDeployer       portainer.KubernetesDeployer
 	helmPackageManager       libhelm.HelmPackageManager
 }
 
 // NewHandler creates a handler to manage endpoint group operations.
-func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, jwtService dataservices.JWTService, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeClusterAccessService kubernetes.KubeClusterAccessService) *Handler {
+func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, jwtService portainer.JWTService, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeClusterAccessService kubernetes.KubeClusterAccessService) *Handler {
 	h := &Handler{
 		Router:                   mux.NewRouter(),
 		requestBouncer:           bouncer,
@@ -52,10 +52,11 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 	h.Handle("/{id}/kubernetes/helm",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.helmInstall))).Methods(http.MethodPost)
 
+	// Deprecated
 	h.Handle("/{id}/kubernetes/helm/repositories",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.userGetHelmRepos))).Methods(http.MethodGet)
+		httperror.LoggerHandler(h.userGetHelmRepos)).Methods(http.MethodGet)
 	h.Handle("/{id}/kubernetes/helm/repositories",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.userCreateHelmRepo))).Methods(http.MethodPost)
+		httperror.LoggerHandler(h.userCreateHelmRepo)).Methods(http.MethodPost)
 
 	return h
 }
@@ -92,7 +93,7 @@ func (handler *Handler) getHelmClusterAccess(r *http.Request) (*options.Kubernet
 		return nil, httperror.InternalServerError("Unable to retrieve user authentication token", err)
 	}
 
-	bearerToken, err := handler.jwtService.GenerateToken(tokenData)
+	bearerToken, _, err := handler.jwtService.GenerateToken(tokenData)
 	if err != nil {
 		return nil, httperror.Unauthorized("Unauthorized", err)
 	}

api/http/handler/helm/helm_delete_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -48,7 +49,7 @@ func Test_helmDelete(t *testing.T) {
 		req := httptest.NewRequest(http.MethodDelete, fmt.Sprintf("/1/kubernetes/helm/%s", options.Name), nil)
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		req.Header.Add("Authorization", "Bearer dummytoken")
+		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/helm/helm_install_test.go

@@ -2,23 +2,24 @@ package helm
 
 import (
 	"bytes"
-	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	"github.com/portainer/portainer/api/datastore"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/pkg/libhelm/binary/test"
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/portainer/portainer/pkg/libhelm/release"
+
+	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -52,7 +53,7 @@ func Test_helmInstall(t *testing.T) {
 		req := httptest.NewRequest(http.MethodPost, "/1/kubernetes/helm", bytes.NewBuffer(optdata))
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		req.Header.Add("Authorization", "Bearer dummytoken")
+		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/helm/helm_list_test.go

@@ -1,7 +1,6 @@
 package helm
 
 import (
-	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -11,14 +10,16 @@ import (
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/pkg/libhelm/binary/test"
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	"github.com/portainer/portainer/pkg/libhelm/release"
-	"github.com/stretchr/testify/assert"
 
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_helmList(t *testing.T) {
@@ -48,7 +49,7 @@ func Test_helmList(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/1/kubernetes/helm", nil)
 		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
 		req = req.WithContext(ctx)
-		req.Header.Add("Authorization", "Bearer dummytoken")
+		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
 
 		rr := httptest.NewRecorder()
 		h.ServeHTTP(rr, req)

api/http/handler/helm/user_helm_repos.go

@@ -27,7 +27,7 @@ func (p *addHelmRepoUrlPayload) Validate(_ *http.Request) error {
 	return libhelm.ValidateHelmRepositoryURL(p.URL, nil)
 }
 
-// @id HelmUserRepositoryCreate
+// @id HelmUserRepositoryCreateDeprecated
 // @summary Create a user helm repository
 // @description Create a user helm repository.
 // @description **Access policy**: authenticated
@@ -42,6 +42,7 @@ func (p *addHelmRepoUrlPayload) Validate(_ *http.Request) error {
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 500 "Server error"
+// @deprecated
 // @router /endpoints/{id}/kubernetes/helm/repositories [post]
 func (handler *Handler) userCreateHelmRepo(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	tokenData, err := security.RetrieveTokenData(r)
@@ -85,7 +86,7 @@ func (handler *Handler) userCreateHelmRepo(w http.ResponseWriter, r *http.Reques
 	return response.JSON(w, record)
 }
 
-// @id HelmUserRepositoriesList
+// @id HelmUserRepositoriesListDeprecated
 // @summary List a users helm repositories
 // @description Inspect a user helm repositories.
 // @description **Access policy**: authenticated
@@ -98,6 +99,7 @@ func (handler *Handler) userCreateHelmRepo(w http.ResponseWriter, r *http.Reques
 // @failure 400 "Invalid request"
 // @failure 403 "Permission denied"
 // @failure 500 "Server error"
+// @deprecated
 // @router /endpoints/{id}/kubernetes/helm/repositories [get]
 func (handler *Handler) userGetHelmRepos(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	tokenData, err := security.RetrieveTokenData(r)

api/http/handler/hostmanagement/fdo/profile_create.go

@@ -69,7 +69,7 @@ func (handler *Handler) createFDOProfileFromFileContent(w http.ResponseWriter, r
 	}
 
 	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A profile with the name '%s' already exists", payload.Name), Err: errors.New("a profile already exists with this name")}
+		return httperror.Conflict(fmt.Sprintf("A profile with the name '%s' already exists", payload.Name), errors.New("a profile already exists with this name"))
 	}
 
 	profileID := handler.DataStore.FDOProfile().GetNextIdentifier()

api/http/handler/hostmanagement/fdo/profile_update.go

@@ -49,7 +49,7 @@ func (handler *Handler) updateProfile(w http.ResponseWriter, r *http.Request) *h
 		return httperror.InternalServerError(err.Error(), err)
 	}
 	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A profile with the name '%s' already exists", payload.Name), Err: errors.New("a profile already exists with this name")}
+		return httperror.Conflict(fmt.Sprintf("A profile with the name '%s' already exists", payload.Name), errors.New("a profile already exists with this name"))
 	}
 
 	filePath, err := handler.FileService.StoreFDOProfileFileFromBytes(strconv.Itoa(int(profile.ID)), []byte(payload.ProfileFileContent))

api/http/handler/hostmanagement/openamt/amtrpc.go

@@ -2,7 +2,6 @@ package openamt
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -20,6 +19,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 )
 
 type HostInfo struct {

api/http/handler/kubernetes/configmaps_and_secrets.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"net/http"
-	"strconv"
 
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -10,38 +9,19 @@ import (
 )
 
 func (handler *Handler) getKubernetesConfigMapsAndSecrets(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
+	}
+
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	configmaps, err := cli.GetConfigMapsAndSecrets(namespace)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve configmaps and secrets",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve configmaps and secrets", err)
 	}
 
 	return response.JSON(w, configmaps)

api/http/handler/kubernetes/handler.go

@@ -26,12 +26,12 @@ type Handler struct {
 	authorizationService     *authorization.Service
 	DataStore                dataservices.DataStore
 	KubernetesClientFactory  *cli.ClientFactory
-	JwtService               dataservices.JWTService
+	JwtService               portainer.JWTService
 	kubeClusterAccessService kubernetes.KubeClusterAccessService
 }
 
 // NewHandler creates a handler to process pre-proxied requests to external APIs.
-func NewHandler(bouncer security.BouncerService, authorizationService *authorization.Service, dataStore dataservices.DataStore, jwtService dataservices.JWTService, kubeClusterAccessService kubernetes.KubeClusterAccessService, kubernetesClientFactory *cli.ClientFactory, kubernetesClient portainer.KubeClient) *Handler {
+func NewHandler(bouncer security.BouncerService, authorizationService *authorization.Service, dataStore dataservices.DataStore, jwtService portainer.JWTService, kubeClusterAccessService kubernetes.KubeClusterAccessService, kubernetesClientFactory *cli.ClientFactory, kubernetesClient portainer.KubeClient) *Handler {
 	h := &Handler{
 		Router:                   mux.NewRouter(),
 		authorizationService:     authorizationService,
@@ -50,23 +50,24 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	endpointRouter := kubeRouter.PathPrefix("/{id}").Subrouter()
 	endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
 	endpointRouter.Use(kubeOnlyMiddleware)
-	endpointRouter.Use(h.kubeClient)
+	endpointRouter.Use(h.kubeClientMiddleware)
 
-	endpointRouter.PathPrefix("/nodes_limits").Handler(httperror.LoggerHandler(h.getKubernetesNodesLimits)).Methods(http.MethodGet)
-	endpointRouter.Path("/metrics/nodes").Handler(httperror.LoggerHandler(h.getKubernetesMetricsForAllNodes)).Methods(http.MethodGet)
-	endpointRouter.Path("/metrics/nodes/{name}").Handler(httperror.LoggerHandler(h.getKubernetesMetricsForNode)).Methods(http.MethodGet)
-	endpointRouter.Path("/metrics/pods/namespace/{namespace}").Handler(httperror.LoggerHandler(h.getKubernetesMetricsForAllPods)).Methods(http.MethodGet)
-	endpointRouter.Path("/metrics/pods/namespace/{namespace}/{name}").Handler(httperror.LoggerHandler(h.getKubernetesMetricsForPod)).Methods(http.MethodGet)
+	endpointRouter.Handle("/nodes_limits", httperror.LoggerHandler(h.getKubernetesNodesLimits)).Methods(http.MethodGet)
+	endpointRouter.Handle("/max_resource_limits", httperror.LoggerHandler(h.getKubernetesMaxResourceLimits)).Methods(http.MethodGet)
+	endpointRouter.Handle("/metrics/nodes", httperror.LoggerHandler(h.getKubernetesMetricsForAllNodes)).Methods(http.MethodGet)
+	endpointRouter.Handle("/metrics/nodes/{name}", httperror.LoggerHandler(h.getKubernetesMetricsForNode)).Methods(http.MethodGet)
+	endpointRouter.Handle("/metrics/pods/namespace/{namespace}", httperror.LoggerHandler(h.getKubernetesMetricsForAllPods)).Methods(http.MethodGet)
+	endpointRouter.Handle("/metrics/pods/namespace/{namespace}/{name}", httperror.LoggerHandler(h.getKubernetesMetricsForPod)).Methods(http.MethodGet)
 	endpointRouter.Handle("/ingresscontrollers", httperror.LoggerHandler(h.getKubernetesIngressControllers)).Methods(http.MethodGet)
 	endpointRouter.Handle("/ingresscontrollers", httperror.LoggerHandler(h.updateKubernetesIngressControllers)).Methods(http.MethodPut)
 	endpointRouter.Handle("/ingresses/delete", httperror.LoggerHandler(h.deleteKubernetesIngresses)).Methods(http.MethodPost)
 	endpointRouter.Handle("/services/delete", httperror.LoggerHandler(h.deleteKubernetesServices)).Methods(http.MethodPost)
-	endpointRouter.Path("/rbac_enabled").Handler(httperror.LoggerHandler(h.isRBACEnabled)).Methods(http.MethodGet)
-	endpointRouter.Path("/namespaces").Handler(httperror.LoggerHandler(h.createKubernetesNamespace)).Methods(http.MethodPost)
-	endpointRouter.Path("/namespaces").Handler(httperror.LoggerHandler(h.updateKubernetesNamespace)).Methods(http.MethodPut)
-	endpointRouter.Path("/namespaces").Handler(httperror.LoggerHandler(h.getKubernetesNamespaces)).Methods(http.MethodGet)
-	endpointRouter.Path("/namespace/{namespace}").Handler(httperror.LoggerHandler(h.deleteKubernetesNamespace)).Methods(http.MethodDelete)
-	endpointRouter.Path("/namespaces/{namespace}").Handler(httperror.LoggerHandler(h.getKubernetesNamespace)).Methods(http.MethodGet)
+	endpointRouter.Handle("/rbac_enabled", httperror.LoggerHandler(h.isRBACEnabled)).Methods(http.MethodGet)
+	endpointRouter.Handle("/namespaces", httperror.LoggerHandler(h.createKubernetesNamespace)).Methods(http.MethodPost)
+	endpointRouter.Handle("/namespaces", httperror.LoggerHandler(h.updateKubernetesNamespace)).Methods(http.MethodPut)
+	endpointRouter.Handle("/namespaces", httperror.LoggerHandler(h.getKubernetesNamespaces)).Methods(http.MethodGet)
+	endpointRouter.Handle("/namespace/{namespace}", httperror.LoggerHandler(h.deleteKubernetesNamespace)).Methods(http.MethodDelete)
+	endpointRouter.Handle("/namespaces/{namespace}", httperror.LoggerHandler(h.getKubernetesNamespace)).Methods(http.MethodGet)
 
 	// namespaces
 	// in the future this piece of code might be in another package (or a few different packages - namespaces/namespace?)
@@ -110,94 +111,85 @@ func kubeOnlyMiddleware(next http.Handler) http.Handler {
 	})
 }
 
-func (handler *Handler) kubeClient(next http.Handler) http.Handler {
+// getProxyKubeClient gets a kubeclient for the user.  It's generally what you want as it retrieves the kubeclient
+// from the Authorization token of the currently logged in user.  The kubeclient that is not from the proxy is actually using
+// admin permissions.  If you're unsure which one to use, use this.
+func (h *Handler) getProxyKubeClient(r *http.Request) (*cli.KubeClient, *httperror.HandlerError) {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return nil, httperror.BadRequest("Invalid environment identifier route variable", err)
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return nil, httperror.Forbidden("Permission denied to access environment", err)
+	}
+
+	cli, ok := h.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), tokenData.Username)
+	if !ok {
+		return nil, httperror.InternalServerError("Failed to lookup KubeClient", nil)
+	}
+
+	return cli, nil
+}
+
+func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-		if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusBadRequest,
-				"Invalid environment identifier route variable",
-				err,
-			)
-			return
-		}
-
-		endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-		if handler.DataStore.IsErrObjectNotFound(err) {
-			httperror.WriteError(
-				w,
-				http.StatusNotFound,
-				"Unable to find an environment with the specified identifier inside the database",
-				err,
-			)
-			return
-		} else if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusInternalServerError,
-				"Unable to find an environment with the specified identifier inside the database",
-				err,
-			)
-			return
-		}
-
 		if handler.KubernetesClientFactory == nil {
 			next.ServeHTTP(w, r)
 			return
 		}
-		// Generate a proxied kubeconfig, then create a kubeclient using it.
+
+		endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+		if err != nil {
+			httperror.WriteError(w, http.StatusBadRequest, "Invalid environment identifier route variable", err)
+			return
+		}
+
 		tokenData, err := security.RetrieveTokenData(r)
 		if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusForbidden,
-				"Permission denied to access environment",
-				err,
-			)
+			httperror.WriteError(w, http.StatusForbidden, "Permission denied to access environment", err)
+		}
+
+		// Check if we have a kubeclient against this auth token already, otherwise generate a new one
+		_, ok := handler.KubernetesClientFactory.GetProxyKubeClient(strconv.Itoa(endpointID), tokenData.Username)
+		if ok {
+			next.ServeHTTP(w, r)
 			return
 		}
+
+		endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+		if err != nil {
+			if handler.DataStore.IsErrObjectNotFound(err) {
+				httperror.WriteError(
+					w,
+					http.StatusNotFound,
+					"Unable to find an environment with the specified identifier inside the database",
+					err,
+				)
+				return
+			}
+
+			httperror.WriteError(w, http.StatusInternalServerError, "Unable to read the environment from the database", err)
+			return
+		}
+
 		bearerToken, err := handler.JwtService.GenerateTokenForKubeconfig(tokenData)
 		if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusInternalServerError,
-				"Unable to create JWT token",
-				err,
-			)
+			httperror.WriteError(w, http.StatusInternalServerError, "Unable to create JWT token", err)
 			return
 		}
-		singleEndpointList := []portainer.Endpoint{
-			*endpoint,
-		}
-		config := handler.buildConfig(
-			r,
-			tokenData,
-			bearerToken,
-			singleEndpointList,
-			true,
-		)
 
+		config := handler.buildConfig(r, tokenData, bearerToken, []portainer.Endpoint{*endpoint}, true)
 		if len(config.Clusters) == 0 {
-			httperror.WriteError(
-				w,
-				http.StatusInternalServerError,
-				"Unable build cluster kubeconfig",
-				errors.New("Unable build cluster kubeconfig"),
-			)
+			httperror.WriteError(w, http.StatusInternalServerError, "Unable build cluster kubeconfig", nil)
 			return
 		}
 
-		// Manually setting the localhost to route
-		// the request to proxy server
+		// Manually setting serverURL to localhost to route the request to proxy server
 		serverURL, err := url.Parse(config.Clusters[0].Cluster.Server)
 		if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusInternalServerError,
-				"Unable parse cluster's kubeconfig server URL",
-				nil,
-			)
+			httperror.WriteError(w, http.StatusInternalServerError, "Unable parse cluster's kubeconfig server URL", nil)
 			return
 		}
 		serverURL.Scheme = "https"
@@ -216,16 +208,11 @@ func (handler *Handler) kubeClient(next http.Handler) http.Handler {
 		}
 		kubeCli, err := handler.KubernetesClientFactory.CreateKubeClientFromKubeConfig(endpoint.Name, []byte(yaml))
 		if err != nil {
-			httperror.WriteError(
-				w,
-				http.StatusInternalServerError,
-				"Failed to create client from kubeconfig",
-				err,
-			)
+			httperror.WriteError(w, http.StatusInternalServerError, "Failed to create client from kubeconfig", err)
 			return
 		}
 
-		handler.KubernetesClientFactory.SetProxyKubeClient(strconv.Itoa(int(endpoint.ID)), r.Header.Get("Authorization"), kubeCli)
+		handler.KubernetesClientFactory.SetProxyKubeClient(strconv.Itoa(int(endpoint.ID)), tokenData.Username, kubeCli)
 		next.ServeHTTP(w, r)
 	})
 }

api/http/handler/kubernetes/ingresses.go

@@ -2,9 +2,9 @@ package kubernetes
 
 import (
 	"net/http"
-	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/middlewares"
 	models "github.com/portainer/portainer/api/http/models/kubernetes"
 	"github.com/portainer/portainer/api/http/security"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -177,14 +177,9 @@ func (handler *Handler) getKubernetesIngressControllersByNamespace(w http.Respon
 		)
 	}
 
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	currentControllers, err := cli.GetIngressControllers()
@@ -396,42 +391,20 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/namespaces/{namespace}/ingresscontrollers [put]
 func (handler *Handler) updateKubernetesIngressControllersByNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound(
-			"Unable to find an environment with the specified identifier inside the database",
-			err,
-		)
-	} else if err != nil {
-		return httperror.InternalServerError(
-			"Unable to find an environment with the specified identifier inside the database",
-			err,
-		)
+		return httperror.NotFound("Unable to find an environment on request context", err)
 	}
 
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	var payload models.K8sIngressControllers
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
 	existingClasses := endpoint.Kubernetes.Configuration.IngressClasses
@@ -497,18 +470,13 @@ PayloadLoop:
 			updatedClasses = append(updatedClasses, existingClass)
 		}
 	}
-
 	endpoint.Kubernetes.Configuration.IngressClasses = updatedClasses
-	err = handler.DataStore.Endpoint().UpdateEndpoint(
-		portainer.EndpointID(endpointID),
-		endpoint,
-	)
+
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to update the BlockedIngressClasses inside the database",
-			err,
-		)
+		return httperror.InternalServerError("Unable to update the BlockedIngressClasses inside the database", err)
 	}
+
 	return response.Empty(w)
 }
 
@@ -531,36 +499,17 @@ PayloadLoop:
 func (handler *Handler) getKubernetesIngresses(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	ingresses, err := cli.GetIngresses(namespace)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve ingresses",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve ingresses", err)
 	}
 
 	return response.JSON(w, ingresses)
@@ -585,27 +534,13 @@ func (handler *Handler) getKubernetesIngresses(w http.ResponseWriter, r *http.Re
 func (handler *Handler) createKubernetesIngress(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	var payload models.K8sIngressInfo
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
-	}
-
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
 	owner := "admin"
@@ -614,23 +549,16 @@ func (handler *Handler) createKubernetesIngress(w http.ResponseWriter, r *http.R
 		owner = tokenData.Username
 	}
 
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	err = cli.CreateIngress(namespace, payload, owner)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve the ingress",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve the ingress", err)
 	}
+
 	return response.Empty(w)
 }
 
@@ -650,37 +578,22 @@ func (handler *Handler) createKubernetesIngress(w http.ResponseWriter, r *http.R
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/ingresses/delete [post]
 func (handler *Handler) deleteKubernetesIngresses(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	var payload models.K8sIngressDeleteRequests
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
 	err = cli.DeleteIngresses(payload)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to delete ingresses",
-			err,
-		)
+		return httperror.InternalServerError("Unable to delete ingresses", err)
 	}
+
 	return response.Empty(w)
 }
 
@@ -703,45 +616,24 @@ func (handler *Handler) deleteKubernetesIngresses(w http.ResponseWriter, r *http
 func (handler *Handler) updateKubernetesIngress(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	var payload models.K8sIngressInfo
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	err = cli.UpdateIngress(namespace, payload)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to update the ingress",
-			err,
-		)
+		return httperror.InternalServerError("Unable to update the ingress", err)
 	}
+
 	return response.Empty(w)
 }

api/http/handler/kubernetes/metrics.go

@@ -3,7 +3,7 @@ package kubernetes
 import (
 	"net/http"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/middlewares"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -26,34 +26,19 @@ import (
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/metrics/nodes [get]
 func (handler *Handler) getKubernetesMetricsForAllNodes(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+		return httperror.InternalServerError(err.Error(), err)
 	}
 
 	cli, err := handler.KubernetesClientFactory.CreateRemoteMetricsClient(endpoint)
 	if err != nil {
-		return httperror.InternalServerError(
-			"failed to create metrics KubeClient",
-			nil,
-		)
+		return httperror.InternalServerError("failed to create metrics KubeClient", nil)
 	}
 
 	metrics, err := cli.MetricsV1beta1().NodeMetricses().List(r.Context(), v1.ListOptions{})
 	if err != nil {
-		return httperror.InternalServerError(
-			"Failed to fetch metrics",
-			err,
-		)
+		return httperror.InternalServerError("Failed to fetch metrics", err)
 	}
 
 	return response.JSON(w, metrics)
@@ -75,34 +60,19 @@ func (handler *Handler) getKubernetesMetricsForAllNodes(w http.ResponseWriter, r
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/metrics/nodes/{name} [get]
 func (handler *Handler) getKubernetesMetricsForNode(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+		return httperror.InternalServerError(err.Error(), err)
 	}
 
 	cli, err := handler.KubernetesClientFactory.CreateRemoteMetricsClient(endpoint)
 	if err != nil {
-		return httperror.InternalServerError(
-			"failed to create metrics KubeClient",
-			nil,
-		)
+		return httperror.InternalServerError("failed to create metrics KubeClient", nil)
 	}
 
 	nodeName, err := request.RetrieveRouteVariableValue(r, "name")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid node identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid node identifier route variable", err)
 	}
 
 	metrics, err := cli.MetricsV1beta1().NodeMetricses().Get(
@@ -111,10 +81,7 @@ func (handler *Handler) getKubernetesMetricsForNode(w http.ResponseWriter, r *ht
 		v1.GetOptions{},
 	)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Failed to fetch metrics",
-			err,
-		)
+		return httperror.InternalServerError("Failed to fetch metrics", err)
 	}
 
 	return response.JSON(w, metrics)
@@ -136,42 +103,24 @@ func (handler *Handler) getKubernetesMetricsForNode(w http.ResponseWriter, r *ht
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/metrics/pods/{namespace} [get]
 func (handler *Handler) getKubernetesMetricsForAllPods(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+		return httperror.InternalServerError(err.Error(), err)
 	}
 
 	cli, err := handler.KubernetesClientFactory.CreateRemoteMetricsClient(endpoint)
 	if err != nil {
-		return httperror.InternalServerError(
-			"failed to create metrics KubeClient",
-			nil,
-		)
+		return httperror.InternalServerError("failed to create metrics KubeClient", nil)
 	}
 
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	metrics, err := cli.MetricsV1beta1().PodMetricses(namespace).List(r.Context(), v1.ListOptions{})
 	if err != nil {
-		return httperror.InternalServerError(
-			"Failed to fetch metrics",
-			err,
-		)
+		return httperror.InternalServerError("Failed to fetch metrics", err)
 	}
 
 	return response.JSON(w, metrics)
@@ -194,54 +143,29 @@ func (handler *Handler) getKubernetesMetricsForAllPods(w http.ResponseWriter, r
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/metrics/pods/{namespace}/{name} [get]
 func (handler *Handler) getKubernetesMetricsForPod(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+		return httperror.InternalServerError(err.Error(), err)
 	}
 
 	cli, err := handler.KubernetesClientFactory.CreateRemoteMetricsClient(endpoint)
 	if err != nil {
-		return httperror.InternalServerError(
-			"failed to create metrics KubeClient",
-			nil,
-		)
+		return httperror.InternalServerError("failed to create metrics KubeClient", nil)
 	}
 
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	podName, err := request.RetrieveRouteVariableValue(r, "name")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid pod identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid pod identifier route variable", err)
 	}
 
-	metrics, err := cli.MetricsV1beta1().PodMetricses(namespace).Get(
-		r.Context(),
-		podName,
-		v1.GetOptions{},
-	)
+	metrics, err := cli.MetricsV1beta1().PodMetricses(namespace).Get(r.Context(), podName, v1.GetOptions{})
 	if err != nil {
-		return httperror.InternalServerError(
-			"Failed to fetch metrics",
-			err,
-		)
+		return httperror.InternalServerError("Failed to fetch metrics", err)
 	}
 
 	return response.JSON(w, metrics)

api/http/handler/kubernetes/namespaces.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"net/http"
-	"strconv"
 
 	models "github.com/portainer/portainer/api/http/models/kubernetes"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -25,30 +24,14 @@ import (
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/namespaces [get]
 func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	namespaces, err := cli.GetNamespaces()
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve namespaces",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve namespaces", err)
 	}
 
 	return response.JSON(w, namespaces)
@@ -70,24 +53,6 @@ func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.R
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/namespaces/{namespace} [get]
 func (handler *Handler) getKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
 	ns, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
 		return httperror.BadRequest(
@@ -95,12 +60,15 @@ func (handler *Handler) getKubernetesNamespace(w http.ResponseWriter, r *http.Re
 			err,
 		)
 	}
+
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
 	namespace, err := cli.GetNamespace(ns)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve namespace",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve namespace", err)
 	}
 
 	return response.JSON(w, namespace)
@@ -121,42 +89,24 @@ func (handler *Handler) getKubernetesNamespace(w http.ResponseWriter, r *http.Re
 // @success 200 {string} string "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /kubernetes/{id}/namespaces/{namespace} [post]
+// @router /kubernetes/{id}/namespaces [post]
 func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
 	var payload models.K8sNamespaceDetails
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	err = cli.CreateNamespace(payload)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to create namespace",
-			err,
-		)
+		return httperror.InternalServerError("Unable to create namespace", err)
 	}
+
 	return nil
 }
 
@@ -176,38 +126,25 @@ func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/namespaces/{namespace} [delete]
 func (handler *Handler) deleteKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	var payload models.K8sNamespaceDetails
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
+	}
+
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	err = cli.DeleteNamespace(namespace)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to delete namespace",
-			err,
-		)
+		return httperror.InternalServerError("Unable to delete namespace", err)
 	}
 
 	return nil
@@ -230,30 +167,17 @@ func (handler *Handler) deleteKubernetesNamespace(w http.ResponseWriter, r *http
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/namespaces/{namespace} [put]
 func (handler *Handler) updateKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
 	var payload models.K8sNamespaceDetails
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
 	err = cli.UpdateNamespace(payload)
 	if err != nil {
 		return httperror.InternalServerError("Unable to update namespace", err)

api/http/handler/kubernetes/nodes_limits.go

@@ -3,13 +3,12 @@ package kubernetes
 import (
 	"net/http"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/middlewares"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
-	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
-// @id getKubernetesNodesLimits
+// @id GetKubernetesNodesLimits
 // @summary Get CPU and memory limits of all nodes within k8s cluster
 // @description Get CPU and memory limits of all nodes within k8s cluster
 // @description **Access policy**: authenticated
@@ -27,16 +26,9 @@ import (
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/nodes_limits [get]
 func (handler *Handler) getKubernetesNodesLimits(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	endpoint, err := middlewares.FetchEndpoint(r)
 	if err != nil {
-		return httperror.BadRequest("Invalid environment identifier route variable", err)
-	}
-
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
+		return httperror.NotFound("Unable to find an environment on request context", err)
 	}
 
 	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
@@ -51,3 +43,26 @@ func (handler *Handler) getKubernetesNodesLimits(w http.ResponseWriter, r *http.
 
 	return response.JSON(w, nodesLimits)
 }
+
+func (handler *Handler) getKubernetesMaxResourceLimits(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpoint, err := middlewares.FetchEndpoint(r)
+	if err != nil {
+		return httperror.NotFound("Unable to find an environment on request context", err)
+	}
+
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return httperror.InternalServerError("Failed to lookup KubeClient", err)
+	}
+
+	overCommit := endpoint.Kubernetes.Configuration.EnableResourceOverCommit
+	overCommitPercent := endpoint.Kubernetes.Configuration.ResourceOverCommitPercentage
+
+	// name is set to "" so all namespaces resources are considered when calculating max resource limits
+	resourceLimit, err := cli.GetMaxResourceLimits("", overCommit, overCommitPercent)
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve max resource limit", err)
+	}
+
+	return response.JSON(w, resourceLimit)
+}

api/http/handler/kubernetes/rbac.go

@@ -2,10 +2,8 @@ package kubernetes
 
 import (
 	"net/http"
-	"strconv"
 
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
-	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
@@ -22,26 +20,11 @@ import (
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/rbac_enabled [get]
 func (handler *Handler) isRBACEnabled(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	// with the endpoint id and user auth, create a kube client instance
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
-	// with the kube client instance, check if RBAC is enabled
 	isRBACEnabled, err := cli.IsRBACEnabled()
 	if err != nil {
 		return httperror.InternalServerError("Failed to check RBAC status", err)

api/http/handler/kubernetes/services.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"net/http"
-	"strconv"
 
 	models "github.com/portainer/portainer/api/http/models/kubernetes"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -29,44 +28,22 @@ import (
 func (handler *Handler) getKubernetesServices(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	lookup, err := request.RetrieveBooleanQueryParameter(r, "lookupapplications", true)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid lookupapplications query parameter",
-			err,
-		)
+		return httperror.BadRequest("Invalid lookupapplications query parameter", err)
 	}
 
 	services, err := cli.GetServices(namespace, lookup)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to retrieve services",
-			err,
-		)
+		return httperror.InternalServerError("Unable to retrieve services", err)
 	}
 
 	return response.JSON(w, services)
@@ -91,46 +68,25 @@ func (handler *Handler) getKubernetesServices(w http.ResponseWriter, r *http.Req
 func (handler *Handler) createKubernetesService(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	var payload models.K8sServiceInfo
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
 	err = cli.CreateService(namespace, payload)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to create sercice",
-			err,
-		)
+		return httperror.InternalServerError("Unable to create sercice", err)
 	}
+
 	return nil
 }
 
@@ -150,26 +106,8 @@ func (handler *Handler) createKubernetesService(w http.ResponseWriter, r *http.R
 // @failure 500 "Server error"
 // @router /kubernetes/{id}/services/delete [post]
 func (handler *Handler) deleteKubernetesServices(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
-	}
-
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
-
 	var payload models.K8sServiceDeleteRequests
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return httperror.BadRequest(
 			"Invalid request payload",
@@ -177,6 +115,11 @@ func (handler *Handler) deleteKubernetesServices(w http.ResponseWriter, r *http.
 		)
 	}
 
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
 	err = cli.DeleteServices(payload)
 	if err != nil {
 		return httperror.InternalServerError(
@@ -206,44 +149,24 @@ func (handler *Handler) deleteKubernetesServices(w http.ResponseWriter, r *http.
 func (handler *Handler) updateKubernetesService(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid namespace identifier route variable",
-			err,
-		)
+		return httperror.BadRequest("Invalid namespace identifier route variable", err)
 	}
 
 	var payload models.K8sServiceInfo
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return httperror.BadRequest(
-			"Invalid request payload",
-			err,
-		)
+		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return httperror.BadRequest(
-			"Invalid environment identifier route variable",
-			err,
-		)
+	cli, handlerErr := handler.getProxyKubeClient(r)
+	if handlerErr != nil {
+		return handlerErr
 	}
 
-	cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient(
-		strconv.Itoa(endpointID), r.Header.Get("Authorization"),
-	)
-	if !ok {
-		return httperror.InternalServerError(
-			"Failed to lookup KubeClient",
-			nil,
-		)
-	}
 	err = cli.UpdateService(namespace, payload)
 	if err != nil {
-		return httperror.InternalServerError(
-			"Unable to update service",
-			err,
-		)
+		return httperror.InternalServerError("Unable to update service", err)
 	}
+
 	return nil
 }

api/http/handler/motd/motd.go

@@ -1,7 +1,6 @@
 package motd
 
 import (
-	"encoding/json"
 	"net/http"
 	"strings"
 
@@ -9,6 +8,8 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/pkg/libcrypto"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+
+	"github.com/segmentio/encoding/json"
 )
 
 type motdResponse struct {