fix fallback rule (#9114)

* refactor(edge-stacks): filter endpoints by edgeStack

* feat(api/endpoints): edge stack filter support filtering on status in stack

* refactor(endpoints): use separate query params and not JSON query param when querying for an edge stack

* feat(api/endpoints): handle stack filter on dynamic groups + unique list with multiple groups sharing environments

* fix(app/endpoints): edge stack related query params type definition

* fix(api/endpoints): rebase conflicts on imports

.codeclimate.yml

@@ -1,44 +0,0 @@
-version: "2"
-checks:
-  argument-count:
-    enabled: false
-  complex-logic:
-    enabled: false
-  file-lines:
-    enabled: false
-  method-complexity:
-    enabled: false
-  method-count:
-    enabled: false
-  method-lines:
-    enabled: false
-  nested-control-flow:
-    enabled: false
-  return-statements:
-    enabled: false
-  similar-code:
-    enabled: false
-  identical-code:
-    enabled: false
-plugins:
-  gofmt:
-    enabled: true
-  eslint:
-    enabled: true
-    channel: "eslint-5"
-    config:
-      config: .eslintrc.yml
-exclude_patterns:
-- assets/
-- build/
-- dist/
-- distribution/
-- node_modules
-- test/
-- webpack/
-- gruntfile.js
-- webpack.config.js
-- api/
-- "!app/kubernetes/**"
-- .github/
-- .tmp/

.eslintrc.yml

@@ -86,8 +86,8 @@ overrides:
       no-plusplus: off
       func-style: [error, 'declaration']
       import/prefer-default-export: off
-      no-use-before-define: ['error', { functions: false }]
-      '@typescript-eslint/no-use-before-define': ['error', { functions: false }]
+      no-use-before-define: "off"
+      '@typescript-eslint/no-use-before-define': ['error', { functions: false, "allowNamedExports": true }]
       no-shadow: 'off'
       '@typescript-eslint/no-shadow': off
       jsx-a11y/no-autofocus: warn

.github/workflows/label-conflcts.yaml

@@ -11,5 +11,5 @@ jobs:
         with:
           CONFLICT_LABEL_NAME: 'has conflicts'
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MAX_RETRIES: 5
-          WAIT_MS: 5000
+          MAX_RETRIES: 10
+          WAIT_MS: 60000

.github/workflows/lint.yml

@@ -21,13 +21,12 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: 1.19.4
+          go-version: 1.19.5
       - run: yarn --frozen-lockfile
-
       - name: Run linters
         uses: wearerequired/lint-action@v1
         with:
@@ -42,6 +41,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: latest
+          version: v1.52.2
           working-directory: api
-          args: -c .golangci.yaml
+          args: --timeout=10m -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -1,22 +1,23 @@
 name: Nightly Code Security Scan
 
-on: 
+on:
   schedule:
-    - cron: '0 8 * * *'
+    - cron: '0 20 * * *'
   workflow_dispatch:
-    
+
 jobs:
   client-dependencies:
-    name: Client dependency check
+    name: Client Dependency Check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
-      github.ref == 'refs/heads/develop' 
+      github.ref == 'refs/heads/develop'
     outputs:
       js: ${{ steps.set-matrix.outputs.js_result }}
     steps:
-      - uses: actions/checkout@master
+      - name: checkout repository
+        uses: actions/checkout@master
 
-      - name: Run Snyk to check for vulnerabilities
+      - name: scan vulnerabilities by Snyk
         uses: snyk/actions/node@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
@@ -24,46 +25,48 @@ jobs:
         with:
           json: true
 
-      - name: Upload js security scan result as artifact 
+      - name: upload scan result as develop artifact 
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
           path: snyk.json
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
+      - name: develop scan report export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/js-result")
 
-      - name: Upload js result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-js-result-${{github.run_id}}
           path: js-result.html
 
-      - name: Analyse the js result
+      - name: analyse vulnerabilities 
         id: set-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
-          echo "::set-output name=js_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
+          echo "js_result=${result}" >> $GITHUB_OUTPUT
 
   server-dependencies:
-    name: Server dependency check
+    name: Server Dependency Check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
-      github.ref == 'refs/heads/develop' 
+      github.ref == 'refs/heads/develop'
     outputs:
       go: ${{ steps.set-matrix.outputs.go_result }}
     steps:
-      - uses: actions/checkout@master
+      - name: checkout repository
+        uses: actions/checkout@master
 
-      - uses: actions/setup-go@v3
+      - name: install Go 
+        uses: actions/setup-go@v3
         with:
-          go-version: '1.19.4'
+          go-version: '1.19.5'
 
-      - name: Download go modules
+      - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
 
-      - name: Run Snyk to check for vulnerabilities
+      - name: scan vulnerabilities by Snyk
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
@@ -71,122 +74,91 @@ jobs:
           yarn global add snyk
           snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: Upload go security scan result as artifact
+      - name: upload scan result as develop artifact 
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-develop-result
           path: snyk.json
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
+      - name: develop scan report export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/go-result")
 
-      - name: Upload go result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-go-result-${{github.run_id}}
           path: go-result.html
 
-      - name: Analyse the go result
+      - name: analyse vulnerabilities
         id: set-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
-          echo "::set-output name=go_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
+          echo "go_result=${result}" >> $GITHUB_OUTPUT
 
   image-vulnerability:
-    name: Build docker image and Image vulnerability check
+    name: Image Vulnerability Check
     runs-on: ubuntu-latest
     if: >-
       github.ref == 'refs/heads/develop'
     outputs:
       image: ${{ steps.set-matrix.outputs.image_result }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@master
-
-      - name: Use golang 1.19.4
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.19.4'
-
-      - name: Use Node.js 18.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-
-      - name: Install packages and build
-        run: yarn install && yarn build
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: build/linux/Dockerfile
-          tags: trivy-portainer:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
-
-      - name: Load docker image
-        run: |
-          docker load --input /tmp/trivy-portainer-image.tar
-
-      - name: Run Trivy vulnerability scanner
+      - name: scan vulnerabilities by Trivy 
         uses: docker://docker.io/aquasec/trivy:latest
-        continue-on-error: true 
+        continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
 
-      - name: Upload image security scan result as artifact
+      - name: upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
+      - name: develop scan report export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-result")
 
-      - name: Upload go result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-${{github.run_id}}
           path: image-result.html
 
-      - name: Analyse the trivy result
+      - name: analyse vulnerabilities
         id: set-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
-          echo "::set-output name=image_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
+          echo "image_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
-    name: Analyse scan result
+    name: Analyse Scan Results
     needs: [client-dependencies, server-dependencies, image-vulnerability]
     runs-on: ubuntu-latest
     if: >-
       github.ref == 'refs/heads/develop'
     strategy:
-      matrix: 
+      matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
         image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
     steps:
-      - name: Display the results of js, go and image
+      - name: display the results of js, Go, and image scan
         run: |
-          echo ${{ matrix.js.status }}
-          echo ${{ matrix.go.status }}
-          echo ${{ matrix.image.status }}
-          echo ${{ matrix.js.summary }}
-          echo ${{ matrix.go.summary }}
-          echo ${{ matrix.image.summary }}
+          echo "${{ matrix.js.status }}"
+          echo "${{ matrix.go.status }}"
+          echo "${{ matrix.image.status }}"
+          echo "${{ matrix.js.summary }}"
+          echo "${{ matrix.go.summary }}"
+          echo "${{ matrix.image.summary }}"
 
-      - name: Send Slack message
+      - name: send message to Slack 
         if: >- 
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
           matrix.image.status == 'failure'
-        uses: slackapi/slack-github-action@v1.18.0
+        uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |
             {

.github/workflows/pr-security.yml

@@ -12,10 +12,11 @@ on:
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
-    
+      - '.github/workflows/pr-security.yml'
+
 jobs:
   client-dependencies:
-    name: Client dependency check
+    name: Client Dependency Check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
@@ -23,9 +24,10 @@ jobs:
     outputs:
       jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
     steps:
-      - uses: actions/checkout@master
+      - name: checkout repository
+        uses: actions/checkout@master
 
-      - name: Run Snyk to check for vulnerabilities
+      - name: scan vulnerabilities by Snyk
         uses: snyk/actions/node@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
@@ -33,13 +35,13 @@ jobs:
         with:
           json: true
 
-      - name: Upload js security scan result as artifact
+      - name: upload scan result as pull-request artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-feat-result
           path: snyk.json
 
-      - name: Download artifacts from develop branch
+      - name: download artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -51,24 +53,24 @@ jobs:
             echo "null" > ./js-snyk-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
+      - name: pr vs develop scan report comparison export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=table --export --export-filename="/data/js-result")
 
-      - name: Upload js result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-js-result-compare-to-develop-${{github.run_id}}
           path: js-result.html
 
-      - name: Analyse the js diff result
+      - name: analyse different vulnerabilities against develop branch
         id: set-diff-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
-          echo "::set-output name=js_diff_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=matrix)
+          echo "js_diff_result=${result}" >> $GITHUB_OUTPUT
 
   server-dependencies:
-    name: Server dependency check
+    name: Server Dependency Check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
@@ -76,16 +78,18 @@ jobs:
     outputs:
       godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
     steps:
-      - uses: actions/checkout@master
+      - name: checkout repository
+        uses: actions/checkout@master
 
-      - uses: actions/setup-go@v3
+      - name: install Go
+        uses: actions/setup-go@v3
         with:
-          go-version: '1.19.4'
+          go-version: '1.19.5'
 
-      - name: Download go modules
+      - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
 
-      - name: Run Snyk to check for vulnerabilities
+      - name: scan vulnerabilities by Snyk
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
@@ -93,13 +97,13 @@ jobs:
           yarn global add snyk
           snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: Upload go security scan result as artifact
+      - name: upload scan result as pull-request artifact
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-feature-result
           path: snyk.json
 
-      - name: Download artifacts from develop branch
+      - name: download artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -111,24 +115,24 @@ jobs:
             echo "null" > ./go-snyk-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
+      - name: pr vs develop scan report comparison export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=table --export --export-filename="/data/go-result")
 
-      - name: Upload go result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-go-result-compare-to-develop-${{github.run_id}}
           path: go-result.html
 
-      - name: Analyse the go diff result
+      - name: analyse different vulnerabilities against develop branch
         id: set-diff-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
-          echo "::set-output name=go_diff_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=matrix)
+          echo "go_diff_result=${result}" >> $GITHUB_OUTPUT
 
   image-vulnerability:
-    name: Build docker image and Image vulnerability check
+    name: Image Vulnerability Check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
@@ -136,50 +140,53 @@ jobs:
     outputs:
       imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
     steps:
-      - name: Checkout code
+      - name: checkout code
         uses: actions/checkout@master
 
-      - name: Use golang 1.19.4
+      - name: install Go 1.19.5
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.4'
+          go-version: '1.19.5'
 
-      - name: Use Node.js 18.x
-        uses: actions/setup-node@v1
+      - name: install Node.js 18.x
+        uses: actions/setup-node@v3
         with:
           node-version: 18.x
 
-      - name: Install packages and build
-        run: yarn install && yarn build
+      - name: Install packages
+        run: yarn --frozen-lockfile
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+      - name: build
+        run: make build-all
 
-      - name: Build and push
-        uses: docker/build-push-action@v2
+      - name: set up docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: build and compress image
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: build/linux/Dockerfile
           tags: trivy-portainer:${{ github.sha }}
           outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
 
-      - name: Load docker image
+      - name: load docker image
         run: |
           docker load --input /tmp/trivy-portainer-image.tar
 
-      - name: Run Trivy vulnerability scanner
+      - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
-        continue-on-error: true 
+        continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
-      - name: Upload image security scan result as artifact
+      - name: upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-feature-result
           path: image-trivy.json
 
-      - name: Download artifacts from develop branch
+      - name: download artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -191,46 +198,45 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: Export scan result to html file 
-        run: | 
-          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
+      - name: pr vs develop scan report comparison export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-result")
 
-      - name: Upload image result html file
+      - name: upload html file as artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-compare-to-develop-${{github.run_id}}
           path: image-result.html
 
-      - name: Analyse the image diff result
+      - name: analyse different vulnerabilities against develop branch
         id: set-diff-matrix
-        run: | 
-          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
-          echo "::set-output name=image_diff_result::${result}"
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
+          echo "image_diff_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
-    name: Analyse scan result compared to develop
+    name: Analyse Scan Result Against develop Branch
     needs: [client-dependencies, server-dependencies, image-vulnerability]
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
       github.event.review.body == '/scan'
     strategy:
-      matrix: 
+      matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
         imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
     steps:
-
-      - name: Check job status of diff result
+      - name: check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff.status == 'failure' 
+          matrix.imagediff.status == 'failure'
         run: |
-          echo ${{ matrix.jsdiff.status }}
-          echo ${{ matrix.godiff.status }}
-          echo ${{ matrix.imagediff.status }}
-          echo ${{ matrix.jsdiff.summary }}
-          echo ${{ matrix.godiff.summary }}
-          echo ${{ matrix.imagediff.summary }}
+          echo "${{ matrix.jsdiff.status }}"
+          echo "${{ matrix.godiff.status }}"
+          echo "${{ matrix.imagediff.status }}"
+          echo "${{ matrix.jsdiff.summary }}"
+          echo "${{ matrix.godiff.summary }}"
+          echo "${{ matrix.imagediff.summary }}"
           exit 1

.github/workflows/test.yaml

@@ -8,12 +8,12 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
-        run: yarn test:client
+        run: yarn jest --maxWorkers=2
   # test-server:
   #   runs-on: ubuntu-latest
   #   env:

.github/workflows/validate-openapi-spec.yaml

@@ -0,0 +1,29 @@
+name: Validate OpenAPI specs
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+      - 'release/*'
+
+jobs:
+  openapi-spec:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Download golang modules
+        run: cd ./api && go get -t -v -d ./...
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
+
+      - name: Validate OpenAPI Spec
+        run: make docs-validate

.github/workflows/validate-openapi-spec.yml

@@ -1,53 +0,0 @@
-name: Validate
-
-on:
-  pull_request:
-    branches:
-      - master
-      - develop
-      - 'release/*'
-
-jobs:
-  openapi-spec:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-    - name: Setup Node v14
-      uses: actions/setup-node@v2
-      with:
-        node-version: 14
-
-    # https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - name: Setup Go v1.17.3
-      uses: actions/setup-go@v2
-      with:
-        go-version: '^1.17.3'
-
-    - name: Prebuild docs
-      run: yarn prebuild:docs
-
-    - name: Build OpenAPI 2.0 Spec
-      run: yarn build:docs
-
-    # Install dependencies globally to bypass installing all frontend deps
-    - name: Install swagger2openapi and swagger-cli
-      run: yarn global add swagger2openapi @apidevtools/swagger-cli
-
-    # OpenAPI2.0 does not support multiple body params (which we utilise in some of our handlers).
-    # OAS3.0 however does support multiple body params - hence its best to convert the generated OAS 2.0
-    # to OAS 3.0 and validate the output of generated OAS 3.0 instead.
-    - name: Convert OpenAPI 2.0 to OpenAPI 3.0 and validate spec
-      run: yarn validate:docs

.gitignore

@@ -16,3 +16,5 @@ __debug_bin
 api/docs
 .idea
 .env
+go.work.sum
+

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+yarn lint-staged

.storybook/main.js

@@ -1,55 +0,0 @@
-const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
-
-module.exports = {
-  stories: ['../app/**/*.stories.mdx', '../app/**/*.stories.@(ts|tsx)'],
-  addons: [
-    '@storybook/addon-links',
-    '@storybook/addon-essentials',
-    {
-      name: '@storybook/addon-postcss',
-      options: {
-        cssLoaderOptions: {
-          importLoaders: 1,
-          modules: {
-            localIdentName: '[path][name]__[local]',
-            auto: true,
-            exportLocalsConvention: 'camelCaseOnly',
-          },
-        },
-        postcssLoaderOptions: {
-          implementation: require('postcss'),
-        },
-      },
-    },
-  ],
-  webpackFinal: (config) => {
-    config.resolve.plugins = [
-      ...(config.resolve.plugins || []),
-      new TsconfigPathsPlugin({
-        extensions: config.resolve.extensions,
-      }),
-    ];
-
-    const svgRule = config.module.rules.find((rule) => rule.test && typeof rule.test.test === 'function' && rule.test.test('.svg'));
-    svgRule.test = new RegExp(svgRule.test.source.replace('svg|', ''));
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      type: 'asset',
-      resourceQuery: { not: [/c/] }, // exclude react component if *.svg?url
-    });
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      issuer: /\.(js|ts)(x)?$/,
-      resourceQuery: /c/, // *.svg?c
-      use: [{ loader: '@svgr/webpack', options: { icon: true } }],
-    });
-
-    return config;
-  },
-  core: {
-    builder: 'webpack5',
-  },
-  staticDirs: ['./public'],
-};

.storybook/main.ts

@@ -0,0 +1,95 @@
+import { StorybookConfig } from '@storybook/react-webpack5';
+
+import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
+import { Configuration } from 'webpack';
+import postcss from 'postcss';
+const config: StorybookConfig = {
+  stories: ['../app/**/*.stories.@(ts|tsx)'],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    {
+      name: '@storybook/addon-styling',
+      options: {
+        cssLoaderOptions: {
+          importLoaders: 1,
+          modules: {
+            localIdentName: '[path][name]__[local]',
+            auto: true,
+            exportLocalsConvention: 'camelCaseOnly',
+          },
+        },
+        postCss: {
+          implementation: postcss,
+        },
+      },
+    },
+  ],
+  webpackFinal: (config) => {
+    const rules = config?.module?.rules || [];
+
+    const imageRule = rules.find((rule) => {
+      const test = (rule as { test: RegExp }).test;
+
+      if (!test) {
+        return false;
+      }
+
+      return test.test('.svg');
+    }) as { [key: string]: any };
+
+    imageRule.exclude = /\.svg$/;
+
+    rules.unshift({
+      test: /\.svg$/i,
+      type: 'asset',
+      resourceQuery: {
+        not: [/c/],
+      }, // exclude react component if *.svg?url
+    });
+
+    rules.unshift({
+      test: /\.svg$/i,
+      issuer: /\.(js|ts)(x)?$/,
+      resourceQuery: /c/,
+      // *.svg?c
+      use: [
+        {
+          loader: '@svgr/webpack',
+          options: {
+            icon: true,
+          },
+        },
+      ],
+    });
+    return {
+      ...config,
+      resolve: {
+        ...config.resolve,
+        plugins: [
+          ...(config.resolve?.plugins || []),
+          new TsconfigPathsPlugin({
+            extensions: config.resolve?.extensions,
+          }),
+        ],
+      },
+      module: {
+        ...config.module,
+        rules,
+      },
+    } satisfies Configuration;
+  },
+  staticDirs: ['./public'],
+  typescript: {
+    reactDocgen: 'react-docgen-typescript',
+  },
+  framework: {
+    name: '@storybook/react-webpack5',
+    options: {},
+  },
+  docs: {
+    autodocs: true,
+  },
+};
+
+export default config;

.vscode.example/portainer.code-snippets

@@ -15,6 +15,15 @@
   // 	],
   // 	"description": "Log output to console"
   // }
+  "React Named Export Component": {
+    "prefix": "rnec",
+    "body": [
+      "export function $TM_FILENAME_BASE() {",
+      "  return <div>$TM_FILENAME_BASE</div>;",
+      "}"
+    ],
+    "description": "React Named Export Component"
+  },
   "Component": {
     "scope": "javascript",
     "prefix": "mycomponent",

CONTRIBUTING.md

@@ -79,25 +79,33 @@ The feature request process is similar to the bug report process but has an extr
 
 Ensure you have Docker, Node.js, yarn, and Golang installed in the correct versions.
 
-Install dependencies with yarn:
+Install dependencies:
 
 ```sh
-$ yarn
+$ make deps
 ```
 
 Then build and run the project in a Docker container:
 
 ```sh
-$ yarn start
+$ make dev
 ```
 
-Portainer can now be accessed at <https://localhost:9443>.
+Portainer server can now be accessed at <https://localhost:9443>. and UI dev server runs on <http://localhost:8999>.
+
+if you want to build the project you can run:
+
+```sh
+make build-all
+```
+
+For additional make commands, run `make help`.
 
 Find more detailed steps at <https://docs.portainer.io/contribute/build>.
 
-### Build customisation
+### Build customization
 
-You can customise the following settings:
+You can customize the following settings:
 
 - `PORTAINER_DATA`: The host dir or volume name used by portainer (default is `/tmp/portainer`, which won't persist over reboots).
 - `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`).

Makefile

@@ -0,0 +1,126 @@
+# See: https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63
+# For a list of valid GOOS and GOARCH values
+# Note: these can be overriden on the command line e.g. `make PLATFORM=<platform> ARCH=<arch>`
+PLATFORM=$(shell go env GOOS)
+ARCH=$(shell go env GOARCH)
+
+# build target, can be one of "production", "testing", "development"
+ENV=development
+WEBPACK_CONFIG=webpack/webpack.$(ENV).js
+TAG=latest
+
+SWAG=go run github.com/swaggo/swag/cmd/swag@v1.8.11 
+GOTESTSUM=go run gotest.tools/gotestsum@latest
+
+# Don't change anything below this line unless you know what you're doing
+.DEFAULT_GOAL := help
+
+
+##@ Building
+.PHONY: init-dist build-storybook build build-client build-server build-image devops
+init-dist:
+	@mkdir -p dist
+
+build-all: deps build-server build-client ## Build the client, server and download external dependancies (doesn't build an image)
+
+build-client: init-dist ## Build the client
+	export NODE_ENV=$(ENV) && yarn build --config $(WEBPACK_CONFIG)
+
+build-server: init-dist ## Build the server binary
+	./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
+
+build-image: build-all ## Build the Portainer image locally
+	docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
+
+build-storybook: ## Build and serve the storybook files
+	yarn storybook:build
+
+devops: clean deps build-client ## Build the everything target specifically for CI
+	echo "Building the devops binary..."
+	@./build/build_binary_azuredevops.sh "$(PLATFORM)" "$(ARCH)"
+
+##@ Build dependencies
+.PHONY: deps server-deps client-deps tidy
+deps: server-deps client-deps ## Download all client and server build dependancies
+
+server-deps: init-dist ## Download dependant server binaries
+	@./build/download_binaries.sh $(PLATFORM) $(ARCH)
+
+client-deps: ## Install client dependencies
+	yarn
+
+tidy: ## Tidy up the go.mod file
+	cd api && go mod tidy
+
+
+##@ Cleanup
+.PHONY: clean
+clean: ## Remove all build and download artifacts
+	@echo "Clearing the dist directory..."
+	@rm -rf dist/*
+
+
+##@ Testing
+.PHONY: test test-client test-server
+test: test-server test-client ## Run all tests
+
+test-client: ## Run client tests
+	yarn test
+
+test-server:	## Run server tests
+	cd api && $(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
+
+##@ Dev
+.PHONY: dev dev-client dev-server
+dev: ## Run both the client and server in development mode	
+	make dev-server
+	make dev-client
+
+dev-client: ## Run the client in development mode 
+	yarn dev
+
+dev-server: build-server ## Run the server in development mode
+	@./dev/run_container.sh
+
+
+##@ Format
+.PHONY: format format-client format-server
+
+format: format-client format-server ## Format all code
+
+format-client: ## Format client code
+	yarn format
+
+format-server: ## Format server code
+	cd api && go fmt ./...
+
+##@ Lint
+.PHONY: lint lint-client lint-server
+lint: lint-client lint-server ## Lint all code
+
+lint-client: ## Lint client code
+	yarn lint
+
+lint-server: ## Lint server code
+	cd api && go vet ./...
+
+
+##@ Extension
+.PHONY: dev-extension
+dev-extension: build-server build-client ## Run the extension in development mode
+	make local -f build/docker-extension/Makefile
+
+
+##@ Docs
+.PHONY: docs-build docs-validate docs-clean docs-validate-clean
+docs-build: init-dist ## Build docs
+	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
+
+docs-validate: docs-build ## Validate docs
+	yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
+	yarn swagger-cli validate dist/docs/openapi.yaml
+
+##@ Helpers
+.PHONY: help
+help:  ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

api/.golangci.yaml

@@ -1,8 +1,13 @@
 linters:
-  # Disable all linters.
+  # Disable all linters, the defaults don't pass on our code yet
   disable-all: true
+
+  # Enable these for now
   enable:
     - depguard
+    - govet
+    - errorlint
+    - exportloopref
 linters-settings:
   depguard:
     list-type: denylist
@@ -13,14 +18,12 @@ linters-settings:
     packages-with-error-message:
       - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
     ignore-file-rules:
-      - "**/*_test.go"
-    # Create additional guards that follow the same configuration pattern.
-    # Results from all guards are aggregated together.
-    # additional-guards:
-    #   - list-type: allowlist
-    #     include-go-root: false
-    #     packages:
-    #       - github.com/sirupsen/logrus
-    #     # Specify rules by which the linter ignores certain files for consideration.
-    #     ignore-file-rules:
-    #       - "!**/*_test.go"
+      - '**/*_test.go'
+
+# errorlint is causing a typecheck error for some reason. The go compiler will report these
+# anyway, so ignore them from the linter
+issues:
+  exclude-rules:
+    - path: ./
+      linters:
+        - typecheck

api/agent/version.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"time"
@@ -42,7 +43,9 @@ func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (port
 	if err != nil {
 		return 0, "", err
 	}
-	defer resp.Body.Close()
+
+	io.Copy(io.Discard, resp.Body)
+	resp.Body.Close()
 
 	if resp.StatusCode != http.StatusNoContent {
 		return 0, "", fmt.Errorf("Failed request with status %d", resp.StatusCode)

api/apikey/service_test.go

@@ -22,8 +22,7 @@ func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
 func Test_GenerateApiKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -76,8 +75,7 @@ func Test_GenerateApiKey(t *testing.T) {
 func Test_GetAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -96,8 +94,7 @@ func Test_GetAPIKey(t *testing.T) {
 func Test_GetAPIKeys(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -117,8 +114,7 @@ func Test_GetAPIKeys(t *testing.T) {
 func Test_GetDigestUserAndKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -153,8 +149,7 @@ func Test_GetDigestUserAndKey(t *testing.T) {
 func Test_UpdateAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -199,8 +194,7 @@ func Test_UpdateAPIKey(t *testing.T) {
 func Test_DeleteAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -240,8 +234,7 @@ func Test_DeleteAPIKey(t *testing.T) {
 func Test_InvalidateUserKeyCache(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 

api/archive/targz.go

@@ -3,6 +3,7 @@ package archive
 import (
 	"archive/tar"
 	"compress/gzip"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -84,7 +85,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
 	for {
 		header, err := tarReader.Next()
 
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 
@@ -109,7 +110,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
 			}
 			outFile.Close()
 		default:
-			return fmt.Errorf("Tar: uknown type: %v in %s",
+			return fmt.Errorf("tar: unknown type: %v in %s",
 				header.Typeflag,
 				header.Name)
 		}

api/chisel/service.go

@@ -3,6 +3,7 @@ package chisel
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"sync"
 	"time"
@@ -58,7 +59,11 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 	httpClient := &http.Client{
 		Timeout: 3 * time.Second,
 	}
-	_, err = httpClient.Do(req)
+
+	resp, err := httpClient.Do(req)
+	io.Copy(io.Discard, resp.Body)
+	resp.Body.Close()
+
 	return err
 }
 

api/cli/cli.go

@@ -72,6 +72,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		if err != nil {
 			panic(err)
 		}
+
 		*flags.Assets = filepath.Join(filepath.Dir(ex), *flags.Assets)
 	}
 
@@ -80,7 +81,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 // ValidateFlags validates the values of the flags.
 func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
-
 	displayDeprecationWarnings(flags)
 
 	err := validateEndpointURL(*flags.EndpointURL)
@@ -111,31 +111,38 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 }
 
 func validateEndpointURL(endpointURL string) error {
-	if endpointURL != "" {
-		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
-			return errInvalidEndpointProtocol
-		}
+	if endpointURL == "" {
+		return nil
+	}
 
-		if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
-			socketPath := strings.TrimPrefix(endpointURL, "unix://")
-			socketPath = strings.TrimPrefix(socketPath, "npipe://")
-			if _, err := os.Stat(socketPath); err != nil {
-				if os.IsNotExist(err) {
-					return errSocketOrNamedPipeNotFound
-				}
-				return err
+	if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
+		return errInvalidEndpointProtocol
+	}
+
+	if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
+		socketPath := strings.TrimPrefix(endpointURL, "unix://")
+		socketPath = strings.TrimPrefix(socketPath, "npipe://")
+		if _, err := os.Stat(socketPath); err != nil {
+			if os.IsNotExist(err) {
+				return errSocketOrNamedPipeNotFound
 			}
+
+			return err
 		}
 	}
+
 	return nil
 }
 
 func validateSnapshotInterval(snapshotInterval string) error {
-	if snapshotInterval != "" {
-		_, err := time.ParseDuration(snapshotInterval)
-		if err != nil {
-			return errInvalidSnapshotInterval
-		}
+	if snapshotInterval == "" {
+		return nil
 	}
+
+	_, err := time.ParseDuration(snapshotInterval)
+	if err != nil {
+		return errInvalidSnapshotInterval
+	}
+
 	return nil
 }

api/cli/confirm.go

@@ -12,13 +12,14 @@ func Confirm(message string) (bool, error) {
 	fmt.Printf("%s [y/N]", message)
 
 	reader := bufio.NewReader(os.Stdin)
+
 	answer, err := reader.ReadString('\n')
 	if err != nil {
 		return false, err
 	}
-	answer = strings.Replace(answer, "\n", "", -1)
+
+	answer = strings.ReplaceAll(answer, "\n", "")
 	answer = strings.ToLower(answer)
 
 	return answer == "y" || answer == "yes", nil
-
 }

api/cmd/portainer/main.go

@@ -24,17 +24,18 @@ import (
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/hostmanagement/openamt"
 	"github.com/portainer/portainer/api/http"
-	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/edgestacks"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/internal/upgrade"
@@ -233,8 +234,8 @@ func initSSLService(addr, certPath, keyPath string, fileService portainer.FileSe
 	return sslService, nil
 }
 
-func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
-	return docker.NewClientFactory(signatureService, reverseTunnelService)
+func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *dockerclient.ClientFactory {
+	return dockerclient.NewClientFactory(signatureService, reverseTunnelService)
 }
 
 func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
@@ -244,7 +245,7 @@ func initKubernetesClientFactory(signatureService portainer.DigitalSignatureServ
 func initSnapshotService(
 	snapshotIntervalFromFlag string,
 	dataStore dataservices.DataStore,
-	dockerClientFactory *docker.ClientFactory,
+	dockerClientFactory *dockerclient.ClientFactory,
 	kubernetesClientFactory *kubecli.ClientFactory,
 	shutdownCtx context.Context,
 ) (portainer.SnapshotService, error) {
@@ -346,147 +347,6 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	tlsConfiguration := portainer.TLSConfiguration{
-		TLS:           *flags.TLS,
-		TLSSkipVerify: *flags.TLSSkipVerify,
-	}
-
-	if *flags.TLS {
-		tlsConfiguration.TLSCACertPath = *flags.TLSCacert
-		tlsConfiguration.TLSCertPath = *flags.TLSCert
-		tlsConfiguration.TLSKeyPath = *flags.TLSKey
-	} else if !*flags.TLS && *flags.TLSSkipVerify {
-		tlsConfiguration.TLS = true
-	}
-
-	endpointID := dataStore.Endpoint().GetNextIdentifier()
-	endpoint := &portainer.Endpoint{
-		ID:                 portainer.EndpointID(endpointID),
-		Name:               "primary",
-		URL:                *flags.EndpointURL,
-		GroupID:            portainer.EndpointGroupID(1),
-		Type:               portainer.DockerEnvironment,
-		TLSConfig:          tlsConfiguration,
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
-	}
-
-	if strings.HasPrefix(endpoint.URL, "tcp://") {
-		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
-		if err != nil {
-			return err
-		}
-
-		agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig)
-		if err != nil {
-			return err
-		}
-
-		if agentOnDockerEnvironment {
-			endpoint.Type = portainer.AgentOnDockerEnvironment
-		}
-	}
-
-	err := snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		log.Error().
-			Str("endpoint", endpoint.Name).
-			Str("URL", endpoint.URL).
-			Err(err).
-			Msg("environment snapshot error")
-	}
-
-	return dataStore.Endpoint().Create(endpoint)
-}
-
-func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	if strings.HasPrefix(endpointURL, "tcp://") {
-		_, err := client.ExecutePingOperation(endpointURL, nil)
-		if err != nil {
-			return err
-		}
-	}
-
-	endpointID := dataStore.Endpoint().GetNextIdentifier()
-	endpoint := &portainer.Endpoint{
-		ID:                 portainer.EndpointID(endpointID),
-		Name:               "primary",
-		URL:                endpointURL,
-		GroupID:            portainer.EndpointGroupID(1),
-		Type:               portainer.DockerEnvironment,
-		TLSConfig:          portainer.TLSConfiguration{},
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
-	}
-
-	err := snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		log.Error().
-			Str("endpoint", endpoint.Name).
-			Str("URL", endpoint.URL).Err(err).
-			Msg("environment snapshot error")
-	}
-
-	return dataStore.Endpoint().Create(endpoint)
-}
-
-func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	if *flags.EndpointURL == "" {
-		return nil
-	}
-
-	endpoints, err := dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	if len(endpoints) > 0 {
-		log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
-
-		return nil
-	}
-
-	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, dataStore, snapshotService)
-	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
-}
-
 func loadEncryptionSecretKey(keyfilename string) []byte {
 	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
 	if err != nil {
@@ -627,10 +487,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		}
 	}
 
-	err = initEndpoint(flags, dataStore, snapshotService)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed initializing environment")
-	}
+	// channel to control when the admin user is created
+	adminCreationDone := make(chan struct{}, 1)
+
+	go endpointutils.InitEndpoint(shutdownCtx, adminCreationDone, flags, dataStore, snapshotService)
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
@@ -665,6 +525,9 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 			if err != nil {
 				log.Fatal().Err(err).Msg("failed creating admin user")
 			}
+
+			// notify the admin user is created, the endpoint initialization can start
+			adminCreationDone <- struct{}{}
 		} else {
 			log.Info().Msg("instance already has an administrator user defined, skipping admin password related flags.")
 		}
@@ -676,7 +539,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
+	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer, dockerClientFactory, dataStore)
 	deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
@@ -684,7 +547,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
-	upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer)
+	upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer, kubernetesClientFactory)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing upgrade service")
 	}
@@ -738,6 +601,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		StackDeployer:               stackDeployer,
 		DemoService:                 demoService,
 		UpgradeService:              upgradeService,
+		AdminCreationDone:           adminCreationDone,
 	}
 }
 

api/crypto/ecdsa.go

@@ -7,7 +7,6 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/hex"
-	"math/big"
 
 	"github.com/portainer/libcrypto"
 )
@@ -115,9 +114,6 @@ func (service *ECDSAService) CreateSignature(message string) (string, error) {
 
 	hash := libcrypto.HashFromBytes([]byte(message))
 
-	r := big.NewInt(0)
-	s := big.NewInt(0)
-
 	r, s, err := ecdsa.Sign(rand.Reader, service.privateKey, hash)
 	if err != nil {
 		return "", err

api/crypto/tls.go

@@ -20,6 +20,8 @@ func CreateTLSConfiguration() *tls.Config {
 			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
 			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
 		},
 	}
 }

api/database/boltdb/db.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"time"
@@ -182,7 +183,7 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
 func (connection *DbConnection) ExportRaw(filename string) error {
 	databasePath := connection.GetDatabaseFilePath()
 	if _, err := os.Stat(databasePath); err != nil {
-		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
+		return fmt.Errorf("stat on %s failed, error: %w", databasePath, err)
 	}
 
 	b, err := connection.ExportJSON(databasePath, true)
@@ -201,6 +202,20 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 	return b
 }
 
+// keyToString Converts a key to a string value suitable for logging
+func keyToString(b []byte) string {
+	if len(b) != 8 {
+		return string(b)
+	}
+
+	v := binary.BigEndian.Uint64(b)
+	if v <= math.MaxInt32 {
+		return fmt.Sprintf("%d", v)
+	}
+
+	return string(b)
+}
+
 // CreateBucket is a generic function used to create a bucket inside a database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
 	return connection.UpdateTx(func(tx portainer.Transaction) error {
@@ -237,7 +252,7 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 
 		data := bucket.Get(key)
 		if data == nil {
-			return dserrors.ErrObjectNotFound
+			return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 		}
 
 		err := connection.UnmarshalObjectWithJsoniter(data, object)

api/database/boltdb/json_test.go

@@ -129,7 +129,7 @@ func Test_UnMarshalObjectUnencrypted(t *testing.T) {
 			var object string
 			err := conn.UnmarshalObject(test.object, &object)
 			is.NoError(err)
-			is.Equal(test.expected, string(object))
+			is.Equal(test.expected, object)
 		})
 	}
 }

api/database/boltdb/tx.go

@@ -2,6 +2,7 @@ package boltdb
 
 import (
 	"bytes"
+	"fmt"
 
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
@@ -24,13 +25,10 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 
 	value := bucket.Get(key)
 	if value == nil {
-		return dserrors.ErrObjectNotFound
+		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	data := make([]byte, len(value))
-	copy(data, value)
-
-	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
+	return tx.conn.UnmarshalObjectWithJsoniter(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -48,7 +46,7 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
 	return bucket.Delete(key)
 }
 
-func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	cursor := bucket.Cursor()
@@ -58,7 +56,7 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 			return err
 		}
 
-		if id, ok := matching(obj); ok {
+		if id, ok := matchingFn(obj); ok {
 			err := bucket.Delete(tx.conn.ConvertToKey(id))
 			if err != nil {
 				return err
@@ -74,7 +72,6 @@ func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
 	id, err := bucket.NextSequence()
 	if err != nil {
 		log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifer")
-
 		return 0
 	}
 
@@ -92,7 +89,7 @@ func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, i
 		return err
 	}
 
-	return bucket.Put(tx.conn.ConvertToKey(int(id)), data)
+	return bucket.Put(tx.conn.ConvertToKey(id), data)
 }
 
 func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
@@ -115,45 +112,33 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
 	return bucket.Put(id, data)
 }
 
-func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObject(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
 
 	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
@@ -162,7 +147,7 @@ func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte
 			return err
 		}
 
-		obj, err = append(obj)
+		obj, err = appendFn(obj)
 		if err != nil {
 			return err
 		}

api/database/boltdb/tx_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const testBucketName = "test-bucket"
@@ -97,7 +97,7 @@ func TestTxs(t *testing.T) {
 	err = conn.ViewTx(func(tx portainer.Transaction) error {
 		return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
 	})
-	if err != dserrors.ErrObjectNotFound {
+	if !dataservices.IsErrObjectNotFound(err) {
 		t.Fatal(err)
 	}
 

api/database/database.go

@@ -9,8 +9,7 @@ import (
 
 // NewDatabase should use config options to return a connection to the requested database
 func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
-	switch storeType {
-	case "boltdb":
+	if storeType == "boltdb" {
 		return &boltdb.DbConnection{
 			Path:          storePath,
 			EncryptionKey: encryptionKey,

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -2,10 +2,11 @@ package apikeyrepository
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
 )
@@ -78,12 +79,12 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 			return &portainer.APIKey{}, nil
 		})
 
-	if err == stop {
+	if errors.Is(err, stop) {
 		return k, nil
 	}
 
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err

api/dataservices/customtemplate/customtemplate.go

@@ -1,17 +1,12 @@
 package customtemplate
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "customtemplates"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "customtemplates"
 
 // Service represents a service for managing custom template data.
 type Service struct {
@@ -38,22 +33,11 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 	var customTemplates = make([]portainer.CustomTemplate, 0)
 
-	err := service.connection.GetAll(
+	return customTemplates, service.connection.GetAll(
 		BucketName,
 		&portainer.CustomTemplate{},
-		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
-			customTemplate, ok := obj.(*portainer.CustomTemplate)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
-				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
-			}
-			customTemplates = append(customTemplates, *customTemplate)
-
-			return &portainer.CustomTemplate{}, nil
-		})
-
-	return customTemplates, err
+		dataservices.AppendFn(&customTemplates),
+	)
 }
 
 // CustomTemplate returns an custom template by ID.

api/dataservices/edgegroup/tx.go

@@ -2,11 +2,10 @@ package edgegroup
 
 import (
 	"errors"
-	"fmt"
+
+	"github.com/portainer/portainer/api/dataservices"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
@@ -22,21 +21,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
 	var groups = make([]portainer.EdgeGroup, 0)
 
-	err := service.tx.GetAllWithJsoniter(
+	return groups, service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.EdgeGroup{},
-		func(obj interface{}) (interface{}, error) {
-			group, ok := obj.(*portainer.EdgeGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
-				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
-			}
-			groups = append(groups, *group)
-
-			return &portainer.EdgeGroup{}, nil
-		})
-
-	return groups, err
+		dataservices.AppendFn(&groups),
+	)
 }
 
 // EdgeGroup returns an Edge group by ID.

api/dataservices/edgejob/edgejob.go

@@ -1,11 +1,8 @@
 package edgejob
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -43,22 +40,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
 
-	err := service.connection.GetAll(
+	return edgeJobs, service.connection.GetAll(
 		BucketName,
 		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
+		dataservices.AppendFn(&edgeJobs),
+	)
 }
 
 // EdgeJob returns an Edge job by ID

api/dataservices/edgejob/tx.go

@@ -2,11 +2,9 @@ package edgejob
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
@@ -22,22 +20,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
 
-	err := service.tx.GetAll(
+	return edgeJobs, service.tx.GetAll(
 		BucketName,
 		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
+		dataservices.AppendFn(&edgeJobs),
+	)
 }
 
 // EdgeJob returns an Edge job by ID

api/dataservices/edgestack/edgestack.go

@@ -1,12 +1,10 @@
 package edgestack
 
 import (
-	"fmt"
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -64,22 +62,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.EdgeStack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.EdgeStack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
-				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.EdgeStack{}, nil
-		})
-
-	return stacks, err
+		dataservices.AppendFn(&stacks),
+	)
 }
 
 // EdgeStack returns an Edge stack by ID.
@@ -159,6 +146,11 @@ func (service *Service) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc
 	})
 }
 
+// UpdateEdgeStackFuncTx is a helper function used to call UpdateEdgeStackFunc inside a transaction.
+func (service *Service) UpdateEdgeStackFuncTx(tx portainer.Transaction, ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
+	return service.Tx(tx).UpdateEdgeStackFunc(ID, updateFunc)
+}
+
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	service.mu.Lock()

api/dataservices/edgestack/tx.go

@@ -1,7 +1,6 @@
 package edgestack
 
 import (
-	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
@@ -29,7 +28,7 @@ func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
 			stack, ok := obj.(*portainer.EdgeStack)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
-				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to EdgeStack object: %s", obj)
 			}
 
 			stacks = append(stacks, *stack)
@@ -101,9 +100,16 @@ func (service ServiceTx) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *po
 	return nil
 }
 
-// UpdateEdgeStackFunc is a no-op inside a transaction.
+// Deprecated: use UpdateEdgeStack inside a transaction instead.
 func (service ServiceTx) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
-	return errors.New("cannot be called inside a transaction")
+	edgeStack, err := service.EdgeStack(ID)
+	if err != nil {
+		return err
+	}
+
+	updateFunc(edgeStack)
+
+	return service.UpdateEdgeStack(ID, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.

api/dataservices/endpoint/endpoint.go

@@ -34,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 		idxEdgeID:  make(map[string]portainer.EndpointID),
 	}
 
-	es, err := s.Endpoints()
+	es, err := s.endpoints()
 	if err != nil {
 		return nil, err
 	}
@@ -89,8 +89,7 @@ func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	})
 }
 
-// Endpoints return an array containing all the environments(endpoints).
-func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
+func (service *Service) endpoints() ([]portainer.Endpoint, error) {
 	var endpoints []portainer.Endpoint
 	var err error
 
@@ -99,8 +98,14 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 		return err
 	})
 
+	return endpoints, err
+}
+
+// Endpoints return an array containing all the environments(endpoints).
+func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
+	endpoints, err := service.endpoints()
 	if err != nil {
-		return endpoints, err
+		return nil, err
 	}
 
 	for i, e := range endpoints {

api/dataservices/endpoint/tx.go

@@ -1,9 +1,8 @@
 package endpoint
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -80,22 +79,11 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.tx.GetAllWithJsoniter(
+	return endpoints, service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Endpoint{},
-		func(obj interface{}) (interface{}, error) {
-			endpoint, ok := obj.(*portainer.Endpoint)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
-				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
-			}
-
-			endpoints = append(endpoints, *endpoint)
-
-			return &portainer.Endpoint{}, nil
-		})
-
-	return endpoints, err
+		dataservices.AppendFn(&endpoints),
+	)
 }
 
 func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {

api/dataservices/endpointgroup/endpointgroup.go

@@ -1,11 +1,8 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -70,22 +67,11 @@ func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error
 func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.connection.GetAll(
+	return endpointGroups, service.connection.GetAll(
 		BucketName,
 		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
+		dataservices.AppendFn(&endpointGroups),
+	)
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.

api/dataservices/endpointgroup/tx.go

@@ -1,11 +1,8 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
@@ -46,22 +43,11 @@ func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error
 func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.tx.GetAll(
+	return endpointGroups, service.tx.GetAll(
 		BucketName,
 		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
+		dataservices.AppendFn(&endpointGroups),
+	)
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.

api/dataservices/endpointrelation/endpointrelation.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -14,16 +13,21 @@ const BucketName = "endpoint_relations"
 
 // Service represents a service for managing environment(endpoint) relation data.
 type Service struct {
-	connection    portainer.Connection
-	updateStackFn func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
+	connection      portainer.Connection
+	updateStackFn   func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
+	updateStackFnTx func(tx portainer.Transaction, ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 }
 
 func (service *Service) BucketName() string {
 	return BucketName
 }
 
-func (service *Service) RegisterUpdateStackFunction(updateFunc func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error) {
+func (service *Service) RegisterUpdateStackFunction(
+	updateFunc func(portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
+	updateFuncTx func(portainer.Transaction, portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
+) {
 	service.updateStackFn = updateFunc
+	service.updateStackFnTx = updateFuncTx
 }
 
 // NewService creates a new instance of a service.
@@ -49,22 +53,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.connection.GetAll(
+	return all, service.connection.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID

api/dataservices/endpointrelation/tx.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -22,22 +21,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.tx.GetAll(
+	return all, service.tx.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns an Environment(Endpoint) relation object by EndpointID
@@ -151,7 +139,7 @@ func (service ServiceTx) updateEdgeStacksAfterRelationChange(previousRelationSta
 				}
 			}
 
-			service.service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
+			service.service.updateStackFnTx(service.tx, refStackId, func(edgeStack *portainer.EdgeStack) {
 				edgeStack.NumDeployments = numDeployments
 			})
 		}

api/dataservices/errors/errors.go

@@ -1,9 +1,10 @@
 package errors
 
-import "errors"
+import (
+	"errors"
+)
 
 var (
-	// TODO: i'm pretty sure this needs wrapping at several levels
 	ErrObjectNotFound     = errors.New("object not found inside the database")
 	ErrWrongDBEdition     = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 	ErrDBImportFailed     = errors.New("importing backup failed")

api/dataservices/extension/extension.go

@@ -1,17 +1,12 @@
 package extension
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "extension"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "extension"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,22 +46,12 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.GetAll(
+	return extensions, service.connection.GetAll(
 		BucketName,
 		&portainer.Extension{},
-		func(obj interface{}) (interface{}, error) {
-			extension, ok := obj.(*portainer.Extension)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
-				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
-			}
+		dataservices.AppendFn(&extensions),
+	)
 
-			extensions = append(extensions, *extension)
-
-			return &portainer.Extension{}, nil
-		})
-
-	return extensions, err
 }
 
 // Persist persists a extension inside the database.

api/dataservices/fdoprofile/fdoprofile.go

@@ -1,11 +1,8 @@
 package fdoprofile
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -38,21 +35,11 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
 	var fdoProfiles = make([]portainer.FDOProfile, 0)
 
-	err := service.connection.GetAll(
+	return fdoProfiles, service.connection.GetAll(
 		BucketName,
 		&portainer.FDOProfile{},
-		func(obj interface{}) (interface{}, error) {
-			fdoProfile, ok := obj.(*portainer.FDOProfile)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
-
-				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
-			}
-			fdoProfiles = append(fdoProfiles, *fdoProfile)
-			return &portainer.FDOProfile{}, nil
-		})
-
-	return fdoProfiles, err
+		dataservices.AppendFn(&fdoProfiles),
+	)
 }
 
 // FDOProfile returns an FDO Profile by ID.

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -1,17 +1,12 @@
 package helmuserrepository
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "helm_user_repository"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "helm_user_repository"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -38,46 +33,24 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
 	var repos = make([]portainer.HelmUserRepository, 0)
 
-	err := service.connection.GetAll(
+	return repos, service.connection.GetAll(
 		BucketName,
 		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			repos = append(repos, *r)
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return repos, err
+		dataservices.AppendFn(&repos),
+	)
 }
 
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
 func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
-	err := service.connection.GetAll(
+	return result, service.connection.GetAll(
 		BucketName,
 		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			record, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			if record.UserID == userID {
-				result = append(result, *record)
-			}
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return result, err
+		dataservices.FilterFn(&result, func(e portainer.HelmUserRepository) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // CreateHelmUserRepository creates a new HelmUserRepository object.

api/dataservices/helpers.go

@@ -0,0 +1,68 @@
+package dataservices
+
+import (
+	"errors"
+	"fmt"
+
+	perrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+// ErrStop signals the stop of computation when filtering results
+var ErrStop = errors.New("stop")
+
+func IsErrObjectNotFound(e error) bool {
+	return errors.Is(e, perrors.ErrObjectNotFound)
+}
+
+// AppendFn appends elements to the given collection slice
+func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		*collection = append(*collection, *element)
+
+		return new(T), nil
+	}
+}
+
+// FilterFn appends elements to the given collection when the predicate is true
+func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*element) {
+			*collection = append(*collection, *element)
+		}
+
+		return new(T), nil
+	}
+}
+
+// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
+// success
+func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		e, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*e) {
+			*element = *e
+			return new(T), ErrStop
+		}
+
+		return new(T), nil
+	}
+}

api/dataservices/interface.go

@@ -1,15 +1,11 @@
 package dataservices
 
-// 	"github.com/portainer/portainer/api/dataservices"
-
 import (
 	"io"
 	"time"
 
-	"github.com/portainer/portainer/api/database/models"
-	"github.com/portainer/portainer/api/dataservices/errors"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 )
 
 type (
@@ -97,6 +93,7 @@ type (
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
 		EdgeStackVersion(ID portainer.EdgeStackID) (int, bool)
 		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
+		// Deprecated: Use UpdateEdgeStackFunc instead.
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
@@ -323,7 +320,3 @@ type (
 		BucketName() string
 	}
 )
-
-func IsErrObjectNotFound(e error) bool {
-	return e == errors.ErrObjectNotFound
-}

api/dataservices/registry/registry.go

@@ -1,17 +1,12 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "registries"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "registries"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -34,7 +29,14 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-// Registry returns an registry by ID.
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
+// Registry returns a registry by ID.
 func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
 	var registry portainer.Registry
 	identifier := service.connection.ConvertToKey(int(ID))
@@ -51,25 +53,14 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 func (service *Service) Registries() ([]portainer.Registry, error) {
 	var registries = make([]portainer.Registry, 0)
 
-	err := service.connection.GetAll(
+	return registries, service.connection.GetAll(
 		BucketName,
 		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
+		dataservices.AppendFn(&registries),
+	)
 }
 
-// CreateRegistry creates a new registry.
+// Create creates a new registry.
 func (service *Service) Create(registry *portainer.Registry) error {
 	return service.connection.CreateObject(
 		BucketName,
@@ -80,13 +71,13 @@ func (service *Service) Create(registry *portainer.Registry) error {
 	)
 }
 
-// UpdateRegistry updates an registry.
+// UpdateRegistry updates a registry.
 func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, registry)
 }
 
-// DeleteRegistry deletes an registry.
+// DeleteRegistry deletes a registry.
 func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)

api/dataservices/registry/tx.go

@@ -0,0 +1,62 @@
+package registry
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Registry returns a registry by ID.
+func (service ServiceTx) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
+	var registry portainer.Registry
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &registry)
+	if err != nil {
+		return nil, err
+	}
+
+	return &registry, nil
+}
+
+// Registries returns an array containing all the registries.
+func (service ServiceTx) Registries() ([]portainer.Registry, error) {
+	var registries = make([]portainer.Registry, 0)
+
+	return registries, service.tx.GetAll(
+		BucketName,
+		&portainer.Registry{},
+		dataservices.AppendFn(&registries),
+	)
+}
+
+// Create creates a new registry.
+func (service ServiceTx) Create(registry *portainer.Registry) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			registry.ID = portainer.RegistryID(id)
+			return int(registry.ID), registry
+		},
+	)
+}
+
+// UpdateRegistry updates a registry.
+func (service ServiceTx) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, registry)
+}
+
+// DeleteRegistry deletes a registry.
+func (service ServiceTx) DeleteRegistry(ID portainer.RegistryID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -1,17 +1,17 @@
 package resourcecontrol
 
 import (
+	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "resource_control"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "resource_control"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -34,6 +34,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // ResourceControl returns a ResourceControl object by ID
 func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
 	var resourceControl portainer.ResourceControl
@@ -77,7 +84,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 
 			return &portainer.ResourceControl{}, nil
 		})
-	if err == stop {
+	if errors.Is(err, stop) {
 		return resourceControl, nil
 	}
 
@@ -88,22 +95,11 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.connection.GetAll(
+	return rcs, service.connection.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
+		dataservices.AppendFn(&rcs),
+	)
 }
 
 // CreateResourceControl creates a new ResourceControl object

api/dataservices/resourcecontrol/tx.go

@@ -0,0 +1,104 @@
+package resourcecontrol
+
+import (
+	"errors"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// ResourceControl returns a ResourceControl object by ID
+func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
+	var resourceControl portainer.ResourceControl
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &resourceControl)
+	if err != nil {
+		return nil, err
+	}
+
+	return &resourceControl, nil
+}
+
+// ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
+// to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
+// if no ResourceControl was found.
+func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
+	var resourceControl *portainer.ResourceControl
+	stop := fmt.Errorf("ok")
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.ResourceControl{},
+		func(obj interface{}) (interface{}, error) {
+			rc, ok := obj.(*portainer.ResourceControl)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
+				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+			}
+
+			if rc.ResourceID == resourceID && rc.Type == resourceType {
+				resourceControl = rc
+				return nil, stop
+			}
+
+			for _, subResourceID := range rc.SubResourceIDs {
+				if subResourceID == resourceID {
+					resourceControl = rc
+					return nil, stop
+				}
+			}
+
+			return &portainer.ResourceControl{}, nil
+		})
+	if errors.Is(err, stop) {
+		return resourceControl, nil
+	}
+
+	return nil, err
+}
+
+// ResourceControls returns all the ResourceControl objects
+func (service ServiceTx) ResourceControls() ([]portainer.ResourceControl, error) {
+	var rcs = make([]portainer.ResourceControl, 0)
+
+	return rcs, service.tx.GetAll(
+		BucketName,
+		&portainer.ResourceControl{},
+		dataservices.AppendFn(&rcs),
+	)
+}
+
+// CreateResourceControl creates a new ResourceControl object
+func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			resourceControl.ID = portainer.ResourceControlID(id)
+			return int(resourceControl.ID), resourceControl
+		},
+	)
+}
+
+// UpdateResourceControl saves a ResourceControl object.
+func (service ServiceTx) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, resourceControl)
+}
+
+// DeleteResourceControl deletes a ResourceControl object by ID
+func (service ServiceTx) DeleteResourceControl(ID portainer.ResourceControlID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}

api/dataservices/role/role.go

@@ -1,17 +1,12 @@
 package role
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "roles"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "roles"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -34,6 +29,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // Role returns a Role by ID
 func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
@@ -47,26 +49,15 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	return &set, nil
 }
 
-// Roles return an array containing all the sets.
+// Roles returns an array containing all the sets.
 func (service *Service) Roles() ([]portainer.Role, error) {
 	var sets = make([]portainer.Role, 0)
 
-	err := service.connection.GetAll(
+	return sets, service.connection.GetAll(
 		BucketName,
 		&portainer.Role{},
-		func(obj interface{}) (interface{}, error) {
-			set, ok := obj.(*portainer.Role)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-				return nil, fmt.Errorf("Failed to convert to Role object: %s", obj)
-			}
-
-			sets = append(sets, *set)
-
-			return &portainer.Role{}, nil
-		})
-
-	return sets, err
+		dataservices.AppendFn(&sets),
+	)
 }
 
 // CreateRole creates a new Role.

api/dataservices/role/tx.go

@@ -0,0 +1,57 @@
+package role
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Role returns a Role by ID
+func (service ServiceTx) Role(ID portainer.RoleID) (*portainer.Role, error) {
+	var set portainer.Role
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &set)
+	if err != nil {
+		return nil, err
+	}
+
+	return &set, nil
+}
+
+// Roles returns an array containing all the sets.
+func (service ServiceTx) Roles() ([]portainer.Role, error) {
+	var sets = make([]portainer.Role, 0)
+
+	return sets, service.tx.GetAll(
+		BucketName,
+		&portainer.Role{},
+		dataservices.AppendFn(&sets),
+	)
+}
+
+// CreateRole creates a new Role.
+func (service ServiceTx) Create(role *portainer.Role) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			role.ID = portainer.RoleID(id)
+			return int(role.ID), role
+		},
+	)
+}
+
+// UpdateRole updates a role.
+func (service ServiceTx) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, role)
+}

api/dataservices/schedule/schedule.go

@@ -1,17 +1,12 @@
 package schedule
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "schedules"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "schedules"
 
 // Service represents a service for managing schedule data.
 type Service struct {
@@ -63,22 +58,11 @@ func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			schedules = append(schedules, *schedule)
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.AppendFn(&schedules),
+	)
 }
 
 // SchedulesByJobType return a array containing all the schedules
@@ -86,24 +70,13 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			if schedule.JobType == jobType {
-				schedules = append(schedules, *schedule)
-			}
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.FilterFn(&schedules, func(e portainer.Schedule) bool {
+			return e.JobType == jobType
+		}),
+	)
 }
 
 // Create assign an ID to a new schedule and saves it.

api/dataservices/settings/settings.go

@@ -31,6 +31,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // Settings retrieve the settings object.
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings

api/dataservices/settings/tx.go

@@ -0,0 +1,31 @@
+package settings
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Settings retrieve the settings object.
+func (service ServiceTx) Settings() (*portainer.Settings, error) {
+	var settings portainer.Settings
+
+	err := service.tx.GetObject(BucketName, []byte(settingsKey), &settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return &settings, nil
+}
+
+// UpdateSettings persists a Settings object.
+func (service ServiceTx) UpdateSettings(settings *portainer.Settings) error {
+	return service.tx.UpdateObject(BucketName, []byte(settingsKey), settings)
+}

api/dataservices/snapshot/snapshot.go

@@ -1,11 +1,8 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -53,20 +50,11 @@ func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Sn
 func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
 	var snapshots = make([]portainer.Snapshot, 0)
 
-	err := service.connection.GetAllWithJsoniter(
+	return snapshots, service.connection.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
+		dataservices.AppendFn(&snapshots),
+	)
 }
 
 func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {

api/dataservices/snapshot/tx.go

@@ -1,11 +1,8 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
@@ -32,20 +29,11 @@ func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.S
 func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
 	var snapshots = make([]portainer.Snapshot, 0)
 
-	err := service.tx.GetAllWithJsoniter(
+	return snapshots, service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
+		dataservices.AppendFn(&snapshots),
+	)
 }
 
 func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {

api/dataservices/stack/stack.go

@@ -1,19 +1,16 @@
 package stack
 
 import (
-	"fmt"
+	"errors"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "stacks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "stacks"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,31 +48,22 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 
 // StackByName returns a stack object by name.
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
-	var s *portainer.Stack
+	var s portainer.Stack
 
-	stop := fmt.Errorf("ok")
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 
-			if stack.Name == name {
-				s = stack
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if err == stop {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -85,46 +73,24 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.Name == name {
-				stacks = append(stacks, stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 }
 
 // Stacks returns an array containing all the stacks.
 func (service *Service) Stacks() ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.AppendFn(&stacks),
+	)
 }
 
 // GetNextIdentifier returns the next identifier for a stack.
@@ -152,32 +118,22 @@ func (service *Service) DeleteStack(ID portainer.StackID) error {
 // StackByWebhookID returns a pointer to a stack object by webhook ID.
 // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
 func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	var s *portainer.Stack
-	stop := fmt.Errorf("ok")
+	var s portainer.Stack
+
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			var ok bool
-			s, ok = obj.(*portainer.Stack)
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
 
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-
-				return &portainer.Stack{}, nil
-			}
-
-			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if err == stop {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -188,22 +144,11 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 	stacks := make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
-				stacks = append(stacks, *stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
 }

api/dataservices/stack/tests/stack_test.go

@@ -29,8 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	b := stackBuilder{t: t, store: store}
 	b.createNewStack(newGuidString(t))
@@ -59,7 +58,7 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 		Type:         portainer.DockerComposeStack,
 		EndpointID:   2,
 		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Env:          []portainer.Pair{{"Name1", "Value1"}},
+		Env:          []portainer.Pair{{Name: "Name1", Value: "Value1"}},
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 		ProjectPath:  "/tmp/project",
@@ -87,8 +86,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := datastore.MustNewTestStore(t, true, true)
 
 	staticStack := portainer.Stack{ID: 1}
 	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}

api/dataservices/tag/tag.go

@@ -1,11 +1,8 @@
 package tag
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -45,22 +42,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.connection.GetAll(
+	return tags, service.connection.GetAll(
 		BucketName,
 		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
+		dataservices.AppendFn(&tags),
+	)
 }
 
 // Tag returns a tag by ID.

api/dataservices/tag/tx.go

@@ -2,11 +2,9 @@ package tag
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
@@ -22,22 +20,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.tx.GetAll(
+	return tags, service.tx.GetAll(
 		BucketName,
 		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
+		dataservices.AppendFn(&tags),
+	)
 }
 
 // Tag returns a tag by ID.

api/dataservices/team/team.go

@@ -1,19 +1,16 @@
 package team
 
 import (
-	"fmt"
+	"errors"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "teams"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "teams"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,31 +48,22 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
-	var t *portainer.Team
+	var t portainer.Team
 
-	stop := fmt.Errorf("ok")
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
+		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+			return strings.EqualFold(e.Name, name)
+		}),
+	)
 
-			if strings.EqualFold(team.Name, name) {
-				t = team
-				return nil, stop
-			}
-
-			return &portainer.Team{}, nil
-		})
-	if err == stop {
-		return t, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &t, nil
 	}
+
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -85,22 +73,11 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 func (service *Service) Teams() ([]portainer.Team, error) {
 	var teams = make([]portainer.Team, 0)
 
-	err := service.connection.GetAll(
+	return teams, service.connection.GetAll(
 		BucketName,
 		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
-
-			teams = append(teams, *team)
-
-			return &portainer.Team{}, nil
-		})
-
-	return teams, err
+		dataservices.AppendFn(&teams),
+	)
 }
 
 // UpdateTeam saves a Team.

api/dataservices/team/tests/team_test.go

@@ -10,8 +10,7 @@ import (
 
 func Test_teamByName(t *testing.T) {
 	t.Run("When store is empty should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(t, true, true)
-		defer teardown()
+		_, store := datastore.MustNewTestStore(t, true, true)
 
 		_, err := store.Team().TeamByName("name")
 		assert.ErrorIs(t, err, errors.ErrObjectNotFound)
@@ -19,8 +18,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is no object with the same name should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(t, true, true)
-		defer teardown()
+		_, store := datastore.MustNewTestStore(t, true, true)
 
 		teamBuilder := teamBuilder{
 			t:     t,
@@ -35,8 +33,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is an object with the same name should return the object", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(t, true, true)
-		defer teardown()
+		_, store := datastore.MustNewTestStore(t, true, true)
 
 		teamBuilder := teamBuilder{
 			t:     t,

api/dataservices/teammembership/teammembership.go

@@ -4,14 +4,13 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "team_membership"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "team_membership"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -34,6 +33,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // TeamMembership returns a TeamMembership object by ID
 func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
 	var membership portainer.TeamMembership
@@ -51,70 +57,37 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			memberships = append(memberships, *membership)
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.AppendFn(&memberships),
+	)
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.UserID == userID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.TeamID == teamID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // UpdateTeamMembership saves a TeamMembership object.

api/dataservices/teammembership/tx.go

@@ -0,0 +1,154 @@
+package teammembership
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// TeamMembership returns a TeamMembership object by ID
+func (service ServiceTx) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
+	var membership portainer.TeamMembership
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &membership)
+	if err != nil {
+		return nil, err
+	}
+
+	return &membership, nil
+}
+
+// TeamMemberships return an array containing all the TeamMembership objects.
+func (service ServiceTx) TeamMemberships() ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	return memberships, service.tx.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.AppendFn(&memberships),
+	)
+}
+
+// TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
+func (service ServiceTx) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	return memberships, service.tx.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
+}
+
+// TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
+func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	return memberships, service.tx.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
+}
+
+// UpdateTeamMembership saves a TeamMembership object.
+func (service ServiceTx) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, membership)
+}
+
+// CreateTeamMembership creates a new TeamMembership object.
+func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			membership.ID = portainer.TeamMembershipID(id)
+			return int(membership.ID), membership
+		},
+	)
+}
+
+// DeleteTeamMembership deletes a TeamMembership object.
+func (service ServiceTx) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+// DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
+func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
+	return service.tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.UserID == userID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
+}
+
+// DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
+func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
+	return service.tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.TeamID == teamID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
+}
+
+func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
+	return service.tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.TeamID == teamID && membership.UserID == userID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
+}

api/dataservices/user/user.go

@@ -1,19 +1,16 @@
 package user
 
 import (
-	"fmt"
+	"errors"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "users"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "users"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,33 +48,22 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 
 // UserByUsername returns a user by username.
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
-	var u *portainer.User
-	stop := fmt.Errorf("ok")
+	var u portainer.User
+
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if strings.EqualFold(user.Username, username) {
-				u = user
-				return nil, stop
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	if err == stop {
-		return u, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
 	}
 
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -87,48 +73,24 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 func (service *Service) Users() ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.GetAll(
+	return users, service.connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			users = append(users, *user)
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
+		dataservices.AppendFn(&users),
+	)
 }
 
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.GetAll(
+	return users, service.connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if user.Role == role {
-				users = append(users, *user)
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
 }
 
 // UpdateUser saves a user.

api/dataservices/webhook/webhook.go

@@ -1,18 +1,15 @@
 package webhook
 
 import (
-	"fmt"
+	"errors"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "webhooks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "webhooks"
 
 // Service represents a service for managing webhook data.
 type Service struct {
@@ -39,22 +36,11 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
-	err := service.connection.GetAll(
+	return webhooks, service.connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			webhooks = append(webhooks, *webhook)
-
-			return &portainer.Webhook{}, nil
-		})
-
-	return webhooks, err
+		dataservices.AppendFn(&webhooks),
+	)
 }
 
 // Webhook returns a webhook by ID.
@@ -72,33 +58,22 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 
 // WebhookByResourceID returns a webhook by the ResourceID it is associated with.
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
+	var w portainer.Webhook
+
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.ResourceID == ID
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.ResourceID == ID {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if err == stop {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -106,33 +81,22 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 
 // WebhookByToken returns a webhook by the random token it is associated with.
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
+	var w portainer.Webhook
+
 	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.Token == token
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.Token == token {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if err == stop {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
-		return nil, errors.ErrObjectNotFound
+		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err

api/datastore/backup.go

@@ -115,7 +115,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 
 	if err := store.Close(); err != nil {
 		return options.BackupPath, fmt.Errorf(
-			"error closing datastore before creating backup: %v",
+			"error closing datastore before creating backup: %w",
 			err,
 		)
 	}
@@ -126,7 +126,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 
 	if _, err := store.Open(); err != nil {
 		return options.BackupPath, fmt.Errorf(
-			"error opening datastore after creating backup: %v",
+			"error opening datastore after creating backup: %w",
 			err,
 		)
 	}

api/datastore/backup_test.go

@@ -11,8 +11,7 @@ import (
 )
 
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, true, true)
 
 	connection := store.GetConnection()
 	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
@@ -28,9 +27,7 @@ func TestCreateBackupFolders(t *testing.T) {
 }
 
 func TestStoreCreation(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, true, true)
-	defer teardown()
-
+	_, store := MustNewTestStore(t, true, true)
 	if store == nil {
 		t.Error("Expect to create a store")
 	}
@@ -41,9 +38,8 @@ func TestStoreCreation(t *testing.T) {
 }
 
 func TestBackup(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, true, true)
+	_, store := MustNewTestStore(t, true, true)
 	connection := store.GetConnection()
-	defer teardown()
 
 	t.Run("Backup should create default db backup", func(t *testing.T) {
 		v := models.Version{
@@ -71,8 +67,7 @@ func TestBackup(t *testing.T) {
 }
 
 func TestRemoveWithOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, true, true)
 
 	t.Run("successfully removes file if existent", func(t *testing.T) {
 		store.createBackupFolders()

api/datastore/datastore.go

@@ -1,6 +1,7 @@
 package datastore
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -104,7 +105,7 @@ func (store *Store) edition() portainer.SoftwareEdition {
 
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	return e == portainerErrors.ErrObjectNotFound
+	return errors.Is(e, portainerErrors.ErrObjectNotFound)
 }
 
 func (store *Store) Connection() portainer.Connection {

api/datastore/datastore_test.go

@@ -27,8 +27,7 @@ const (
 // TestStoreFull an eventually comprehensive set of tests for the Store.
 // The idea is what we write to the store, we should read back.
 func TestStoreFull(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, true, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, true, true)
 
 	testCases := map[string]func(t *testing.T){
 		"User Accounts": func(t *testing.T) {

api/datastore/init.go

@@ -1,6 +1,8 @@
 package datastore
 
 import (
+	"os"
+
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -20,6 +22,12 @@ func (store *Store) Init() error {
 }
 
 func (store *Store) checkOrCreateDefaultSettings() error {
+
+	isDDExtention := false
+	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {
+		isDDExtention = true
+	}
+
 	// TODO: these need to also be applied when importing
 	settings, err := store.SettingsService.Settings()
 	if store.IsErrObjectNotFound(err) {
@@ -51,6 +59,8 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,
 			KubectlShellImage:        portainer.DefaultKubectlShellImage,
+
+			IsDockerDesktopExtension: isDDExtention,
 		}
 
 		return store.SettingsService.UpdateSettings(defaultSettings)

api/datastore/migrate_data.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"runtime/debug"
 
-	portaineree "github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/database/models"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
@@ -48,13 +48,16 @@ func (store *Store) MigrateData() error {
 
 	err = store.FailSafeMigrate(migrator, version)
 	if err != nil {
+		err = errors.Wrap(err, "failed to migrate database")
+
+		log.Warn().Msg("migration failed, restoring database to previous version")
 		err = store.restoreWithOptions(&BackupOptions{BackupPath: backupPath})
 		if err != nil {
 			return errors.Wrap(err, "failed to restore database")
 		}
 
 		log.Info().Msg("database restored to previous version")
-		return errors.Wrap(err, "failed to migrate database")
+		return err
 	}
 
 	return nil
@@ -106,7 +109,7 @@ func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version *models
 		return errors.Wrap(err, "while updating version")
 	}
 
-	log.Info().Msg("migrating database from version " + version.SchemaVersion + " to " + portaineree.APIVersion)
+	log.Info().Msg("migrating database from version " + version.SchemaVersion + " to " + portainer.APIVersion)
 
 	err = migrator.Migrate()
 	if err != nil {

api/datastore/migrate_data_test.go

@@ -163,8 +163,7 @@ func TestMigrateData(t *testing.T) {
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, false, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, false, true)
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
 
@@ -182,8 +181,7 @@ func Test_getBackupRestoreOptions(t *testing.T) {
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
 		version := models.Version{SchemaVersion: "2.4.0"}
-		_, store, teardown := MustNewTestStore(t, true, false)
-		defer teardown()
+		_, store := MustNewTestStore(t, true, false)
 
 		err := store.VersionService.UpdateVersion(&version)
 		if err != nil {
@@ -240,7 +238,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanc
 
 	// Parse source json to db.
 	// When we create a new test store, it sets its version field automatically to latest.
-	_, store, _ := MustNewTestStore(t, true, false)
+	_, store := MustNewTestStore(t, true, false)
 
 	fmt.Println("store.path=", store.GetConnection().GetDatabaseFilePath())
 	store.connection.DeleteObject("version", []byte("VERSION"))
@@ -288,7 +286,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanc
 	// Convert database back to json.
 	databasePath := con.GetDatabaseFilePath()
 	if _, err := os.Stat(databasePath); err != nil {
-		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
+		return fmt.Errorf("stat on %s failed: %w", databasePath, err)
 	}
 
 	gotJSON, err := con.ExportJSON(databasePath, false)

api/datastore/migrate_dbversion29_test.go

@@ -14,27 +14,19 @@ const dummyLogoURL = "example.com"
 // for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
 func initTestingSettingsService(dbConn portainer.Connection, preSetObj map[string]interface{}) error {
 	//insert a obj
-	if err := dbConn.UpdateObject("settings", []byte("SETTINGS"), preSetObj); err != nil {
-		return err
-	}
-	return nil
+	return dbConn.UpdateObject("settings", []byte("SETTINGS"), preSetObj)
 }
 
 func setup(store *Store) error {
-	var err error
 	dummySettingsObj := map[string]interface{}{
 		"LogoURL": dummyLogoURL,
 	}
-	err = initTestingSettingsService(store.connection, dummySettingsObj)
-	if err != nil {
-		return err
-	}
-	return nil
+
+	return initTestingSettingsService(store.connection, dummySettingsObj)
 }
 
 func TestMigrateSettings(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, false, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, false, true)
 
 	err := setup(store)
 	if err != nil {
@@ -46,9 +38,11 @@ func TestMigrateSettings(t *testing.T) {
 	if updatedSettings.LogoURL != dummyLogoURL { // ensure a pre-migrate setting isn't unset
 		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
 	}
+
 	if updatedSettings.OAuthSettings.SSO != false { // I recon golang defaulting will make this false
 		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
 	}
+
 	if updatedSettings.OAuthSettings.LogoutURI != "" {
 		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
 	}
@@ -72,18 +66,23 @@ func TestMigrateSettings(t *testing.T) {
 		DockerhubService:        store.DockerHubService,
 		AuthorizationService:    authorization.NewService(store),
 	})
+
 	if err := m.MigrateSettingsToDB30(); err != nil {
 		t.Errorf("failed to update settings: %v", err)
 	}
+
 	if err != nil {
 		t.Errorf("failed to retrieve the updated settings: %v", err)
 	}
+
 	if updatedSettings.LogoURL != dummyLogoURL {
 		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
 	}
+
 	if updatedSettings.OAuthSettings.SSO != false {
 		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
 	}
+
 	if updatedSettings.OAuthSettings.LogoutURI != "" {
 		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
 	}

api/datastore/migrate_dbversion33_test.go

@@ -10,8 +10,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	_, store, teardown := MustNewTestStore(t, false, true)
-	defer teardown()
+	_, store := MustNewTestStore(t, false, true)
 
 	stackService := store.Stack()
 

api/datastore/migrate_post_init.go

@@ -6,22 +6,19 @@ import (
 	"github.com/docker/docker/api/types"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/docker"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/kubernetes/cli"
+
 	"github.com/rs/zerolog/log"
 )
 
 type PostInitMigrator struct {
 	kubeFactory   *cli.ClientFactory
-	dockerFactory *docker.ClientFactory
+	dockerFactory *dockerclient.ClientFactory
 	dataStore     dataservices.DataStore
 }
 
-func NewPostInitMigrator(
-	kubeFactory *cli.ClientFactory,
-	dockerFactory *docker.ClientFactory,
-	dataStore dataservices.DataStore,
-) *PostInitMigrator {
+func NewPostInitMigrator(kubeFactory *cli.ClientFactory, dockerFactory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *PostInitMigrator {
 	return &PostInitMigrator{
 		kubeFactory:   kubeFactory,
 		dockerFactory: dockerFactory,
@@ -44,9 +41,10 @@ func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
 	if err != nil {
 		return err
 	}
+
 	for i := range endpoints {
 		// Early exit if we do not need to migrate!
-		if endpoints[i].PostInitMigrations.MigrateIngresses == false {
+		if !endpoints[i].PostInitMigrations.MigrateIngresses {
 			return nil
 		}
 
@@ -67,10 +65,11 @@ func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
 		log.Err(err).Msg("failure getting endpoints")
 		return
 	}
+
 	for i := range environments {
 		if environments[i].Type == portainer.DockerEnvironment {
 			// // Early exit if we do not need to migrate!
-			if environments[i].PostInitMigrations.MigrateGPUs == false {
+			if !environments[i].PostInitMigrations.MigrateGPUs {
 				return
 			}
 
@@ -102,11 +101,13 @@ func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
 					log.Err(err).Msg("failed to inspect container")
 					return
 				}
+
 				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
 				for _, deviceRequest := range deviceRequests {
 					if deviceRequest.Driver == "nvidia" {
 						environments[i].EnableGPUManagement = true
 						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
+
 						break containersLoop
 					}
 				}

api/datastore/migrator/migrate_dbversion100.go

@@ -0,0 +1,29 @@
+package migrator
+
+import (
+	"os"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDockerDesktopExtentionSetting() error {
+	log.Info().Msg("updating docker desktop extention flag in settings")
+
+	isDDExtension := false
+	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {
+		isDDExtension = true
+	}
+
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	settings.IsDockerDesktopExtension = isDDExtension
+	err = m.settingsService.UpdateSettings(settings)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion26.go

@@ -2,7 +2,7 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/stacks/stackutils"
 
 	"github.com/rs/zerolog/log"
@@ -25,7 +25,7 @@ func (m *Migrator) updateStackResourceControlToDB27() error {
 
 		stack, err := m.stackService.StackByName(stackName)
 		if err != nil {
-			if err == errors.ErrObjectNotFound {
+			if dataservices.IsErrObjectNotFound(err) {
 				continue
 			}
 

api/datastore/migrator/migrate_dbversion31.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
 
@@ -86,7 +86,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 	log.Info().Msg("updating dockerhub")
 
 	dockerhub, err := m.dockerhubService.DockerHub()
-	if err == errors.ErrObjectNotFound {
+	if dataservices.IsErrObjectNotFound(err) {
 		return nil
 	} else if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion71.go

@@ -1,7 +1,7 @@
 package migrator
 
 import (
-	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
@@ -19,7 +19,7 @@ func (m *Migrator) migrateDBVersionToDB71() error {
 		if err == nil {
 			log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("keeping snapshot")
 			continue
-		} else if err != errors.ErrObjectNotFound {
+		} else if !dataservices.IsErrObjectNotFound(err) {
 			log.Debug().Int("endpoint_id", int(s.EndpointID)).Err(err).Msg("database error")
 			return err
 		}

api/datastore/migrator/migrate_dbversion90.go

@@ -4,7 +4,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	portainer "github.com/portainer/portainer/api"
-	portainerDsErrors "github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 func (m *Migrator) migrateDBVersionToDB90() error {
@@ -30,7 +30,7 @@ func (m *Migrator) updateEdgeStackStatusForDB90() error {
 	for _, edgeJob := range edgeJobs {
 		for endpointId := range edgeJob.Endpoints {
 			_, err := m.endpointService.Endpoint(endpointId)
-			if err == portainerDsErrors.ErrObjectNotFound {
+			if dataservices.IsErrObjectNotFound(err) {
 				delete(edgeJob.Endpoints, endpointId)
 
 				err = m.edgeJobService.UpdateEdgeJob(edgeJob.ID, &edgeJob)

api/datastore/migrator/migrator.go

@@ -211,6 +211,8 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.17", m.migrateDBVersionToDB80)
 	m.addMigrations("2.18", m.migrateDBVersionToDB90)
 
+	m.addMigrations("2.19", m.migrateDockerDesktopExtentionSetting)
+
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.
 }

api/datastore/services.go

@@ -104,7 +104,7 @@ func (store *Store) initServices() error {
 		return err
 	}
 	store.EdgeStackService = edgeStackService
-	endpointRelationService.RegisterUpdateStackFunction(edgeStackService.UpdateEdgeStackFunc)
+	endpointRelationService.RegisterUpdateStackFunction(edgeStackService.UpdateEdgeStackFunc, edgeStackService.UpdateEdgeStackFuncTx)
 
 	edgeGroupService, err := edgegroup.NewService(store.connection)
 	if err != nil {

api/datastore/services_tx.go

@@ -42,11 +42,24 @@ func (tx *StoreTx) EndpointRelation() dataservices.EndpointRelationService {
 
 func (tx *StoreTx) FDOProfile() dataservices.FDOProfileService                 { return nil }
 func (tx *StoreTx) HelmUserRepository() dataservices.HelmUserRepositoryService { return nil }
-func (tx *StoreTx) Registry() dataservices.RegistryService                     { return nil }
-func (tx *StoreTx) ResourceControl() dataservices.ResourceControlService       { return nil }
-func (tx *StoreTx) Role() dataservices.RoleService                             { return nil }
-func (tx *StoreTx) APIKeyRepository() dataservices.APIKeyRepository            { return nil }
-func (tx *StoreTx) Settings() dataservices.SettingsService                     { return nil }
+
+func (tx *StoreTx) Registry() dataservices.RegistryService {
+	return tx.store.RegistryService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) ResourceControl() dataservices.ResourceControlService {
+	return tx.store.ResourceControlService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) Role() dataservices.RoleService {
+	return tx.store.RoleService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) APIKeyRepository() dataservices.APIKeyRepository { return nil }
+
+func (tx *StoreTx) Settings() dataservices.SettingsService {
+	return tx.store.SettingsService.Tx(tx.tx)
+}
 
 func (tx *StoreTx) Snapshot() dataservices.SnapshotService {
 	return tx.store.SnapshotService.Tx(tx.tx)
@@ -59,9 +72,12 @@ func (tx *StoreTx) Tag() dataservices.TagService {
 	return tx.store.TagService.Tx(tx.tx)
 }
 
-func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService { return nil }
-func (tx *StoreTx) Team() dataservices.TeamService                     { return nil }
-func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService     { return nil }
-func (tx *StoreTx) User() dataservices.UserService                     { return nil }
-func (tx *StoreTx) Version() dataservices.VersionService               { return nil }
-func (tx *StoreTx) Webhook() dataservices.WebhookService               { return nil }
+func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
+	return tx.store.TeamMembershipService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) Team() dataservices.TeamService                 { return nil }
+func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }
+func (tx *StoreTx) User() dataservices.UserService                 { return nil }
+func (tx *StoreTx) Version() dataservices.VersionService           { return nil }
+func (tx *StoreTx) Webhook() dataservices.WebhookService           { return nil }

api/datastore/test_data/output_24_to_latest.json

@@ -606,6 +606,7 @@
     "InternalAuthSettings": {
       "RequiredPasswordLength": 12
     },
+    "IsDockerDesktopExtension": false,
     "KubeconfigExpiry": "0",
     "KubectlShellImage": "portainer/kubectl-shell",
     "LDAPSettings": {
@@ -713,8 +714,6 @@
               "ID": ""
             },
             "Isolation": "",
-            "KernelMemory": false,
-            "KernelMemoryTCP": false,
             "KernelVersion": "",
             "Labels": null,
             "LiveRestoreEnabled": false,
@@ -945,6 +944,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.18.2\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.19.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/datastore/teststore.go

@@ -15,13 +15,15 @@ func (store *Store) GetConnection() portainer.Connection {
 	return store.connection
 }
 
-func MustNewTestStore(t testing.TB, init, secure bool) (bool, *Store, func()) {
+func MustNewTestStore(t testing.TB, init, secure bool) (bool, *Store) {
 	newStore, store, teardown, err := NewTestStore(t, init, secure)
 	if err != nil {
 		log.Fatal().Err(err).Msg("")
 	}
 
-	return newStore, store, teardown
+	t.Cleanup(teardown)
+
+	return newStore, store
 }
 
 func NewTestStore(t testing.TB, init, secure bool) (bool, *Store, func(), error) {

api/docker/client/client.go

@@ -1,4 +1,4 @@
-package docker
+package client
 
 import (
 	"errors"
@@ -38,17 +38,19 @@ func NewClientFactory(signatureService portainer.DigitalSignatureService, revers
 // with an agent enabled environment(endpoint) to target a specific node in an agent cluster.
 // The underlying http client timeout may be specified, a default value is used otherwise.
 func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string, timeout *time.Duration) (*client.Client, error) {
-	if endpoint.Type == portainer.AzureEnvironment {
+	switch endpoint.Type {
+	case portainer.AzureEnvironment:
 		return nil, errUnsupportedEnvironmentType
-	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
+	case portainer.AgentOnDockerEnvironment:
 		return createAgentClient(endpoint, factory.signatureService, nodeName, timeout)
-	} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+	case portainer.EdgeAgentOnDockerEnvironment:
 		return createEdgeClient(endpoint, factory.signatureService, factory.reverseTunnelService, nodeName, timeout)
 	}
 
 	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
 		return createLocalClient(endpoint)
 	}
+
 	return createTCPClient(endpoint, timeout)
 }
 

api/docker/consts/labels.go

@@ -0,0 +1,8 @@
+package consts
+
+const (
+	ComposeStackNameLabel = "com.docker.compose.project"
+	SwarmStackNameLabel   = "com.docker.stack.namespace"
+	SwarmServiceIdLabel   = "com.docker.swarm.service.id"
+	SwarmNodeIdLabel      = "com.docker.swarm.node.id"
+)

api/docker/container.go

@@ -0,0 +1,232 @@
+package docker
+
+import (
+	"context"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	dockercontainer "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/docker/images"
+	"github.com/rs/zerolog/log"
+)
+
+type ContainerService struct {
+	factory   *dockerclient.ClientFactory
+	dataStore dataservices.DataStore
+	sr        *serviceRestore
+}
+
+func NewContainerService(factory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *ContainerService {
+	return &ContainerService{
+		factory:   factory,
+		dataStore: dataStore,
+		sr:        &serviceRestore{},
+	}
+}
+
+// Recreate a container
+func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerId string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error) {
+	cli, err := c.factory.CreateClient(endpoint, nodeName, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "create client error")
+	}
+
+	defer func(cli *client.Client) {
+		cli.Close()
+	}(cli)
+
+	log.Debug().Str("container_id", containerId).Msg("starting to fetch container information")
+
+	container, _, err := cli.ContainerInspectWithRaw(ctx, containerId, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch container information error")
+	}
+
+	log.Debug().Str("image", container.Config.Image).Msg("starting to parse image")
+	img, err := images.ParseImage(images.ParseImageOptions{
+		Name: container.Config.Image,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "parse image error")
+	}
+
+	if imageTag != "" {
+		err = img.WithTag(imageTag)
+		if err != nil {
+			return nil, errors.Wrapf(err, "set image tag error %s", imageTag)
+		}
+
+		log.Debug().Str("image", container.Config.Image).Msg("new image with tag")
+
+		container.Config.Image = img.FullName()
+	}
+
+	// 1. pull image if you need force pull
+	if forcePullImage {
+		puller := images.NewPuller(cli, images.NewRegistryClient(c.dataStore), c.dataStore)
+		err = puller.Pull(ctx, img)
+		if err != nil {
+			return nil, errors.Wrapf(err, "pull image error %s", img.FullName())
+		}
+	}
+
+	// 2. stop the current container
+	log.Debug().Str("container_id", containerId).Msg("starting to stop the container")
+	err = cli.ContainerStop(ctx, containerId, dockercontainer.StopOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "stop container error")
+	}
+
+	// 3. rename the current container
+	log.Debug().Str("container_id", containerId).Msg("starting to rename the container")
+	err = cli.ContainerRename(ctx, containerId, container.Name+"-old")
+	if err != nil {
+		return nil, errors.Wrap(err, "rename container error")
+	}
+
+	networkWithCreation := network.NetworkingConfig{
+		EndpointsConfig: make(map[string]*network.EndpointSettings),
+	}
+
+	// 4. disconnect all networks from the current container
+	for name, network := range container.NetworkSettings.Networks {
+		// This allows new container to use the same IP address if specified
+		err = cli.NetworkDisconnect(ctx, network.NetworkID, containerId, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "disconnect network from old container error")
+		}
+
+		// 5. get the first network attached to the current container
+		if len(networkWithCreation.EndpointsConfig) == 0 {
+			// Retrieve the first network that is linked to the present container, which
+			// will be utilized when creating the container.
+			networkWithCreation.EndpointsConfig[name] = network
+		}
+	}
+	c.sr.enable()
+	defer c.sr.close()
+	defer c.sr.restore()
+
+	c.sr.push(func() {
+		log.Debug().Str("container_id", containerId).Str("container", container.Name).Msg("restoring the container")
+		cli.ContainerRename(ctx, containerId, container.Name)
+		for _, network := range container.NetworkSettings.Networks {
+			cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
+		}
+		cli.ContainerStart(ctx, containerId, types.ContainerStartOptions{})
+	})
+
+	log.Debug().Str("container", strings.Split(container.Name, "/")[1]).Msg("starting to create a new container")
+
+	// 6. create a new container
+	// when a container is created without a network, docker connected it by default to the
+	// bridge network with a random IP, also it can only connect to one network on creation.
+	// to retain the same network settings we have to connect on creation to one of the old
+	// container's networks, and connect to the other networks after creation.
+	// see: https://portainer.atlassian.net/browse/EE-5448
+	create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &networkWithCreation, nil, container.Name)
+
+	c.sr.push(func() {
+		log.Debug().Str("container_id", create.ID).Msg("removing the new container")
+		cli.ContainerStop(ctx, create.ID, dockercontainer.StopOptions{})
+		cli.ContainerRemove(ctx, create.ID, types.ContainerRemoveOptions{})
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "create container error")
+	}
+
+	newContainerId := create.ID
+
+	// 7. connect to networks
+	// docker can connect to only one network at creation, so we need to connect to networks after creation
+	// see https://github.com/moby/moby/issues/17750
+	log.Debug().Str("container_id", newContainerId).Msg("connecting networks to container")
+	networks := container.NetworkSettings.Networks
+	for key, network := range networks {
+		_, ok := networkWithCreation.EndpointsConfig[key]
+		if ok {
+			// skip the network that is used during container creation
+			continue
+		}
+
+		err = cli.NetworkConnect(ctx, network.NetworkID, newContainerId, network)
+		if err != nil {
+			return nil, errors.Wrap(err, "connect container network error")
+		}
+	}
+
+	// 8. start the new container
+	log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
+	err = cli.ContainerStart(ctx, newContainerId, types.ContainerStartOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "start container error")
+	}
+
+	// 9. delete the old container
+	log.Debug().Str("container_id", containerId).Msg("starting to remove the old container")
+	_ = cli.ContainerRemove(ctx, containerId, types.ContainerRemoveOptions{})
+
+	c.sr.disable()
+
+	newContainer, _, err := cli.ContainerInspectWithRaw(ctx, newContainerId, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch container information error")
+	}
+
+	return &newContainer, nil
+}
+
+type serviceRestore struct {
+	restoreC chan struct{}
+	fs       []func()
+}
+
+func (sr *serviceRestore) enable() {
+	sr.restoreC = make(chan struct{}, 1)
+	sr.fs = make([]func(), 0)
+	sr.restoreC <- struct{}{}
+}
+
+func (sr *serviceRestore) disable() {
+	select {
+	case <-sr.restoreC:
+	default:
+	}
+}
+
+func (sr *serviceRestore) push(f func()) {
+	sr.fs = append(sr.fs, f)
+}
+
+func (sr *serviceRestore) restore() {
+	select {
+	case <-sr.restoreC:
+		l := len(sr.fs)
+		if l > 0 {
+			for i := l - 1; i >= 0; i-- {
+				sr.fs[i]()
+			}
+		}
+	default:
+	}
+}
+
+func (sr *serviceRestore) close() {
+	if sr == nil || sr.restoreC == nil {
+		return
+	}
+
+	select {
+	case <-sr.restoreC:
+	default:
+	}
+
+	close(sr.restoreC)
+}

api/docker/images/digest.go

@@ -0,0 +1,184 @@
+package images
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+
+	"github.com/containers/image/v5/docker"
+	imagetypes "github.com/containers/image/v5/types"
+	"github.com/docker/docker/api/types"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+// Options holds docker registry object options
+type Options struct {
+	Auth    imagetypes.DockerAuthConfig
+	Timeout time.Duration
+}
+
+type DigestClient struct {
+	clientFactory  *dockerclient.ClientFactory
+	opts           Options
+	sysCtx         *imagetypes.SystemContext
+	registryClient *RegistryClient
+}
+
+func NewClientWithRegistry(registryClient *RegistryClient, clientFactory *dockerclient.ClientFactory) *DigestClient {
+	return &DigestClient{
+		clientFactory:  clientFactory,
+		registryClient: registryClient,
+	}
+}
+
+func (c *DigestClient) RemoteDigest(image Image) (digest.Digest, error) {
+	ctx, cancel := c.timeoutContext()
+	defer cancel()
+	// Docker references with both a tag and digest are currently not supported
+	if image.Tag != "" && image.Digest != "" {
+		err := image.trimDigest()
+		if err != nil {
+			return "", err
+		}
+	}
+
+	rmRef, err := ParseReference(image.String())
+	if err != nil {
+		return "", errors.Wrap(err, "Cannot parse the image reference")
+	}
+
+	sysCtx := c.sysCtx
+	if c.registryClient != nil {
+		username, password, err := c.registryClient.RegistryAuth(image)
+		if err != nil {
+			log.Info().Str("image up to date indicator", image.String()).Msg("No environment registry credentials found, using anonymous access")
+		} else {
+			sysCtx = &imagetypes.SystemContext{
+				DockerAuthConfig: &imagetypes.DockerAuthConfig{
+					Username: username,
+					Password: password,
+				},
+			}
+		}
+	}
+
+	// Retrieve remote digest through HEAD request
+	rmDigest, err := docker.GetDigest(ctx, sysCtx, rmRef)
+	if err != nil {
+		// fallback to public registry for hub
+		if image.HubLink != "" {
+			rmDigest, err = docker.GetDigest(ctx, c.sysCtx, rmRef)
+			if err == nil {
+				return rmDigest, nil
+			}
+		}
+
+		log.Debug().Err(err).Msg("get remote digest error")
+
+		return "", errors.Wrap(err, "Cannot get image digest from HEAD request")
+	}
+
+	return rmDigest, nil
+}
+
+func ParseLocalImage(inspect types.ImageInspect) (*Image, error) {
+	if IsLocalImage(inspect) || IsDanglingImage(inspect) {
+		return nil, errors.New("the image is not regular")
+	}
+
+	fromRepoDigests, err := ParseImage(ParseImageOptions{
+		// including image name but no tag
+		Name: inspect.RepoDigests[0],
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if IsNoTagImage(inspect) {
+		return &fromRepoDigests, nil
+	}
+
+	fromRepoTags, err := ParseImage(ParseImageOptions{
+		Name: inspect.RepoTags[0],
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	fromRepoDigests.Tag = fromRepoTags.Tag
+
+	return &fromRepoDigests, nil
+}
+
+func ParseRepoDigests(repoDigests []string) []digest.Digest {
+	digests := make([]digest.Digest, 0)
+	for _, repoDigest := range repoDigests {
+		d := ParseRepoDigest(repoDigest)
+		if d == "" {
+			continue
+		}
+
+		digests = append(digests, d)
+	}
+
+	return digests
+}
+
+func ParseRepoTags(repoTags []string) []*Image {
+	images := make([]*Image, 0)
+	for _, repoTag := range repoTags {
+		image := ParseRepoTag(repoTag)
+		if image != nil {
+			images = append(images, image)
+		}
+	}
+
+	return images
+}
+
+func ParseRepoDigest(repoDigest string) digest.Digest {
+	if !strings.ContainsAny(repoDigest, "@") {
+		return ""
+	}
+
+	d, err := digest.Parse(strings.Split(repoDigest, "@")[1])
+	if err != nil {
+		log.Warn().Msgf("Skip invalid repo digest item: %s [error: %v]", repoDigest, err)
+
+		return ""
+	}
+
+	return d
+}
+
+func ParseRepoTag(repoTag string) *Image {
+	if repoTag == "" {
+		return nil
+	}
+
+	image, err := ParseImage(ParseImageOptions{
+		Name: repoTag,
+	})
+	if err != nil {
+		log.Warn().Err(err).Str("repoTag", repoTag).Msg("RepoTag cannot be parsed.")
+
+		return nil
+	}
+
+	return &image
+}
+
+func (c *DigestClient) timeoutContext() (context.Context, context.CancelFunc) {
+	ctx := context.Background()
+	var cancel context.CancelFunc = func() {}
+
+	if c.opts.Timeout > 0 {
+		ctx, cancel = context.WithTimeout(ctx, c.opts.Timeout)
+	}
+
+	return ctx, cancel
+}

api/docker/images/image.go

@@ -0,0 +1,182 @@
+package images
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/docker/docker/api/types"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+type ImageID string
+
+// Image holds information about an image.
+type Image struct {
+	// Domain is the registry host of this image
+	Domain string
+	// Path may include username like portainer/portainer-ee, no Tag or Digest
+	Path    string
+	Tag     string
+	Digest  digest.Digest
+	HubLink string
+	named   reference.Named
+	opts    ParseImageOptions
+}
+
+// ParseImageOptions holds image options for parsing.
+type ParseImageOptions struct {
+	Name   string
+	HubTpl string
+}
+
+// Name returns the full name representation of an image but no Tag or Digest.
+func (i *Image) Name() string {
+	return i.named.Name()
+}
+
+// FullName return the real full name may include Tag or Digest of the image, Tag first.
+func (i *Image) FullName() string {
+	if i.Tag == "" {
+		return fmt.Sprintf("%s@%s", i.Name(), i.Digest)
+	}
+	return fmt.Sprintf("%s:%s", i.Name(), i.Tag)
+}
+
+// String returns the string representation of an image, including Tag and Digest if existed.
+func (i *Image) String() string {
+	return i.named.String()
+}
+
+// Reference returns either the digest if it is non-empty or the tag for the image.
+func (i *Image) Reference() string {
+	if len(i.Digest.String()) > 1 {
+		return i.Digest.String()
+	}
+
+	return i.Tag
+}
+
+// WithDigest sets the digest for an image.
+func (i *Image) WithDigest(digest digest.Digest) (err error) {
+	i.Digest = digest
+	i.named, err = reference.WithDigest(i.named, digest)
+	return err
+}
+
+func (i *Image) WithTag(tag string) (err error) {
+	i.Tag = tag
+	i.named, err = reference.WithTag(i.named, tag)
+	return err
+}
+
+func (i *Image) trimDigest() error {
+	i.Digest = ""
+	named, err := ParseImage(ParseImageOptions{Name: i.FullName()})
+	if err != nil {
+		return err
+	}
+	i.named = &named
+	return nil
+}
+
+// ParseImage returns an Image struct with all the values filled in for a given image.
+func ParseImage(parseOpts ParseImageOptions) (Image, error) {
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(parseOpts.Name)
+	if err != nil {
+		return Image{}, errors.Wrapf(err, "parsing image %s failed", parseOpts.Name)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+
+	i := Image{
+		opts:   parseOpts,
+		named:  named,
+		Domain: reference.Domain(named),
+		Path:   reference.Path(named),
+	}
+
+	// Hub link
+	i.HubLink, err = i.hubLink()
+	if err != nil {
+		return Image{}, errors.Wrap(err, fmt.Sprintf("resolving hub link for image %s failed", parseOpts.Name))
+	}
+
+	// Add the tag if there was one.
+	if tagged, ok := named.(reference.Tagged); ok {
+		i.Tag = tagged.Tag()
+	}
+
+	// Add the digest if there was one.
+	if canonical, ok := named.(reference.Canonical); ok {
+		i.Digest = canonical.Digest()
+	}
+
+	return i, nil
+}
+
+func (i *Image) hubLink() (string, error) {
+	if i.opts.HubTpl != "" {
+		var out bytes.Buffer
+		tmpl, err := template.New("tmpl").
+			Option("missingkey=error").
+			Parse(i.opts.HubTpl)
+		if err != nil {
+			return "", err
+		}
+		err = tmpl.Execute(&out, i)
+		return out.String(), err
+	}
+
+	switch i.Domain {
+	case "docker.io":
+		prefix := "r"
+		path := i.Path
+		if strings.HasPrefix(i.Path, "library/") {
+			prefix = "_"
+			path = strings.Replace(i.Path, "library/", "", 1)
+		}
+		return fmt.Sprintf("https://hub.docker.com/%s/%s", prefix, path), nil
+	case "docker.bintray.io", "jfrog-docker-reg2.bintray.io":
+		return fmt.Sprintf("https://bintray.com/jfrog/reg2/%s", strings.ReplaceAll(i.Path, "/", "%3A")), nil
+	case "docker.pkg.github.com":
+		return fmt.Sprintf("https://github.com/%s/packages", filepath.ToSlash(filepath.Dir(i.Path))), nil
+	case "gcr.io":
+		return fmt.Sprintf("https://%s/%s", i.Domain, i.Path), nil
+	case "ghcr.io":
+		ref := strings.Split(i.Path, "/")
+		ghUser, ghPackage := ref[0], ref[1]
+		return fmt.Sprintf("https://github.com/users/%s/packages/container/package/%s", ghUser, ghPackage), nil
+	case "quay.io":
+		return fmt.Sprintf("https://quay.io/repository/%s", i.Path), nil
+	case "registry.access.redhat.com":
+		return fmt.Sprintf("https://access.redhat.com/containers/#/registry.access.redhat.com/%s", i.Path), nil
+	case "registry.gitlab.com":
+		return fmt.Sprintf("https://gitlab.com/%s/container_registry", i.Path), nil
+	default:
+		return "", nil
+	}
+}
+
+// IsLocalImage checks if the image has been built locally
+func IsLocalImage(image types.ImageInspect) bool {
+	return len(image.RepoDigests) == 0
+}
+
+// IsDanglingImage returns whether the given image is "dangling" which means
+// that there are no repository references to the given image and it has no
+// child images
+func IsDanglingImage(image types.ImageInspect) bool {
+	return len(image.RepoTags) == 1 && image.RepoTags[0] == "<none>:<none>" && len(image.RepoDigests) == 1 && image.RepoDigests[0] == "<none>@<none>"
+}
+
+// IsNoTagImage returns whether the given image is damaged, has no tags
+func IsNoTagImage(image types.ImageInspect) bool {
+	return len(image.RepoTags) == 0
+}

api/docker/images/image_test.go

@@ -0,0 +1,119 @@
+package images
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestImageParser(t *testing.T) {
+	is := assert.New(t)
+
+	// portainer/portainer-ee
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "portainer/portainer-ee",
+		})
+		is.NoError(err, "")
+		is.Equal("docker.io/portainer/portainer-ee:latest", image.FullName())
+		is.Equal("portainer/portainer-ee", image.opts.Name)
+		is.Equal("latest", image.Tag)
+		is.Equal("portainer/portainer-ee", image.Path)
+		is.Equal("docker.io", image.Domain)
+		is.Equal("docker.io/portainer/portainer-ee", image.Name())
+		is.Equal("latest", image.Reference())
+		is.Equal("docker.io/portainer/portainer-ee:latest", image.String())
+
+	})
+	// gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+}
+
+func TestUpdateParsedImage(t *testing.T) {
+	is := assert.New(t)
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.WithTag("v0.0.31")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.31", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.31", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.31@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.WithDigest("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.trimDigest()
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("v0.0.30", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.String())
+	})
+}

api/docker/images/puller.go

@@ -0,0 +1,50 @@
+package images
+
+import (
+	"context"
+	"io"
+
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/rs/zerolog/log"
+)
+
+type Puller struct {
+	client         *client.Client
+	registryClient *RegistryClient
+	dataStore      dataservices.DataStore
+}
+
+func NewPuller(client *client.Client, registryClient *RegistryClient, dataStore dataservices.DataStore) *Puller {
+	return &Puller{
+		client:         client,
+		registryClient: registryClient,
+		dataStore:      dataStore,
+	}
+}
+
+func (puller *Puller) Pull(ctx context.Context, image Image) error {
+	log.Debug().Str("image", image.FullName()).Msg("starting to pull the image")
+
+	registryAuth, err := puller.registryClient.EncodedRegistryAuth(image)
+	if err != nil {
+		log.Debug().
+			Str("image", image.FullName()).
+			Err(err).
+			Msg("failed to get an encoded registry auth via image, try to pull image without registry auth")
+	}
+
+	out, err := puller.client.ImagePull(ctx, image.FullName(), types.ImagePullOptions{
+		RegistryAuth: registryAuth,
+	})
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.ReadAll(out)
+
+	return err
+}

api/docker/images/ref.go

@@ -0,0 +1,16 @@
+package images
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/types"
+)
+
+func ParseReference(imageStr string) (types.ImageReference, error) {
+	if !strings.HasPrefix(imageStr, "//") {
+		imageStr = fmt.Sprintf("//%s", imageStr)
+	}
+	return docker.ParseReference(imageStr)
+}

api/docker/images/registry.go

@@ -0,0 +1,147 @@
+package images
+
+import (
+	"strings"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/registryutils"
+)
+
+var (
+	_registriesCache = cache.New(5*time.Minute, 5*time.Minute)
+)
+
+type (
+	RegistryClient struct {
+		dataStore dataservices.DataStore
+	}
+)
+
+func NewRegistryClient(dataStore dataservices.DataStore) *RegistryClient {
+	return &RegistryClient{dataStore: dataStore}
+}
+
+func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
+	registries, err := c.dataStore.Registry().Registries()
+	if err != nil {
+		return "", "", err
+	}
+
+	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	if err != nil {
+		return "", "", err
+	}
+
+	if !registry.Authentication {
+		return "", "", errors.New("authentication is disabled")
+	}
+
+	return c.CertainRegistryAuth(registry)
+}
+
+func (c *RegistryClient) CertainRegistryAuth(registry *portainer.Registry) (string, string, error) {
+	err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
+	if err != nil {
+		return "", "", err
+	}
+
+	if !registry.Authentication {
+		return "", "", errors.New("authentication is disabled")
+	}
+
+	return registryutils.GetRegEffectiveCredential(registry)
+}
+
+func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
+	registries, err := c.dataStore.Registry().Registries()
+	if err != nil {
+		return "", err
+	}
+
+	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	if err != nil {
+		return "", err
+	}
+
+	if !registry.Authentication {
+		return "", errors.New("authentication is disabled")
+	}
+
+	return c.EncodedCertainRegistryAuth(registry)
+}
+
+func (c *RegistryClient) EncodedCertainRegistryAuth(registry *portainer.Registry) (string, error) {
+	err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
+	if err != nil {
+		return "", err
+	}
+
+	return registryutils.GetRegistryAuthHeader(registry)
+}
+
+// findBestMatchRegistry finds out the best match registry for repository @Meng
+// matching precedence:
+// 1. both domain name and username matched (for dockerhub only)
+// 2. only URL matched
+// 3. pick up the first dockerhub registry
+func findBestMatchRegistry(repository string, registries []portainer.Registry) (*portainer.Registry, error) {
+	cachedRegistry, err := cachedRegistry(repository)
+	if err == nil {
+		return cachedRegistry, nil
+	}
+
+	var match1, match2, match3 *portainer.Registry
+	for i := 0; i < len(registries); i++ {
+		registry := registries[i]
+		if registry.Type == portainer.DockerHubRegistry {
+
+			// try to match repository examples:
+			//   <USERNAME>/nginx:latest
+			//   docker.io/<USERNAME>/nginx:latest
+			if strings.HasPrefix(repository, registry.Username+"/") || strings.HasPrefix(repository, registry.URL+"/"+registry.Username+"/") {
+				match1 = &registry
+			}
+
+			// try to match repository examples:
+			//   portainer/portainer-ee:latest
+			//   <NON-USERNAME>/portainer-ee:latest
+			if match3 == nil {
+				match3 = &registry
+			}
+		}
+
+		if strings.Contains(repository, registry.URL) {
+			match2 = &registry
+		}
+	}
+
+	match := match1
+	if match == nil {
+		match = match2
+	}
+	if match == nil {
+		match = match3
+	}
+
+	if match == nil {
+		return nil, errors.New("no registries matched")
+	}
+	_registriesCache.Set(repository, match, 0)
+	return match, nil
+}
+
+func cachedRegistry(cacheKey string) (*portainer.Registry, error) {
+	r, ok := _registriesCache.Get(cacheKey)
+	if ok {
+		registry, ok := r.(portainer.Registry)
+		if ok {
+			return &registry, nil
+		}
+	}
+
+	return nil, errors.Errorf("no registry found in cache: %s", cacheKey)
+}