fix(app): show services explanation in app form [EE-7284]

Co-authored-by: testa113 <testa113>

.eslintrc.yml

@@ -10,6 +10,7 @@ globals:
 extends:
   - 'eslint:recommended'
   - 'plugin:storybook/recommended'
+  - 'plugin:import/typescript'
   - prettier
 
 plugins:
@@ -23,10 +24,13 @@ parserOptions:
     modules: true
 
 rules:
+  no-console: error
+  no-alert: error
   no-control-regex: 'off'
   no-empty: warn
   no-empty-function: warn
   no-useless-escape: 'off'
+  import/named: error
   import/order:
     [
       'error',
@@ -41,6 +45,12 @@ rules:
         pathGroupsExcludedImportTypes: ['internal'],
       },
     ]
+  no-restricted-imports:
+    - error
+    - patterns:
+        - group:
+            - '@/react/test-utils/*'
+          message: 'These utils are just for test files'
 
 settings:
   'import/resolver':
@@ -49,6 +59,8 @@ settings:
         - ['@@', './app/react/components']
         - ['@', './app']
       extensions: ['.js', '.ts', '.tsx']
+    typescript: true
+    node: true
 
 overrides:
   - files:
@@ -73,7 +85,9 @@ overrides:
     settings:
       react:
         version: 'detect'
+
     rules:
+      no-console: error
       import/order:
         [
           'error',
@@ -86,8 +100,8 @@ overrides:
       no-plusplus: off
       func-style: [error, 'declaration']
       import/prefer-default-export: off
-      no-use-before-define: "off"
-      '@typescript-eslint/no-use-before-define': ['error', { functions: false, "allowNamedExports": true }]
+      no-use-before-define: 'off'
+      '@typescript-eslint/no-use-before-define': ['error', { functions: false, 'allowNamedExports': true }]
       no-shadow: 'off'
       '@typescript-eslint/no-shadow': off
       jsx-a11y/no-autofocus: warn
@@ -106,6 +120,12 @@ overrides:
       'no-await-in-loop': 'off'
       'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
       'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
+      '@typescript-eslint/no-restricted-imports':
+        - error
+        - patterns:
+            - group:
+                - '@/react/test-utils/*'
+              message: 'These utils are just for test files'
     overrides: # allow props spreading for hoc files
       - files:
           - app/**/with*.ts{,x}
@@ -114,13 +134,18 @@ overrides:
   - files:
       - app/**/*.test.*
     extends:
-      - 'plugin:jest/recommended'
-      - 'plugin:jest/style'
+      - 'plugin:vitest/recommended'
     env:
-      'jest/globals': true
+      'vitest/env': true
     rules:
       'react/jsx-no-constructed-context-values': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
+      'react/jsx-props-no-spreading': off
   - files:
       - app/**/*.stories.*
     rules:
       'no-alert': off
+      '@typescript-eslint/no-restricted-imports': off
+      no-restricted-imports: off
+      'react/jsx-props-no-spreading': off

.github/DISCUSSION_TEMPLATE/help.yaml

@@ -0,0 +1,11 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before asking a question, make sure it hasn't been already asked and answered. You can search our [discussions](https://github.com/orgs/portainer/discussions) and [bug reports](https://github.com/portainer/portainer/issues) in GitHub.  Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io/) first.
+
+  - type: textarea
+    attributes:
+      label: Ask a Question!
+    validations:
+      required: true

.github/DISCUSSION_TEMPLATE/ideas.yaml

@@ -0,0 +1,38 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        # Welcome!
+        
+        Thanks for suggesting an idea for Portainer!
+
+        Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion cagetory](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
+
+        Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io) as they may point you toward a solution.
+        
+        **DO NOT FILE DUPLICATE REQUESTS.**
+
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      description: Short list of what the feature request aims to address.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,54 +0,0 @@
----
-name: Bug report
-about: Create a bug report
-title: ''
-labels: bug/need-confirmation, kind/bug
-assignees: ''
----
-
-<!--
-
-Thanks for reporting a bug for Portainer !
-
-You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
-
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
-
-Before opening a new issue, make sure that we do not have any duplicates
-already open. You can ensure this by searching the issue list for this
-repository. If there is a duplicate, please close your issue and add a comment
-to the existing issue instead.
-
-Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
--->
-
-**Bug description**
-A clear and concise description of what the bug is.
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Portainer Logs**
-Provide the logs of your Portainer container or Service.
-You can see how [here](https://documentation.portainer.io/r/portainer-logs)
-
-**Steps to reproduce the issue:**
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Technical details:**
-
-- Portainer version:
-- Docker version (managed by Portainer):
-- Kubernetes version (managed by Portainer):
-- Platform (windows/linux):
-- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
-- Browser:
-- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
-- Have you reviewed our technical documentation and knowledge base? Yes/No
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,172 @@
+name: Bug Report
+description: Create a report to help us improve.
+labels: kind/bug,bug/need-confirmation
+body:
+
+  - type: markdown
+    attributes:
+      value: |
+        # Welcome!
+        
+        The issue tracker is for reporting bugs. If you have an [idea for a new feature](https://github.com/orgs/portainer/discussions/categories/ideas) or a [general question about Portainer](https://github.com/orgs/portainer/discussions/categories/help) please post in our [GitHub Discussions](https://github.com/orgs/portainer/discussions).
+        
+        You can also ask for help in our [community Slack channel](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA).
+        
+        **DO NOT FILE ISSUES FOR GENERAL SUPPORT QUESTIONS**.
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Before you start please confirm the following.
+      options:
+        - label: Yes, I've searched similar issues on [GitHub](https://github.com/portainer/portainer/issues).
+          required: true
+        - label: Yes, I've checked whether this issue is covered in the Portainer [documentation](https://docs.portainer.io) or [knowledge base](https://portal.portainer.io/knowledge).
+          required: true
+
+  - type: markdown
+    attributes:
+      value: |
+        # About your issue
+
+        Tell us a bit about the issue you're having.
+
+        How to write a good bug report:
+
+        - Respect the issue template as much as possible.
+        - Summarize the issue so that we understand what is going wrong.
+        - Describe what you would have expected to have happened, and what actually happened instead.
+        - Provide easy to follow steps to reproduce the issue. 
+        - Remain clear and concise.
+        - Format your messages to help the reader focus on what matters and understand the structure of your message, use [Markdown syntax](https://help.github.com/articles/github-flavored-markdown).
+
+  - type: textarea
+    attributes:
+      label: Problem Description
+      description: A clear and concise description of what the bug is. 
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of what you expected to happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Actual Behavior
+      description: A clear and concise description of what actually happens.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Steps to Reproduce
+      description: Please be as detailed as possible when providing steps to reproduce.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. Scroll down to '....'
+        4. See error   
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Portainer logs or screenshots
+      description: Provide Portainer container logs or any screenshots related to the issue.
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        # About your environment
+
+        Tell us a bit about your Portainer environment.
+
+  - type: dropdown
+    attributes:
+      label: Portainer version
+      description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
+      multiple: false
+      options:
+        - '2.20.3'
+        - '2.20.2'
+        - '2.20.1'
+        - '2.20.0'
+        - '2.19.5'
+        - '2.19.4'
+        - '2.19.3'
+        - '2.19.2'
+        - '2.19.1'
+        - '2.19.0'
+        - '2.18.4'
+        - '2.18.3'
+        - '2.18.2'
+        - '2.18.1'
+        - '2.17.1'
+        - '2.17.0'
+        - '2.16.2'
+        - '2.16.1'
+        - '2.16.0'
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Portainer Edition
+      multiple: false
+      options:
+        - 'Business Edition (BE/EE) with 5NF / 3NF license'
+        - 'Business Edition (BE/EE) with Home & Student license'
+        - 'Business Edition (BE/EE) with Starter license'
+        - 'Business Edition (BE/EE) with Professional or Enterprise license'
+        - 'Community Edition (CE)'
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Platform and Version
+      description: |
+        Enter your container management platform (Docker | Swarm | Kubernetes) along with the version. 
+        Example: Docker 24.0.3 | Docker Swarm 24.0.3 | Kubernetes 1.26
+        You can find our supported platforms [in our documentation](https://docs.portainer.io/start/requirements-and-prerequisites).
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: OS and Architecture
+      description: |
+        Enter your Operating System, Version and Architecture. Example: Ubuntu 22.04, AMD64 | Raspbian OS, ARM64
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Browser
+      description: | 
+        Enter your browser and version. Example: Google Chrome 114.0
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: What command did you use to deploy Portainer?
+      description: |
+        Example: `docker run -d -p 8000:8000 -p 9443:9443 --name portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce:latest`
+        If you deployed Portainer using a compose file or manifest you can provide this here as well.
+      render: bash
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: Additional Information
+      description: Any additional information about your environment, the bug, or anything else you think might be helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,11 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Portainer Business Edition - Get 5 nodes free
-    url: https://portainer.io/pricing/take5 
-    about: Portainer Business Edition has more features, more support and you can now get 5 nodes free for as long as you want.
+  - name: Question
+    url: https://github.com/orgs/portainer/discussions/new?category=help
+    about: Ask us a question about Portainer usage or deployment.
+  - name: Idea or Feature Request
+    url: https://github.com/orgs/portainer/discussions/new?category=ideas
+    about: Suggest an idea or feature/enhancement that should be added in Portainer.
+  - name: Portainer Business Edition - Get 3 Nodes Free
+    url: https://www.portainer.io/take-3
+    about: Portainer Business Edition has more features, more support and you can now get 3 nodes free for as long as you want.

.github/workflows/ci.yaml

@@ -0,0 +1,176 @@
+name: ci
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'develop'
+      - 'release/*'
+  pull_request:
+    branches:
+      - 'develop'
+      - 'release/*'
+      - 'feat/*'
+      - 'fix/*'
+      - 'refactor/*'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  DOCKER_HUB_REPO: portainerci/portainer-ce
+  EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
+  GO_VERSION: 1.21.9
+  NODE_VERSION: 18.x
+
+jobs:
+  build_images:
+    strategy:
+      matrix:
+        config:
+          - { platform: linux, arch: amd64, version: "" }
+          - { platform: linux, arch: arm64, version: "" }
+          - { platform: linux, arch: arm, version: "" }
+          - { platform: linux, arch: ppc64le, version: "" }
+          - { platform: linux, arch: s390x, version: "" }
+          - { platform: windows, arch: amd64, version: 1809 }
+          - { platform: windows, arch: amd64, version: ltsc2022 }
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+      - name: '[preparation] checkout the current branch'
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: ${{ github.event.inputs.branch }}
+      - name: '[preparation] set up golang'
+        uses: actions/setup-go@v5.0.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: '[preparation] set up node.js'
+        uses: actions/setup-node@v4.0.1
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+      - name: '[preparation] set up qemu'
+        uses: docker/setup-qemu-action@v3.0.0
+      - name: '[preparation] set up docker context for buildx'
+        run: docker context create builders
+      - name: '[preparation] set up docker buildx'
+        uses: docker/setup-buildx-action@v3.0.0
+        with:
+          endpoint: builders
+      - name: '[preparation] docker login'
+        uses: docker/login-action@v3.0.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: '[preparation] set the container image tag'
+        run: |
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+          
+          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}" >> $GITHUB_ENV
+      - name: '[execution] build linux & windows portainer binaries'
+        run: |
+          export YARN_VERSION=$(yarn --version)
+          export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
+          export BUILDNUMBER=${GITHUB_RUN_NUMBER}
+          GIT_COMMIT_HASH_LONG=${{ github.sha }}
+          export GIT_COMMIT_HASH_SHORT={GIT_COMMIT_HASH_LONG:0:7}
+          
+          NODE_ENV="testing"
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            NODE_ENV="production"
+          fi
+          
+          make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+      - name: '[execution] build and push docker images'
+        run: |
+          if [ "${{ matrix.config.platform }}" == "windows" ]; then
+            mv dist/portainer dist/portainer.exe
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+          else 
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            
+            if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+              docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+            fi
+          fi
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+  build_manifests:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    needs: [build_images]
+    steps:
+      - name: '[preparation] docker login'
+        uses: docker/login-action@v3.0.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: '[preparation] set up docker context for buildx'
+        run: docker version && docker context create builders
+      - name: '[preparation] set up docker buildx'
+        uses: docker/setup-buildx-action@v3.0.0
+        with:
+          endpoint: builders
+      - name: '[execution] build and push manifests'
+        run: |
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            # use the release branch name as the tag for release branches
+            # for instance, release/2.19 becomes 2.19
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
+          elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            # use pr${{ github.event.number }} as the tag for pull requests
+            # for instance, pr123
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            # replace / with - in the branch name
+            # for instance, feature/1.0.0 -> feature-1.0.0
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
+          
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
+            
+          if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
+            docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x"
+            
+            docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
+              "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
+          fi

.github/workflows/lint.yml

@@ -11,21 +11,31 @@ on:
       - master
       - develop
       - release/*
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  GO_VERSION: 1.21.9
+  NODE_VERSION: 18.x
 
 jobs:
   run-linters:
     name: Run linters
     runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '18'
+          node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - uses: actions/setup-go@v4
         with:
-          go-version: 1.19.5
+          go-version: ${{ env.GO_VERSION }}
       - run: yarn --frozen-lockfile
       - name: Run linters
         uses: wearerequired/lint-action@v1
@@ -41,6 +51,5 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
-          working-directory: api
+          version: v1.55.2
           args: --timeout=10m -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -5,6 +5,9 @@ on:
     - cron: '0 20 * * *'
   workflow_dispatch:
 
+env:
+  GO_VERSION: 1.21.9
+
 jobs:
   client-dependencies:
     name: Client Dependency Check
@@ -25,7 +28,7 @@ jobs:
         with:
           json: true
 
-      - name: upload scan result as develop artifact 
+      - name: upload scan result as develop artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
@@ -41,7 +44,7 @@ jobs:
           name: html-js-result-${{github.run_id}}
           path: js-result.html
 
-      - name: analyse vulnerabilities 
+      - name: analyse vulnerabilities
         id: set-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
@@ -58,10 +61,10 @@ jobs:
       - name: checkout repository
         uses: actions/checkout@master
 
-      - name: install Go 
+      - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -72,9 +75,9 @@ jobs:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: upload scan result as develop artifact 
+      - name: upload scan result as develop artifact
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-develop-result
@@ -102,35 +105,68 @@ jobs:
     if: >-
       github.ref == 'refs/heads/develop'
     outputs:
-      image: ${{ steps.set-matrix.outputs.image_result }}
+      image-trivy: ${{ steps.set-trivy-matrix.outputs.image_trivy_result }}
+      image-docker-scout: ${{ steps.set-docker-scout-matrix.outputs.image_docker_scout_result }}
     steps:
-      - name: scan vulnerabilities by Trivy 
+      - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
           args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
 
-      - name: upload image security scan result as artifact
+      - name: upload Trivy image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: develop scan report export to html
+      - name: develop Trivy scan report export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-trivy-result")
 
-      - name: upload html file as artifact
+      - name: upload html file as Trivy artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-${{github.run_id}}
-          path: image-result.html
+          path: image-trivy-result.html
 
-      - name: analyse vulnerabilities
-        id: set-matrix
+      - name: analyse vulnerabilities from Trivy
+        id: set-trivy-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
-          echo "image_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_trivy_result=${result}" >> $GITHUB_OUTPUT
+
+      - name: scan vulnerabilities by Docker Scout
+        uses: docker/scout-action@v1
+        continue-on-error: true
+        with:
+          command: cves
+          image: portainerci/portainer:develop
+          sarif-file: image-docker-scout.json
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: upload Docker Scout image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-develop-result
+          path: image-docker-scout.json
+
+      - name: develop Docker Scout scan report export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
+
+      - name: upload html file as Docker Scout artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-${{github.run_id}}
+          path: image-docker-scout-result.html
+
+      - name: analyse vulnerabilities from Docker Scout
+        id: set-docker-scout-matrix
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=matrix)
+          echo "image_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Results
@@ -142,22 +178,26 @@ jobs:
       matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
-        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
+        image-trivy: ${{fromJson(needs.image-vulnerability.outputs.image-trivy)}}
+        image-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.image-docker-scout)}}
     steps:
       - name: display the results of js, Go, and image scan
         run: |
           echo "${{ matrix.js.status }}"
           echo "${{ matrix.go.status }}"
-          echo "${{ matrix.image.status }}"
+          echo "${{ matrix.image-trivy.status }}"
+          echo "${{ matrix.image-docker-scout.status }}"
           echo "${{ matrix.js.summary }}"
           echo "${{ matrix.go.summary }}"
-          echo "${{ matrix.image.summary }}"
+          echo "${{ matrix.image-trivy.summary }}"
+          echo "${{ matrix.image-docker-scout.summary }}"
 
-      - name: send message to Slack 
-        if: >- 
+      - name: send message to Slack
+        if: >-
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
-          matrix.image.status == 'failure'
+          matrix.image-trivy.status == 'failure' || 
+          matrix.image-docker-scout.status == 'failure'
         uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |
@@ -193,7 +233,14 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
+                        "text": "*Image Trivy vulnerability check*: *${{ matrix.image-trivy.status }}*\n${{ matrix.image-trivy.summary }}\n"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Image Docker Scout vulnerability check*: *${{ matrix.image-docker-scout.status }}*\n${{ matrix.image-docker-scout.summary }}\n"
                       }
                     }
                   ]

.github/workflows/pr-security.yml

@@ -7,20 +7,24 @@ on:
       - edited
     paths:
       - 'package.json'
-      - 'api/go.mod'
-      - 'gruntfile.js'
+      - 'go.mod'
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
       - '.github/workflows/pr-security.yml'
 
+env:
+  GO_VERSION: 1.21.9
+  NODE_VERSION: 18.x
+
 jobs:
   client-dependencies:
     name: Client Dependency Check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan'
+      github.event.review.body == '/scan' &&
+      github.event.pull_request.draft == false
     outputs:
       jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
     steps:
@@ -74,7 +78,8 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan'
+      github.event.review.body == '/scan' &&
+      github.event.pull_request.draft == false
     outputs:
       godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
     steps:
@@ -84,7 +89,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -95,7 +100,7 @@ jobs:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
       - name: upload scan result as pull-request artifact
         uses: actions/upload-artifact@v3
@@ -136,22 +141,24 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan'
+      github.event.review.body == '/scan' &&
+      github.event.pull_request.draft == false
     outputs:
-      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
+      imagediff-trivy: ${{ steps.set-diff-trivy-matrix.outputs.image_diff_trivy_result }}
+      imagediff-docker-scout: ${{ steps.set-diff-docker-scout-matrix.outputs.image_diff_docker_scout_result }}
     steps:
       - name: checkout code
         uses: actions/checkout@master
 
-      - name: install Go 1.19.5
+      - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: ${{ env.GO_VERSION }}
 
-      - name: install Node.js 18.x
+      - name: install Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 18.x
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Install packages
         run: yarn --frozen-lockfile
@@ -167,26 +174,26 @@ jobs:
         with:
           context: .
           file: build/linux/Dockerfile
-          tags: trivy-portainer:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+          tags: local-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/local-portainer-image.tar
 
       - name: load docker image
         run: |
-          docker load --input /tmp/trivy-portainer-image.tar
+          docker load --input /tmp/local-portainer-image.tar
 
       - name: scan vulnerabilities by Trivy
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress local-portainer:${{ github.sha }}
 
-      - name: upload image security scan result as artifact
+      - name: upload Trivy image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-feature-result
           path: image-trivy.json
 
-      - name: download artifacts from develop branch built by nightly scan
+      - name: download Trivy artifacts from develop branch built by nightly scan
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -198,21 +205,65 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: pr vs develop scan report comparison export to html
+      - name: pr vs develop Trivy scan report comparison export to html
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-trivy-result")
 
-      - name: upload html file as artifact
+      - name: upload html file as Trivy artifact
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-result.html
+          path: image-trivy-result.html
 
-      - name: analyse different vulnerabilities against develop branch
-        id: set-diff-matrix
+      - name: analyse different vulnerabilities against develop branch by Trivy
+        id: set-diff-trivy-matrix
         run: |
           result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
-          echo "image_diff_result=${result}" >> $GITHUB_OUTPUT
+          echo "image_diff_trivy_result=${result}" >> $GITHUB_OUTPUT
+
+      - name: scan vulnerabilities by Docker Scout
+        uses: docker/scout-action@v1
+        continue-on-error: true
+        with:
+          command: cves
+          image: local-portainer:${{ github.sha }}
+          sarif-file: image-docker-scout.json
+          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
+          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: upload Docker Scout image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-feature-result
+          path: image-docker-scout.json
+
+      - name: download Docker Scout artifacts from develop branch built by nightly scan
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./image-docker-scout.json ./image-docker-scout-feature.json
+          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./image-docker-scout.json ]]; then
+            mv ./image-docker-scout.json ./image-docker-scout-develop.json
+          else
+            echo "null" > ./image-docker-scout-develop.json
+          fi
+
+      - name: pr vs develop Docker Scout scan report comparison export to html
+        run: |
+          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
+
+      - name: upload html file as Docker Scout artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-compare-to-develop-${{github.run_id}}
+          path: image-docker-scout-result.html
+
+      - name: analyse different vulnerabilities against develop branch by Docker Scout
+        id: set-diff-docker-scout-matrix
+        run: |
+          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=matrix)
+          echo "image_diff_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
     name: Analyse Scan Result Against develop Branch
@@ -220,23 +271,28 @@ jobs:
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
-      github.event.review.body == '/scan'
+      github.event.review.body == '/scan' &&
+      github.event.pull_request.draft == false
     strategy:
       matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
-        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
+        imagediff-trivy: ${{fromJson(needs.image-vulnerability.outputs.imagediff-trivy)}}
+        imagediff-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.imagediff-docker-scout)}}
     steps:
       - name: check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff.status == 'failure'
+          matrix.imagediff-trivy.status == 'failure' ||
+          matrix.imagediff-docker-scout.status == 'failure'
         run: |
           echo "${{ matrix.jsdiff.status }}"
           echo "${{ matrix.godiff.status }}"
-          echo "${{ matrix.imagediff.status }}"
+          echo "${{ matrix.imagediff-trivy.status }}"
+          echo "${{ matrix.imagediff-docker-scout.status }}"
           echo "${{ matrix.jsdiff.summary }}"
           echo "${{ matrix.godiff.summary }}"
-          echo "${{ matrix.imagediff.summary }}"
+          echo "${{ matrix.imagediff-trivy.summary }}"
+          echo "${{ matrix.imagediff-docker-scout.summary }}"
           exit 1

.github/workflows/stale.yml

@@ -1,7 +1,8 @@
 name: Close Stale Issues
 on:
   schedule:
-  - cron: '0 12 * * *'
+    - cron: '0 12 * * *'
+  workflow_dispatch:
 jobs:
   stale:
     runs-on: ubuntu-latest
@@ -9,7 +10,7 @@ jobs:
       issues: write
 
     steps:
-    - uses: actions/stale@v4.0.0
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/test.yaml

@@ -1,29 +1,56 @@
 name: Test
-on: push
+
+env:
+  GO_VERSION: 1.21.9
+  NODE_VERSION: 18.x
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+      - release/*
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+  push:
+    branches:
+      - master
+      - develop
+      - release/*
+
 jobs:
   test-client:
     runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '18'
+          node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
-        run: yarn jest --maxWorkers=2
-  # test-server:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     GOPRIVATE: "github.com/portainer"
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - uses: actions/setup-go@v3
-  #       with:
-  #         go-version: '1.18'
-  #     - name: Run tests
-  #       run: |
-  #         cd api
-  #         go test ./...
+        run: make test-client ARGS="--maxWorkers=2 --minWorkers=1"
+  test-server:
+    strategy:
+      matrix:
+        config:
+          - { platform: linux, arch: amd64 }
+          - { platform: linux, arch: arm64 }
+          - { platform: windows, arch: amd64, version: 1809 }
+          - { platform: windows, arch: amd64, version: ltsc2022 }
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Run tests
+        run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -6,22 +6,32 @@ on:
       - master
       - develop
       - 'release/*'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+env:
+  GO_VERSION: 1.21.9
+  NODE_VERSION: 18.x
 
 jobs:
   openapi-spec:
     runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
     steps:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: ${{ env.GO_VERSION }}
 
       - name: Download golang modules
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: '18'
+          node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

.gitignore

@@ -11,7 +11,7 @@ storybook-static
 *.DS_Store
 
 .eslintcache
-__debug_bin
+__debug_bin*
 
 api/docs
 .idea

.golangci.yaml

@@ -0,0 +1,40 @@
+linters:
+  # Disable all linters, the defaults don't pass on our code yet
+  disable-all: true
+
+  # Enable these for now
+  enable:
+    - unused
+    - depguard
+    - gosimple
+    - govet
+    - errorlint
+    - exportloopref
+
+linters-settings:
+  depguard:
+    rules:
+      main:
+        deny:
+          - pkg: 'encoding/json'
+            desc: 'use github.com/segmentio/encoding/json'
+          - pkg: 'github.com/sirupsen/logrus'
+            desc: 'logging is allowed only by github.com/rs/zerolog'
+          - pkg: 'golang.org/x/exp'
+            desc: 'exp is not allowed'
+          - pkg: 'github.com/portainer/libcrypto'
+            desc: 'use github.com/portainer/portainer/pkg/libcrypto'
+          - pkg: 'github.com/portainer/libhttp'
+            desc: 'use github.com/portainer/portainer/pkg/libhttp'
+        files:
+          - '!**/*_test.go'
+          - '!**/base.go'
+          - '!**/base_tx.go'
+
+# errorlint is causing a typecheck error for some reason. The go compiler will report these
+# anyway, so ignore them from the linter
+issues:
+  exclude-rules:
+    - path: ./
+      linters:
+        - typecheck

.prettierrc

@@ -2,18 +2,24 @@
   "printWidth": 180,
   "singleQuote": true,
   "htmlWhitespaceSensitivity": "strict",
+  "trailingComma": "es5",
   "overrides": [
     {
-      "files": ["*.html"],
+      "files": [
+        "*.html"
+      ],
       "options": {
         "parser": "angular"
       }
     },
     {
-      "files": ["*.{j,t}sx", "*.ts"],
+      "files": [
+        "*.{j,t}sx",
+        "*.ts"
+      ],
       "options": {
         "printWidth": 80
       }
     }
   ]
-}
+}

.storybook/main.ts

@@ -3,6 +3,7 @@ import { StorybookConfig } from '@storybook/react-webpack5';
 import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
 import { Configuration } from 'webpack';
 import postcss from 'postcss';
+
 const config: StorybookConfig = {
   stories: ['../app/**/*.stories.@(ts|tsx)'],
   addons: [
@@ -87,9 +88,6 @@ const config: StorybookConfig = {
     name: '@storybook/react-webpack5',
     options: {},
   },
-  docs: {
-    autodocs: true,
-  },
 };
 
 export default config;

.storybook/preview.tsx

@@ -1,23 +1,25 @@
 import '../app/assets/css';
-
+import React from 'react';
 import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
-import { initialize as initMSW, mswDecorator } from 'msw-storybook-addon';
-import { handlers } from '@/setup-tests/server-handlers';
-import { QueryClient, QueryClientProvider } from 'react-query';
+import { initialize as initMSW, mswLoader } from 'msw-storybook-addon';
+import { handlers } from '../app/setup-tests/server-handlers';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 
-// Initialize MSW
-initMSW({
-  onUnhandledRequest: ({ method, url }) => {
-    if (url.pathname.startsWith('/api')) {
-      console.error(`Unhandled ${method} request to ${url}.
+initMSW(
+  {
+    onUnhandledRequest: ({ method, url }) => {
+      if (url.startsWith('/api')) {
+        console.error(`Unhandled ${method} request to ${url}.
 
         This exception has been only logged in the console, however, it's strongly recommended to resolve this error as you don't want unmocked data in Storybook stories.
 
         If you wish to mock an error response, please refer to this guide: https://mswjs.io/docs/recipes/mocking-error-responses
       `);
-    }
+      }
+    },
   },
-});
+  handlers
+);
 
 export const parameters = {
   actions: { argTypesRegex: '^on[A-Z].*' },
@@ -44,5 +46,6 @@ export const decorators = [
       </UIRouter>
     </QueryClientProvider>
   ),
-  mswDecorator,
 ];
+
+export const loaders = [mswLoader];

.storybook/public/mockServiceWorker.js

@@ -2,22 +2,22 @@
 /* tslint:disable */
 
 /**
- * Mock Service Worker (0.36.3).
+ * Mock Service Worker (2.0.11).
  * @see https://github.com/mswjs/msw
  * - Please do NOT modify this file.
  * - Please do NOT serve this file on production.
  */
 
-const INTEGRITY_CHECKSUM = '02f4ad4a2797f85668baf196e553d929';
-const bypassHeaderName = 'x-msw-bypass';
+const INTEGRITY_CHECKSUM = 'c5f7f8e188b673ea4e677df7ea3c5a39';
+const IS_MOCKED_RESPONSE = Symbol('isMockedResponse');
 const activeClientIds = new Set();
 
 self.addEventListener('install', function () {
-  return self.skipWaiting();
+  self.skipWaiting();
 });
 
-self.addEventListener('activate', async function (event) {
-  return self.clients.claim();
+self.addEventListener('activate', function (event) {
+  event.waitUntil(self.clients.claim());
 });
 
 self.addEventListener('message', async function (event) {
@@ -33,7 +33,9 @@ self.addEventListener('message', async function (event) {
     return;
   }
 
-  const allClients = await self.clients.matchAll();
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  });
 
   switch (event.data) {
     case 'KEEPALIVE_REQUEST': {
@@ -83,165 +85,8 @@ self.addEventListener('message', async function (event) {
   }
 });
 
-// Resolve the "main" client for the given event.
-// Client that issues a request doesn't necessarily equal the client
-// that registered the worker. It's with the latter the worker should
-// communicate with during the response resolving phase.
-async function resolveMainClient(event) {
-  const client = await self.clients.get(event.clientId);
-
-  if (client.frameType === 'top-level') {
-    return client;
-  }
-
-  const allClients = await self.clients.matchAll();
-
-  return allClients
-    .filter((client) => {
-      // Get only those clients that are currently visible.
-      return client.visibilityState === 'visible';
-    })
-    .find((client) => {
-      // Find the client ID that's recorded in the
-      // set of clients that have registered the worker.
-      return activeClientIds.has(client.id);
-    });
-}
-
-async function handleRequest(event, requestId) {
-  const client = await resolveMainClient(event);
-  const response = await getResponse(event, client, requestId);
-
-  // Send back the response clone for the "response:*" life-cycle events.
-  // Ensure MSW is active and ready to handle the message, otherwise
-  // this message will pend indefinitely.
-  if (client && activeClientIds.has(client.id)) {
-    (async function () {
-      const clonedResponse = response.clone();
-      sendToClient(client, {
-        type: 'RESPONSE',
-        payload: {
-          requestId,
-          type: clonedResponse.type,
-          ok: clonedResponse.ok,
-          status: clonedResponse.status,
-          statusText: clonedResponse.statusText,
-          body: clonedResponse.body === null ? null : await clonedResponse.text(),
-          headers: serializeHeaders(clonedResponse.headers),
-          redirected: clonedResponse.redirected,
-        },
-      });
-    })();
-  }
-
-  return response;
-}
-
-async function getResponse(event, client, requestId) {
-  const { request } = event;
-  const requestClone = request.clone();
-  const getOriginalResponse = () => fetch(requestClone);
-
-  // Bypass mocking when the request client is not active.
-  if (!client) {
-    return getOriginalResponse();
-  }
-
-  // Bypass initial page load requests (i.e. static assets).
-  // The absence of the immediate/parent client in the map of the active clients
-  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
-  // and is not ready to handle requests.
-  if (!activeClientIds.has(client.id)) {
-    return await getOriginalResponse();
-  }
-
-  // Bypass requests with the explicit bypass header
-  if (requestClone.headers.get(bypassHeaderName) === 'true') {
-    const cleanRequestHeaders = serializeHeaders(requestClone.headers);
-
-    // Remove the bypass header to comply with the CORS preflight check.
-    delete cleanRequestHeaders[bypassHeaderName];
-
-    const originalRequest = new Request(requestClone, {
-      headers: new Headers(cleanRequestHeaders),
-    });
-
-    return fetch(originalRequest);
-  }
-
-  // Send the request to the client-side MSW.
-  const reqHeaders = serializeHeaders(request.headers);
-  const body = await request.text();
-
-  const clientMessage = await sendToClient(client, {
-    type: 'REQUEST',
-    payload: {
-      id: requestId,
-      url: request.url,
-      method: request.method,
-      headers: reqHeaders,
-      cache: request.cache,
-      mode: request.mode,
-      credentials: request.credentials,
-      destination: request.destination,
-      integrity: request.integrity,
-      redirect: request.redirect,
-      referrer: request.referrer,
-      referrerPolicy: request.referrerPolicy,
-      body,
-      bodyUsed: request.bodyUsed,
-      keepalive: request.keepalive,
-    },
-  });
-
-  switch (clientMessage.type) {
-    case 'MOCK_SUCCESS': {
-      return delayPromise(() => respondWithMock(clientMessage), clientMessage.payload.delay);
-    }
-
-    case 'MOCK_NOT_FOUND': {
-      return getOriginalResponse();
-    }
-
-    case 'NETWORK_ERROR': {
-      const { name, message } = clientMessage.payload;
-      const networkError = new Error(message);
-      networkError.name = name;
-
-      // Rejecting a request Promise emulates a network error.
-      throw networkError;
-    }
-
-    case 'INTERNAL_ERROR': {
-      const parsedBody = JSON.parse(clientMessage.payload.body);
-
-      console.error(
-        `\
-[MSW] Uncaught exception in the request handler for "%s %s":
-
-${parsedBody.location}
-
-This exception has been gracefully handled as a 500 response, however, it's strongly recommended to resolve this error, as it indicates a mistake in your code. If you wish to mock an error response, please see this guide: https://mswjs.io/docs/recipes/mocking-error-responses\
-`,
-        request.method,
-        request.url
-      );
-
-      return respondWithMock(clientMessage);
-    }
-  }
-
-  return getOriginalResponse();
-}
-
 self.addEventListener('fetch', function (event) {
   const { request } = event;
-  const accept = request.headers.get('accept') || '';
-
-  // Bypass server-sent events.
-  if (accept.includes('text/event-stream')) {
-    return;
-  }
 
   // Bypass navigation requests.
   if (request.mode === 'navigate') {
@@ -261,36 +106,149 @@ self.addEventListener('fetch', function (event) {
     return;
   }
 
-  const requestId = uuidv4();
-
-  return event.respondWith(
-    handleRequest(event, requestId).catch((error) => {
-      if (error.name === 'NetworkError') {
-        console.warn('[MSW] Successfully emulated a network error for the "%s %s" request.', request.method, request.url);
-        return;
-      }
-
-      // At this point, any exception indicates an issue with the original request/response.
-      console.error(
-        `\
-[MSW] Caught an exception from the "%s %s" request (%s). This is probably not a problem with Mock Service Worker. There is likely an additional logging output above.`,
-        request.method,
-        request.url,
-        `${error.name}: ${error.message}`
-      );
-    })
-  );
+  // Generate unique request ID.
+  const requestId = crypto.randomUUID();
+  event.respondWith(handleRequest(event, requestId));
 });
 
-function serializeHeaders(headers) {
-  const reqHeaders = {};
-  headers.forEach((value, name) => {
-    reqHeaders[name] = reqHeaders[name] ? [].concat(reqHeaders[name]).concat(value) : value;
-  });
-  return reqHeaders;
+async function handleRequest(event, requestId) {
+  const client = await resolveMainClient(event);
+  const response = await getResponse(event, client, requestId);
+
+  // Send back the response clone for the "response:*" life-cycle events.
+  // Ensure MSW is active and ready to handle the message, otherwise
+  // this message will pend indefinitely.
+  if (client && activeClientIds.has(client.id)) {
+    (async function () {
+      const responseClone = response.clone();
+
+      sendToClient(
+        client,
+        {
+          type: 'RESPONSE',
+          payload: {
+            requestId,
+            isMockedResponse: IS_MOCKED_RESPONSE in response,
+            type: responseClone.type,
+            status: responseClone.status,
+            statusText: responseClone.statusText,
+            body: responseClone.body,
+            headers: Object.fromEntries(responseClone.headers.entries()),
+          },
+        },
+        [responseClone.body]
+      );
+    })();
+  }
+
+  return response;
 }
 
-function sendToClient(client, message) {
+// Resolve the main client for the given event.
+// Client that issues a request doesn't necessarily equal the client
+// that registered the worker. It's with the latter the worker should
+// communicate with during the response resolving phase.
+async function resolveMainClient(event) {
+  const client = await self.clients.get(event.clientId);
+
+  if (client?.frameType === 'top-level') {
+    return client;
+  }
+
+  const allClients = await self.clients.matchAll({
+    type: 'window',
+  });
+
+  return allClients
+    .filter((client) => {
+      // Get only those clients that are currently visible.
+      return client.visibilityState === 'visible';
+    })
+    .find((client) => {
+      // Find the client ID that's recorded in the
+      // set of clients that have registered the worker.
+      return activeClientIds.has(client.id);
+    });
+}
+
+async function getResponse(event, client, requestId) {
+  const { request } = event;
+
+  // Clone the request because it might've been already used
+  // (i.e. its body has been read and sent to the client).
+  const requestClone = request.clone();
+
+  function passthrough() {
+    const headers = Object.fromEntries(requestClone.headers.entries());
+
+    // Remove internal MSW request header so the passthrough request
+    // complies with any potential CORS preflight checks on the server.
+    // Some servers forbid unknown request headers.
+    delete headers['x-msw-intention'];
+
+    return fetch(requestClone, { headers });
+  }
+
+  // Bypass mocking when the client is not active.
+  if (!client) {
+    return passthrough();
+  }
+
+  // Bypass initial page load requests (i.e. static assets).
+  // The absence of the immediate/parent client in the map of the active clients
+  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
+  // and is not ready to handle requests.
+  if (!activeClientIds.has(client.id)) {
+    return passthrough();
+  }
+
+  // Bypass requests with the explicit bypass header.
+  // Such requests can be issued by "ctx.fetch()".
+  const mswIntention = request.headers.get('x-msw-intention');
+  if (['bypass', 'passthrough'].includes(mswIntention)) {
+    return passthrough();
+  }
+
+  // Notify the client that a request has been intercepted.
+  const requestBuffer = await request.arrayBuffer();
+  const clientMessage = await sendToClient(
+    client,
+    {
+      type: 'REQUEST',
+      payload: {
+        id: requestId,
+        url: request.url,
+        mode: request.mode,
+        method: request.method,
+        headers: Object.fromEntries(request.headers.entries()),
+        cache: request.cache,
+        credentials: request.credentials,
+        destination: request.destination,
+        integrity: request.integrity,
+        redirect: request.redirect,
+        referrer: request.referrer,
+        referrerPolicy: request.referrerPolicy,
+        body: requestBuffer,
+        keepalive: request.keepalive,
+      },
+    },
+    [requestBuffer]
+  );
+
+  switch (clientMessage.type) {
+    case 'MOCK_RESPONSE': {
+      return respondWithMock(clientMessage.data);
+    }
+
+    case 'MOCK_NOT_FOUND': {
+      return passthrough();
+    }
+  }
+
+  return passthrough();
+}
+
+function sendToClient(client, message, transferrables = []) {
   return new Promise((resolve, reject) => {
     const channel = new MessageChannel();
 
@@ -302,27 +260,25 @@ function sendToClient(client, message) {
       resolve(event.data);
     };
 
-    client.postMessage(JSON.stringify(message), [channel.port2]);
+    client.postMessage(message, [channel.port2].concat(transferrables.filter(Boolean)));
   });
 }
 
-function delayPromise(cb, duration) {
-  return new Promise((resolve) => {
-    setTimeout(() => resolve(cb()), duration);
-  });
-}
+async function respondWithMock(response) {
+  // Setting response status code to 0 is a no-op.
+  // However, when responding with a "Response.error()", the produced Response
+  // instance will have status code set to 0. Since it's not possible to create
+  // a Response instance with status code 0, handle that use-case separately.
+  if (response.status === 0) {
+    return Response.error();
+  }
 
-function respondWithMock(clientMessage) {
-  return new Response(clientMessage.payload.body, {
-    ...clientMessage.payload,
-    headers: clientMessage.payload.headers,
-  });
-}
+  const mockedResponse = new Response(response.body, response);
 
-function uuidv4() {
-  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-    const r = (Math.random() * 16) | 0;
-    const v = c == 'x' ? r : (r & 0x3) | 0x8;
-    return v.toString(16);
+  Reflect.defineProperty(mockedResponse, IS_MOCKED_RESPONSE, {
+    value: true,
+    enumerable: true,
   });
+
+  return mockedResponse;
 }

Makefile

@@ -7,9 +7,9 @@ ARCH=$(shell go env GOARCH)
 # build target, can be one of "production", "testing", "development"
 ENV=development
 WEBPACK_CONFIG=webpack/webpack.$(ENV).js
-TAG=latest
+TAG=local
 
-SWAG=go run github.com/swaggo/swag/cmd/swag@v1.8.11 
+SWAG=go run github.com/swaggo/swag/cmd/swag@v1.16.2 
 GOTESTSUM=go run gotest.tools/gotestsum@latest
 
 # Don't change anything below this line unless you know what you're doing
@@ -65,10 +65,10 @@ clean: ## Remove all build and download artifacts
 test: test-server test-client ## Run all tests
 
 test-client: ## Run client tests
-	yarn test
+	yarn test $(ARGS)
 
 test-server:	## Run server tests
-	cd api && $(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
+	$(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
 
 ##@ Dev
 .PHONY: dev dev-client dev-server
@@ -92,7 +92,7 @@ format-client: ## Format client code
 	yarn format
 
 format-server: ## Format server code
-	cd api && go fmt ./...
+	go fmt ./...
 
 ##@ Lint
 .PHONY: lint lint-client lint-server
@@ -102,7 +102,7 @@ lint-client: ## Lint client code
 	yarn lint
 
 lint-server: ## Lint server code
-	cd api && go vet ./...
+	golangci-lint run --timeout=10m -c .golangci.yaml
 
 
 ##@ Extension
@@ -114,7 +114,7 @@ dev-extension: build-server build-client ## Run the extension in development mod
 ##@ Docs
 .PHONY: docs-build docs-validate docs-clean docs-validate-clean
 docs-build: init-dist ## Build docs
-	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
+	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./ 
 
 docs-validate: docs-build ## Validate docs
 	yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml

README.md

@@ -9,7 +9,7 @@ Portainer consists of a single container that can run on any cluster. It can be
 **Portainer Business Edition** builds on the open-source base and includes a range of advanced features and functions (like RBAC and Support) that are specific to the needs of business users.
 
 - [Compare Portainer CE and Compare Portainer BE](https://portainer.io/products)
-- [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
+- [Take3 – get 3 free nodes of Portainer Business for as long as you want them](https://www.portainer.io/take-3)
 - [Portainer BE install guide](https://install.portainer.io)
 
 ## Latest Version
@@ -21,8 +21,8 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
 ## Getting started
 
 - [Deploy Portainer](https://docs.portainer.io/start/install)
-- [Documentation](https://documentation.portainer.io)
-- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
+- [Documentation](https://docs.portainer.io)
+- [Contribute to the project](https://docs.portainer.io/contribute/contribute)
 
 ## Features & Functions
 
@@ -30,23 +30,22 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 - [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
 - [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
-- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
 
 ## Getting help
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
+Learn more about Portainer's community support channels [here.](https://www.portainer.io/get-support-for-portainer)
 
 - Issues: https://github.com/portainer/portainer/issues
 - Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
 
-You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
+You can join the Portainer Community by visiting [https://www.portainer.io/join-our-community](https://www.portainer.io/join-our-community). This will give you advance notice of events, content and other related Portainer content.
 
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
-- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://documentation.portainer.io/contributing/instructions/) to build it locally and make a pull request.
+- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
 
 ## Security
 
@@ -60,7 +59,7 @@ If you are a developer, and our code in this repo makes sense to you, we would l
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
 
-When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/documentation/in-app-analytics-and-privacy-policy/). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
+When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
 
 ## Limitations
 

api/.golangci.yaml

@@ -1,29 +0,0 @@
-linters:
-  # Disable all linters, the defaults don't pass on our code yet
-  disable-all: true
-
-  # Enable these for now
-  enable:
-    - depguard
-    - govet
-    - errorlint
-    - exportloopref
-linters-settings:
-  depguard:
-    list-type: denylist
-    include-go-root: true
-    packages:
-      - github.com/sirupsen/logrus
-      - golang.org/x/exp
-    packages-with-error-message:
-      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
-    ignore-file-rules:
-      - '**/*_test.go'
-
-# errorlint is causing a typecheck error for some reason. The go compiler will report these
-# anyway, so ignore them from the linter
-issues:
-  exclude-rules:
-    - path: ./
-      linters:
-        - typecheck

api/adminmonitor/admin_monitor.go

@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
-	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/rs/zerolog/log"
 )

api/apikey/apikey.go

@@ -1,30 +1,17 @@
 package apikey
 
 import (
-	"crypto/rand"
-	"io"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
 // APIKeyService represents a service for managing API keys.
 type APIKeyService interface {
-	HashRaw(rawKey string) []byte
+	HashRaw(rawKey string) string
 	GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error)
 	GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error)
 	GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey, error)
-	GetDigestUserAndKey(digest []byte) (portainer.User, portainer.APIKey, error)
+	GetDigestUserAndKey(digest string) (portainer.User, portainer.APIKey, error)
 	UpdateAPIKey(apiKey *portainer.APIKey) error
 	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
 	InvalidateUserKeyCache(userId portainer.UserID) bool
 }
-
-// generateRandomKey generates a random key of specified length
-// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
-func generateRandomKey(length int) []byte {
-	k := make([]byte, length)
-	if _, err := io.ReadFull(rand.Reader, k); err != nil {
-		return nil
-	}
-	return k
-}

api/apikey/apikey_test.go

@@ -3,6 +3,7 @@ package apikey
 import (
 	"testing"
 
+	"github.com/portainer/portainer/api/internal/securecookie"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,7 +34,7 @@ func Test_generateRandomKey(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := generateRandomKey(tt.wantLenth)
+			got := securecookie.GenerateRandomKey(tt.wantLenth)
 			is.Equal(tt.wantLenth, len(got))
 		})
 	}
@@ -41,7 +42,7 @@ func Test_generateRandomKey(t *testing.T) {
 	t.Run("Generated keys are unique", func(t *testing.T) {
 		keys := make(map[string]bool)
 		for i := 0; i < 100; i++ {
-			key := generateRandomKey(8)
+			key := securecookie.GenerateRandomKey(8)
 			_, ok := keys[string(key)]
 			is.False(ok)
 			keys[string(key)] = true

api/apikey/cache.go

@@ -33,8 +33,8 @@ func NewAPIKeyCache(cacheSize int) *apiKeyCache {
 // Get returns the user/key associated to an api-key's digest
 // This is required because HTTP requests will contain the digest of the API key in header,
 // the digest value must be mapped to a portainer user.
-func (c *apiKeyCache) Get(digest []byte) (portainer.User, portainer.APIKey, bool) {
-	val, ok := c.cache.Get(string(digest))
+func (c *apiKeyCache) Get(digest string) (portainer.User, portainer.APIKey, bool) {
+	val, ok := c.cache.Get(digest)
 	if !ok {
 		return portainer.User{}, portainer.APIKey{}, false
 	}
@@ -44,23 +44,23 @@ func (c *apiKeyCache) Get(digest []byte) (portainer.User, portainer.APIKey, bool
 }
 
 // Set persists a user/key entry to the cache
-func (c *apiKeyCache) Set(digest []byte, user portainer.User, apiKey portainer.APIKey) {
-	c.cache.Add(string(digest), entry{
+func (c *apiKeyCache) Set(digest string, user portainer.User, apiKey portainer.APIKey) {
+	c.cache.Add(digest, entry{
 		user:   user,
 		apiKey: apiKey,
 	})
 }
 
 // Delete evicts a digest's user/key entry key from the cache
-func (c *apiKeyCache) Delete(digest []byte) {
-	c.cache.Remove(string(digest))
+func (c *apiKeyCache) Delete(digest string) {
+	c.cache.Remove(digest)
 }
 
 // InvalidateUserKeyCache loops through all the api-keys associated to a user and removes them from the cache
 func (c *apiKeyCache) InvalidateUserKeyCache(userId portainer.UserID) bool {
 	present := false
 	for _, k := range c.cache.Keys() {
-		user, _, _ := c.Get([]byte(k.(string)))
+		user, _, _ := c.Get(k.(string))
 		if user.ID == userId {
 			present = c.cache.Remove(k)
 		}

api/apikey/cache_test.go

@@ -17,25 +17,25 @@ func Test_apiKeyCacheGet(t *testing.T) {
 	keyCache.cache.Add(string(""), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
 
 	tests := []struct {
-		digest []byte
+		digest string
 		found  bool
 	}{
 		{
-			digest: []byte("foo"),
+			digest: "foo",
 			found:  true,
 		},
 		{
-			digest: []byte(""),
+			digest: "",
 			found:  true,
 		},
 		{
-			digest: []byte("bar"),
+			digest: "bar",
 			found:  false,
 		},
 	}
 
 	for _, test := range tests {
-		t.Run(string(test.digest), func(t *testing.T) {
+		t.Run(test.digest, func(t *testing.T) {
 			_, _, found := keyCache.Get(test.digest)
 			is.Equal(test.found, found)
 		})
@@ -48,11 +48,11 @@ func Test_apiKeyCacheSet(t *testing.T) {
 	keyCache := NewAPIKeyCache(10)
 
 	// pre-populate cache
-	keyCache.Set([]byte("bar"), portainer.User{ID: 2}, portainer.APIKey{})
-	keyCache.Set([]byte("foo"), portainer.User{ID: 1}, portainer.APIKey{})
+	keyCache.Set("bar", portainer.User{ID: 2}, portainer.APIKey{})
+	keyCache.Set("foo", portainer.User{ID: 1}, portainer.APIKey{})
 
 	// overwrite existing entry
-	keyCache.Set([]byte("foo"), portainer.User{ID: 3}, portainer.APIKey{})
+	keyCache.Set("foo", portainer.User{ID: 3}, portainer.APIKey{})
 
 	val, ok := keyCache.cache.Get(string("bar"))
 	is.True(ok)
@@ -74,14 +74,14 @@ func Test_apiKeyCacheDelete(t *testing.T) {
 
 	t.Run("Delete an existing entry", func(t *testing.T) {
 		keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
-		keyCache.Delete([]byte("foo"))
+		keyCache.Delete("foo")
 
 		_, ok := keyCache.cache.Get(string("foo"))
 		is.False(ok)
 	})
 
 	t.Run("Delete a non-existing entry", func(t *testing.T) {
-		nonPanicFunc := func() { keyCache.Delete([]byte("non-existent-key")) }
+		nonPanicFunc := func() { keyCache.Delete("non-existent-key") }
 		is.NotPanics(nonPanicFunc)
 	})
 }
@@ -131,16 +131,16 @@ func Test_apiKeyCacheLRU(t *testing.T) {
 			keyCache := NewAPIKeyCache(test.cacheLen)
 
 			for _, key := range test.key {
-				keyCache.Set([]byte(key), portainer.User{ID: 1}, portainer.APIKey{})
+				keyCache.Set(key, portainer.User{ID: 1}, portainer.APIKey{})
 			}
 
 			for _, key := range test.foundKeys {
-				_, _, found := keyCache.Get([]byte(key))
+				_, _, found := keyCache.Get(key)
 				is.True(found, "Key %s not found", key)
 			}
 
 			for _, key := range test.evictedKeys {
-				_, _, found := keyCache.Get([]byte(key))
+				_, _, found := keyCache.Get(key)
 				is.False(found, "key %s should have been evicted", key)
 			}
 		})

api/apikey/service.go

@@ -8,6 +8,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/securecookie"
 
 	"github.com/pkg/errors"
 )
@@ -31,15 +32,15 @@ func NewAPIKeyService(apiKeyRepository dataservices.APIKeyRepository, userReposi
 }
 
 // HashRaw computes a hash digest of provided raw API key.
-func (a *apiKeyService) HashRaw(rawKey string) []byte {
+func (a *apiKeyService) HashRaw(rawKey string) string {
 	hashDigest := sha256.Sum256([]byte(rawKey))
-	return hashDigest[:]
+	return base64.StdEncoding.EncodeToString(hashDigest[:])
 }
 
 // GenerateApiKey generates a raw API key for a user (for one-time display).
 // The generated API key is stored in the cache and database.
 func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
-	randKey := generateRandomKey(32)
+	randKey := securecookie.GenerateRandomKey(32)
 	encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
 	prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
 
@@ -53,7 +54,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 		Digest:      hashDigest,
 	}
 
-	err := a.apiKeyRepository.CreateAPIKey(apiKey)
+	err := a.apiKeyRepository.Create(apiKey)
 	if err != nil {
 		return "", nil, errors.Wrap(err, "Unable to create API key")
 	}
@@ -66,7 +67,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 
 // GetAPIKey returns an API key by its ID.
 func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	return a.apiKeyRepository.GetAPIKey(apiKeyID)
+	return a.apiKeyRepository.Read(apiKeyID)
 }
 
 // GetAPIKeys returns all the API keys associated to a user.
@@ -76,7 +77,7 @@ func (a *apiKeyService) GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey,
 
 // GetDigestUserAndKey returns the user and api-key associated to a specified hash digest.
 // A cache lookup is performed first; if the user/api-key is not found in the cache, respective database lookups are performed.
-func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, portainer.APIKey, error) {
+func (a *apiKeyService) GetDigestUserAndKey(digest string) (portainer.User, portainer.APIKey, error) {
 	// get api key from cache if possible
 	cachedUser, cachedKey, ok := a.cache.Get(digest)
 	if ok {
@@ -88,7 +89,7 @@ func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, port
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve API key")
 	}
 
-	user, err := a.userRepository.User(apiKey.UserID)
+	user, err := a.userRepository.Read(apiKey.UserID)
 	if err != nil {
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve digest user")
 	}
@@ -106,20 +107,20 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
 		return errors.Wrap(err, "Unable to retrieve API key")
 	}
 	a.cache.Set(apiKey.Digest, user, *apiKey)
-	return a.apiKeyRepository.UpdateAPIKey(apiKey)
+	return a.apiKeyRepository.Update(apiKey.ID, apiKey)
 }
 
 // DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
 func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
 	// get api-key digest to remove from cache
-	apiKey, err := a.apiKeyRepository.GetAPIKey(apiKeyID)
+	apiKey, err := a.apiKeyRepository.Read(apiKeyID)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
 	}
 
 	// delete the user/api-key from cache
 	a.cache.Delete(apiKey.Digest)
-	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
+	return a.apiKeyRepository.Delete(apiKeyID)
 }
 
 func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {

api/apikey/service_test.go

@@ -2,6 +2,7 @@ package apikey
 
 import (
 	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 	"strings"
 	"testing"
@@ -68,7 +69,7 @@ func Test_GenerateApiKey(t *testing.T) {
 
 		generatedDigest := sha256.Sum256([]byte(rawKey))
 
-		is.Equal(apiKey.Digest, generatedDigest[:])
+		is.Equal(apiKey.Digest, base64.StdEncoding.EncodeToString(generatedDigest[:]))
 	})
 }
 

api/archive/targz.go

@@ -48,18 +48,6 @@ func TarGzDir(absolutePath string) (string, error) {
 }
 
 func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info os.FileInfo) error {
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-
-	header.Name = pathInArchive // use relative paths in archive
-
-	err = tarWriter.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
 	if info.IsDir() {
 		return nil
 	}
@@ -68,6 +56,26 @@ func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info
 	if err != nil {
 		return err
 	}
+
+	stat, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := tar.FileInfoHeader(stat, stat.Name())
+	if err != nil {
+		return err
+	}
+	header.Name = pathInArchive // use relative paths in archive
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+	if stat.IsDir() {
+		return nil
+	}
+
 	_, err = io.Copy(tarWriter, file)
 	return err
 }
@@ -98,7 +106,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
 			// skip, dir will be created with a file
 		case tar.TypeReg:
 			p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
-			if err := os.MkdirAll(filepath.Dir(p), 0744); err != nil {
+			if err := os.MkdirAll(filepath.Dir(p), 0o744); err != nil {
 				return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
 			}
 			outFile, err := os.Create(p)

api/backup/backup.go

@@ -17,7 +17,7 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const rwxr__r__ os.FileMode = 0744
+const rwxr__r__ os.FileMode = 0o744
 
 var filesToBackup = []string{
 	"certs",
@@ -30,6 +30,7 @@ var filesToBackup = []string{
 	"portainer.key",
 	"portainer.pub",
 	"tls",
+	"chisel",
 }
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
@@ -81,14 +82,9 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 }
 
 func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
-	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
-	if err != nil {
-		return err
-	}
-	if err = datastore.BackupTo(backupWriter); err != nil {
-		return err
-	}
-	return backupWriter.Close()
+	dbFileName := datastore.Connection().GetDatabaseFileName()
+	_, err := datastore.Backup(filepath.Join(backupDirPath, dbFileName))
+	return err
 }
 
 func encrypt(path string, passphrase string) (string, error) {

api/backup/restore.go

@@ -26,7 +26,7 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
 	if password != "" {
 		archive, err = decrypt(archive, password)
 		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
+			return errors.Wrap(err, "failed to decrypt the archive. Please ensure the password is correct and try again")
 		}
 	}
 

api/build/variables.go

@@ -1,9 +1,12 @@
 package build
 
+import "runtime"
+
 // Variables to be set during the build time
 var BuildNumber string
 var ImageTag string
 var NodejsVersion string
 var YarnVersion string
 var WebpackVersion string
-var GoVersion string
+var GoVersion string = runtime.Version()
+var GitCommit string

api/chisel/crypto/crypto.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+
+	chshare "github.com/jpillora/chisel/share"
+)
+
+var one = new(big.Int).SetInt64(1)
+
+// GenerateGo119CompatibleKey This function is basically copied from chshare.GenerateKey.
+func GenerateGo119CompatibleKey(seed string) ([]byte, error) {
+	r := chshare.NewDetermRand([]byte(seed))
+	priv, err := ecdsaGenerateKey(elliptic.P256(), r)
+	if err != nil {
+		return nil, err
+	}
+	b, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to marshal ECDSA private key: %w", err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil
+}
+
+// This function is copied from Go1.19
+func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
+	params := c.Params()
+	// Note that for P-521 this will actually be 63 bits more than the order, as
+	// division rounds down, but the extra bit is inconsequential.
+	b := make([]byte, params.N.BitLen()/8+8)
+	_, err = io.ReadFull(rand, b)
+	if err != nil {
+		return
+	}
+
+	k = new(big.Int).SetBytes(b)
+	n := new(big.Int).Sub(params.N, one)
+	k.Mod(k, n)
+	k.Add(k, one)
+	return
+}
+
+// This function is copied from Go1.19
+func ecdsaGenerateKey(c elliptic.Curve, rand io.Reader) (*ecdsa.PrivateKey, error) {
+	k, err := randFieldElement(c, rand)
+	if err != nil {
+		return nil, err
+	}
+
+	priv := new(ecdsa.PrivateKey)
+	priv.PublicKey.Curve = c
+	priv.D = k
+	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
+	return priv, nil
+}

api/chisel/crypto/crypto_test.go

@@ -0,0 +1,37 @@
+package crypto
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGenerateGo119CompatibleKey(t *testing.T) {
+	type args struct {
+		seed string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name:    "Generate Go 1.19 compatible private key with a given seed",
+			args:    args{seed: "94qh17MCIk8BOkiI"},
+			want:    []byte("-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIHeohwk0Gy3RHVVViaHz7pz/HOiqA7fkv1FTM3mGgfT3oAoGCCqGSM49\nAwEHoUQDQgAEN7riX06xDsLNPuUmOvYFluNEakcFwZZRVvOcIYk/9VYnanDzW0Km\n8/BUUiKyJDuuGdS4fj9SlQ4iL8yBK01uKg==\n-----END EC PRIVATE KEY-----\n"),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GenerateGo119CompatibleKey(tt.args.seed)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GenerateGo119CompatibleKey() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GenerateGo119CompatibleKey()\ngot: Z %v\nwant: %v", got, tt.want)
+			}
+		})
+	}
+}

api/chisel/schedules.go

@@ -5,6 +5,17 @@ import (
 	"github.com/portainer/portainer/api/internal/edge/cache"
 )
 
+// EdgeJobs retrieves the edge jobs for the given environment
+func (service *Service) EdgeJobs(endpointID portainer.EndpointID) []portainer.EdgeJob {
+	service.mu.RLock()
+	defer service.mu.RUnlock()
+
+	return append(
+		make([]portainer.EdgeJob, 0, len(service.edgeJobs[endpointID])),
+		service.edgeJobs[endpointID]...,
+	)
+}
+
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
 func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portainer.EdgeJob) {
 	if endpoint.Edge.AsyncMode {
@@ -12,41 +23,40 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	}
 
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpoint.ID)
+	defer service.mu.Unlock()
 
 	existingJobIndex := -1
-	for idx, existingJob := range tunnel.Jobs {
+	for idx, existingJob := range service.edgeJobs[endpoint.ID] {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
+
 			break
 		}
 	}
 
 	if existingJobIndex == -1 {
-		tunnel.Jobs = append(tunnel.Jobs, *edgeJob)
+		service.edgeJobs[endpoint.ID] = append(service.edgeJobs[endpoint.ID], *edgeJob)
 	} else {
-		tunnel.Jobs[existingJobIndex] = *edgeJob
+		service.edgeJobs[endpoint.ID][existingJobIndex] = *edgeJob
 	}
 
 	cache.Del(endpoint.ID)
-
-	service.mu.Unlock()
 }
 
 // RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
 func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
 
-	for endpointID, tunnel := range service.tunnelDetailsMap {
+	for endpointID := range service.edgeJobs {
 		n := 0
-		for _, edgeJob := range tunnel.Jobs {
+		for _, edgeJob := range service.edgeJobs[endpointID] {
 			if edgeJob.ID != edgeJobID {
-				tunnel.Jobs[n] = edgeJob
+				service.edgeJobs[endpointID][n] = edgeJob
 				n++
 			}
 		}
 
-		tunnel.Jobs = tunnel.Jobs[:n]
+		service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 		cache.Del(endpointID)
 	}
@@ -56,19 +66,17 @@ func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 
 func (service *Service) RemoveEdgeJobFromEndpoint(endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID) {
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
+	defer service.mu.Unlock()
 
 	n := 0
-	for _, edgeJob := range tunnel.Jobs {
+	for _, edgeJob := range service.edgeJobs[endpointID] {
 		if edgeJob.ID != edgeJobID {
-			tunnel.Jobs[n] = edgeJob
+			service.edgeJobs[endpointID][n] = edgeJob
 			n++
 		}
 	}
 
-	tunnel.Jobs = tunnel.Jobs[:n]
+	service.edgeJobs[endpointID] = service.edgeJobs[endpointID][:n]
 
 	cache.Del(endpointID)
-
-	service.mu.Unlock()
 }

api/chisel/service.go

@@ -12,103 +12,134 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 
-	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
+	"github.com/jpillora/chisel/share/ccrypto"
 	"github.com/rs/zerolog/log"
 )
 
 const (
 	tunnelCleanupInterval = 10 * time.Second
-	requiredTimeout       = 15 * time.Second
 	activeTimeout         = 4*time.Minute + 30*time.Second
+	pingTimeout           = 3 * time.Second
 )
 
 // Service represents a service to manage the state of multiple reverse tunnels.
 // It is used to start a reverse tunnel server and to manage the connection status of each tunnel
 // connected to the tunnel server.
 type Service struct {
-	serverFingerprint string
-	serverPort        string
-	tunnelDetailsMap  map[portainer.EndpointID]*portainer.TunnelDetails
-	dataStore         dataservices.DataStore
-	snapshotService   portainer.SnapshotService
-	chiselServer      *chserver.Server
-	shutdownCtx       context.Context
-	ProxyManager      *proxy.Manager
-	mu                sync.Mutex
+	serverFingerprint      string
+	serverPort             string
+	activeTunnels          map[portainer.EndpointID]*portainer.TunnelDetails
+	edgeJobs               map[portainer.EndpointID][]portainer.EdgeJob
+	dataStore              dataservices.DataStore
+	snapshotService        portainer.SnapshotService
+	chiselServer           *chserver.Server
+	shutdownCtx            context.Context
+	ProxyManager           *proxy.Manager
+	mu                     sync.RWMutex
+	fileService            portainer.FileService
+	defaultCheckinInterval int
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
+func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
+	defaultCheckinInterval := portainer.DefaultEdgeAgentCheckinIntervalInSeconds
+
+	settings, err := dataStore.Settings().Settings()
+	if err == nil {
+		defaultCheckinInterval = settings.EdgeAgentCheckinInterval
+	} else {
+		log.Error().Err(err).Msg("unable to retrieve the settings from the database")
+	}
+
 	return &Service{
-		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
-		dataStore:        dataStore,
-		shutdownCtx:      shutdownCtx,
+		activeTunnels:          make(map[portainer.EndpointID]*portainer.TunnelDetails),
+		edgeJobs:               make(map[portainer.EndpointID][]portainer.EdgeJob),
+		dataStore:              dataStore,
+		shutdownCtx:            shutdownCtx,
+		fileService:            fileService,
+		defaultCheckinInterval: defaultCheckinInterval,
 	}
 }
 
 // pingAgent ping the given agent so that the agent can keep the tunnel alive
 func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
-	tunnel := service.GetTunnelDetails(endpointID)
-	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
+	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
+	if err != nil {
+		return err
+	}
+
+	tunnelAddr, err := service.TunnelAddr(endpoint)
+	if err != nil {
+		return err
+	}
+
+	requestURL := fmt.Sprintf("http://%s/ping", tunnelAddr)
 	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
 	if err != nil {
 		return err
 	}
 
 	httpClient := &http.Client{
-		Timeout: 3 * time.Second,
+		Timeout: pingTimeout,
 	}
 
 	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
 	io.Copy(io.Discard, resp.Body)
 	resp.Body.Close()
 
-	return err
+	return nil
 }
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
 func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
-	go func() {
-		log.Debug().
-			Int("endpoint_id", int(endpointID)).
-			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("start")
+	go service.keepTunnelAlive(endpointID, ctx, maxAlive)
+}
 
-		maxAliveTicker := time.NewTicker(maxAlive)
-		defer maxAliveTicker.Stop()
-		pingTicker := time.NewTicker(tunnelCleanupInterval)
-		defer pingTicker.Stop()
+func (service *Service) keepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
+	log.Debug().
+		Int("endpoint_id", int(endpointID)).
+		Float64("max_alive_minutes", maxAlive.Minutes()).
+		Msg("KeepTunnelAlive: start")
 
-		for {
-			select {
-			case <-pingTicker.C:
-				service.SetTunnelStatusToActive(endpointID)
-				err := service.pingAgent(endpointID)
-				if err != nil {
-					log.Debug().
-						Int("endpoint_id", int(endpointID)).
-						Err(err).
-						Msg("ping agent")
-				}
-			case <-maxAliveTicker.C:
-				log.Debug().
-					Int("endpoint_id", int(endpointID)).
-					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("tunnel keep alive timeout")
+	maxAliveTicker := time.NewTicker(maxAlive)
+	defer maxAliveTicker.Stop()
 
-				return
-			case <-ctx.Done():
-				err := ctx.Err()
+	pingTicker := time.NewTicker(tunnelCleanupInterval)
+	defer pingTicker.Stop()
+
+	for {
+		select {
+		case <-pingTicker.C:
+			service.UpdateLastActivity(endpointID)
+
+			if err := service.pingAgent(endpointID); err != nil {
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("tunnel stop")
-
-				return
+					Msg("KeepTunnelAlive: ping agent")
 			}
+		case <-maxAliveTicker.C:
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Float64("timeout_minutes", maxAlive.Minutes()).
+				Msg("KeepTunnelAlive: tunnel keep alive timeout")
+
+			return
+		case <-ctx.Done():
+			err := ctx.Err()
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("KeepTunnelAlive: tunnel stop")
+
+			return
 		}
-	}()
+	}
 }
 
 // StartTunnelServer starts a tunnel server on the specified addr and port.
@@ -117,14 +148,14 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // It starts the tunnel status verification process in the background.
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
-	keySeed, err := service.retrievePrivateKeySeed()
+	privateKeyFile, err := service.retrievePrivateKeyFile()
 	if err != nil {
 		return err
 	}
 
 	config := &chserver.Config{
 		Reverse: true,
-		KeySeed: keySeed,
+		KeyFile: privateKeyFile,
 	}
 
 	chiselServer, err := chserver.NewServer(config)
@@ -135,21 +166,21 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	service.serverFingerprint = chiselServer.GetFingerprint()
 	service.serverPort = port
 
-	err = chiselServer.Start(addr, port)
-	if err != nil {
+	if err := chiselServer.Start(addr, port); err != nil {
 		return err
 	}
+
 	service.chiselServer = chiselServer
 
 	// TODO: work-around Chisel default behavior.
 	// By default, Chisel will allow anyone to connect if no user exists.
 	username, password := generateRandomCredentials()
-	err = service.chiselServer.AddUser(username, password, "127.0.0.1")
-	if err != nil {
+	if err = service.chiselServer.AddUser(username, password, "127.0.0.1"); err != nil {
 		return err
 	}
 
 	service.snapshotService = snapshotService
+
 	go service.startTunnelVerificationLoop()
 
 	return nil
@@ -160,26 +191,43 @@ func (service *Service) StopTunnelServer() error {
 	return service.chiselServer.Close()
 }
 
-func (service *Service) retrievePrivateKeySeed() (string, error) {
-	var serverInfo *portainer.TunnelServerInfo
+func (service *Service) retrievePrivateKeyFile() (string, error) {
+	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
 
-	serverInfo, err := service.dataStore.TunnelServer().Info()
-	if service.dataStore.IsErrObjectNotFound(err) {
-		keySeed := uniuri.NewLen(16)
+	if exists, _ := service.fileService.FileExists(privateKeyFile); exists {
+		log.Info().
+			Str("private-key", privateKeyFile).
+			Msg("found Chisel private key file on disk")
 
-		serverInfo = &portainer.TunnelServerInfo{
-			PrivateKeySeed: keySeed,
-		}
+		return privateKeyFile, nil
+	}
+
+	log.Debug().
+		Str("private-key", privateKeyFile).
+		Msg("chisel private key file does not exist")
+
+	privateKey, err := ccrypto.GenerateKey("")
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to generate chisel private key")
 
-		err := service.dataStore.TunnelServer().UpdateInfo(serverInfo)
-		if err != nil {
-			return "", err
-		}
-	} else if err != nil {
 		return "", err
 	}
 
-	return serverInfo.PrivateKeySeed, nil
+	if err = service.fileService.StoreChiselPrivateKey(privateKey); err != nil {
+		log.Error().
+			Err(err).
+			Msg("failed to save Chisel private key to disk")
+
+		return "", err
+	}
+
+	log.Info().
+		Str("private-key", privateKeyFile).
+		Msg("generated a new Chisel private key file")
+
+	return privateKeyFile, nil
 }
 
 func (service *Service) startTunnelVerificationLoop() {
@@ -206,63 +254,45 @@ func (service *Service) startTunnelVerificationLoop() {
 	}
 }
 
+// checkTunnels finds the first tunnel that has not had any activity recently
+// and attempts to take a snapshot, then closes it and returns
 func (service *Service) checkTunnels() {
-	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
+	service.mu.RLock()
 
-	service.mu.Lock()
-	for key, tunnel := range service.tunnelDetailsMap {
-		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
-			continue
-		}
-
-		if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
-			continue
-		}
-
-		tunnels[key] = *tunnel
-	}
-	service.mu.Unlock()
-
-	for endpointID, tunnel := range tunnels {
+	for endpointID, tunnel := range service.activeTunnels {
 		elapsed := time.Since(tunnel.LastActivity)
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
-			Str("status", tunnel.Status).
-			Float64("status_time_seconds", elapsed.Seconds()).
+			Float64("last_activity_seconds", elapsed.Seconds()).
 			Msg("environment tunnel monitoring")
 
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", requiredTimeout.Seconds()).
-				Msg("REQUIRED state timeout exceeded")
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed < activeTimeout {
+			continue
 		}
 
-		if tunnel.Status == portainer.EdgeAgentActive && elapsed > activeTimeout {
-			log.Debug().
-				Int("endpoint_id", int(endpointID)).
-				Str("status", tunnel.Status).
-				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", activeTimeout.Seconds()).
-				Msg("ACTIVE state timeout exceeded")
+		tunnelPort := tunnel.Port
 
-			err := service.snapshotEnvironment(endpointID, tunnel.Port)
-			if err != nil {
-				log.Error().
-					Int("endpoint_id", int(endpointID)).
-					Err(err).
-					Msg("unable to snapshot Edge environment")
-			}
+		service.mu.RUnlock()
+
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Float64("last_activity_seconds", elapsed.Seconds()).
+			Float64("timeout_seconds", activeTimeout.Seconds()).
+			Msg("last activity timeout exceeded")
+
+		if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
+			log.Error().
+				Int("endpoint_id", int(endpointID)).
+				Err(err).
+				Msg("unable to snapshot Edge environment")
 		}
 
-		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
+		service.close(endpointID)
+
+		return
 	}
+
+	service.mu.RUnlock()
 }
 
 func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
@@ -271,14 +301,7 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 		return err
 	}
 
-	endpointURL := endpoint.URL
-
 	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
-	err = service.snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		return err
-	}
 
-	endpoint.URL = endpointURL
-	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
+	return service.snapshotService.SnapshotEndpoint(endpoint)
 }

api/chisel/service_test.go

@@ -0,0 +1,51 @@
+package chisel
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPingAgentPanic(t *testing.T) {
+	endpoint := &portainer.Endpoint{
+		ID:   1,
+		Type: portainer.EdgeAgentOnDockerEnvironment,
+	}
+
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	s := NewService(store, nil, nil)
+
+	defer func() {
+		require.Nil(t, recover())
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(pingTimeout + 1*time.Second)
+	})
+
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
+	require.NoError(t, err)
+
+	srv := &http.Server{Handler: mux}
+
+	errCh := make(chan error)
+	go func() {
+		errCh <- srv.Serve(ln)
+	}()
+
+	s.Open(endpoint)
+	s.activeTunnels[endpoint.ID].Port = ln.Addr().(*net.TCPAddr).Port
+
+	require.Error(t, s.pingAgent(endpoint.ID))
+	require.NoError(t, srv.Shutdown(context.Background()))
+	require.ErrorIs(t, <-errCh, http.ErrServerClosed)
+}

api/chisel/tunnel.go

@@ -5,14 +5,18 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"strings"
 	"time"
 
-	"github.com/portainer/libcrypto"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/pkg/libcrypto"
 
 	"github.com/dchest/uniuri"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -20,18 +24,181 @@ const (
 	maxAvailablePort = 65535
 )
 
+// Open will mark the tunnel as REQUIRED so the agent opens it
+func (s *Service) Open(endpoint *portainer.Endpoint) error {
+	if !endpointutils.IsEdgeEndpoint(endpoint) {
+		return errors.New("cannot open a tunnel for non-edge environments")
+	}
+
+	if endpoint.Edge.AsyncMode {
+		return errors.New("cannot open a tunnel for async edge environments")
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.activeTunnels[endpoint.ID]; ok {
+		return nil
+	}
+
+	defer cache.Del(endpoint.ID)
+
+	tun := &portainer.TunnelDetails{
+		Status:       portainer.EdgeAgentManagementRequired,
+		Port:         s.getUnusedPort(),
+		LastActivity: time.Now(),
+	}
+
+	username, password := generateRandomCredentials()
+
+	if s.chiselServer != nil {
+		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tun.Port)
+
+		if err := s.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
+			return err
+		}
+	}
+
+	credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
+	if err != nil {
+		return err
+	}
+
+	tun.Credentials = credentials
+
+	s.activeTunnels[endpoint.ID] = tun
+
+	return nil
+}
+
+// close removes the tunnel from the map so the agent will close it
+func (s *Service) close(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tun, ok := s.activeTunnels[endpointID]
+	if !ok {
+		return
+	}
+
+	if len(tun.Credentials) > 0 && s.chiselServer != nil {
+		user, _, _ := strings.Cut(tun.Credentials, ":")
+		s.chiselServer.DeleteUser(user)
+	}
+
+	if s.ProxyManager != nil {
+		s.ProxyManager.DeleteEndpointProxy(endpointID)
+	}
+
+	delete(s.activeTunnels, endpointID)
+
+	cache.Del(endpointID)
+}
+
+// Config returns the tunnel details needed for the agent to connect
+func (s *Service) Config(endpointID portainer.EndpointID) portainer.TunnelDetails {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		return *tun
+	}
+
+	return portainer.TunnelDetails{Status: portainer.EdgeAgentIdle}
+}
+
+// TunnelAddr returns the address of the local tunnel, including the port, it
+// will block until the tunnel is ready
+func (s *Service) TunnelAddr(endpoint *portainer.Endpoint) (string, error) {
+	if err := s.Open(endpoint); err != nil {
+		return "", err
+	}
+
+	tun := s.Config(endpoint.ID)
+	checkinInterval := time.Duration(s.tryEffectiveCheckinInterval(endpoint)) * time.Second
+
+	for t0 := time.Now(); ; {
+		if time.Since(t0) > 2*checkinInterval {
+			s.close(endpoint.ID)
+
+			return "", errors.New("unable to open the tunnel")
+		}
+
+		// Check if the tunnel is established
+		conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: tun.Port})
+		if err != nil {
+			time.Sleep(checkinInterval / 100)
+
+			continue
+		}
+
+		conn.Close()
+
+		break
+	}
+
+	s.UpdateLastActivity(endpoint.ID)
+
+	return fmt.Sprintf("127.0.0.1:%d", tun.Port), nil
+}
+
+// tryEffectiveCheckinInterval avoids a potential deadlock by returning a
+// previous known value after a timeout
+func (s *Service) tryEffectiveCheckinInterval(endpoint *portainer.Endpoint) int {
+	ch := make(chan int, 1)
+
+	go func() {
+		ch <- edge.EffectiveCheckinInterval(s.dataStore, endpoint)
+	}()
+
+	select {
+	case <-time.After(50 * time.Millisecond):
+		s.mu.RLock()
+		defer s.mu.RUnlock()
+
+		return s.defaultCheckinInterval
+	case i := <-ch:
+		s.mu.Lock()
+		s.defaultCheckinInterval = i
+		s.mu.Unlock()
+
+		return i
+	}
+}
+
+// UpdateLastActivity sets the current timestamp to avoid the tunnel timeout
+func (s *Service) UpdateLastActivity(endpointID portainer.EndpointID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if tun, ok := s.activeTunnels[endpointID]; ok {
+		tun.LastActivity = time.Now()
+	}
+}
+
 // NOTE: it needs to be called with the lock acquired
 // getUnusedPort is used to generate an unused random port in the dynamic port range.
 // Dynamic ports (also called private ports) are 49152 to 65535.
 func (service *Service) getUnusedPort() int {
 	port := randomInt(minAvailablePort, maxAvailablePort)
 
-	for _, tunnel := range service.tunnelDetailsMap {
+	for _, tunnel := range service.activeTunnels {
 		if tunnel.Port == port {
 			return service.getUnusedPort()
 		}
 	}
 
+	conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
+	if err == nil {
+		conn.Close()
+
+		log.Debug().
+			Int("port", port).
+			Msg("selected port is in use, trying a different one")
+
+		return service.getUnusedPort()
+	}
+
 	return port
 }
 
@@ -39,146 +206,10 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
-// NOTE: it needs to be called with the lock acquired
-func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
-
-	if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
-		return tunnel
-	}
-
-	tunnel := &portainer.TunnelDetails{
-		Status: portainer.EdgeAgentIdle,
-	}
-
-	service.tunnelDetailsMap[endpointID] = tunnel
-
-	cache.Del(endpointID)
-
-	return tunnel
-}
-
-// GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
-func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	return *service.getTunnelDetails(endpointID)
-}
-
-// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
-	if endpoint.Edge.AsyncMode {
-		return portainer.TunnelDetails{}, errors.New("cannot open tunnel on async endpoint")
-	}
-
-	tunnel := service.GetTunnelDetails(endpoint.ID)
-
-	if tunnel.Status == portainer.EdgeAgentActive {
-		// update the LastActivity
-		service.SetTunnelStatusToActive(endpoint.ID)
-	}
-
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := service.dataStore.Settings().Settings()
-			if err != nil {
-				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
-	}
-
-	return service.GetTunnelDetails(endpoint.ID), nil
-}
-
-// SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to ACTIVE.
-func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentActive
-	tunnel.Credentials = ""
-	tunnel.LastActivity = time.Now()
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to IDLE.
-// It removes any existing credentials associated to the tunnel.
-func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
-	service.mu.Lock()
-
-	tunnel := service.getTunnelDetails(endpointID)
-	tunnel.Status = portainer.EdgeAgentIdle
-	tunnel.Port = 0
-	tunnel.LastActivity = time.Now()
-
-	credentials := tunnel.Credentials
-	if credentials != "" {
-		tunnel.Credentials = ""
-		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
-	}
-
-	service.ProxyManager.DeleteEndpointProxy(endpointID)
-
-	service.mu.Unlock()
-
-	cache.Del(endpointID)
-}
-
-// SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
-// It sets the status to REQUIRED.
-// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
-// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
-// Credentials are encrypted using the Edge ID associated to the environment(endpoint).
-func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
-	defer cache.Del(endpointID)
-
-	tunnel := service.getTunnelDetails(endpointID)
-
-	service.mu.Lock()
-	defer service.mu.Unlock()
-
-	if tunnel.Port == 0 {
-		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
-		if err != nil {
-			return err
-		}
-
-		tunnel.Status = portainer.EdgeAgentManagementRequired
-		tunnel.Port = service.getUnusedPort()
-		tunnel.LastActivity = time.Now()
-
-		username, password := generateRandomCredentials()
-		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-		err = service.chiselServer.AddUser(username, password, authorizedRemote)
-		if err != nil {
-			return err
-		}
-
-		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
-		if err != nil {
-			return err
-		}
-		tunnel.Credentials = credentials
-	}
-
-	return nil
-}
-
 func generateRandomCredentials() (string, string) {
 	username := uniuri.NewLen(8)
 	password := uniuri.NewLen(8)
+
 	return username, password
 }
 

api/cli/cli.go

@@ -34,7 +34,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		DemoEnvironment:           kingpin.Flag("demo", "Demo environment").Bool(),
 		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
 		FeatureFlags:              kingpin.Flag("feat", "List of feature flags").Strings(),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
@@ -49,7 +48,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
@@ -62,7 +61,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
-		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()

api/cli/confirm.go

@@ -9,7 +9,7 @@ import (
 
 // Confirm starts a rollback db cli application
 func Confirm(message string) (bool, error) {
-	fmt.Printf("%s [y/N]", message)
+	fmt.Printf("%s [y/N] ", message)
 
 	reader := bufio.NewReader(os.Stdin)
 

api/cmd/portainer/log.go

@@ -39,9 +39,16 @@ func setLoggingMode(mode string) {
 	case "PRETTY":
 		log.Logger = log.Output(zerolog.ConsoleWriter{
 			Out:           os.Stderr,
-			NoColor:       true,
 			TimeFormat:    "2006/01/02 03:04PM",
-			FormatMessage: formatMessage})
+			FormatMessage: formatMessage,
+		})
+	case "NOCOLOR":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage,
+			NoColor:       true,
+		})
 	case "JSON":
 		log.Logger = log.Output(os.Stderr)
 	}
@@ -51,5 +58,6 @@ func formatMessage(i interface{}) string {
 	if i == nil {
 		return ""
 	}
+
 	return fmt.Sprintf("%s |", i)
 }

api/cmd/portainer/main.go

@@ -3,14 +3,10 @@ package main
 import (
 	"context"
 	"crypto/sha256"
-	"math/rand"
 	"os"
 	"path"
 	"strings"
-	"time"
 
-	libstack "github.com/portainer/docker-compose-wrapper"
-	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/build"
@@ -22,7 +18,8 @@ import (
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
-	"github.com/portainer/portainer/api/demo"
+	"github.com/portainer/portainer/api/datastore/migrator"
+	"github.com/portainer/portainer/api/datastore/postinit"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/exec"
@@ -44,10 +41,15 @@ import (
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/pendingactions"
+	"github.com/portainer/portainer/api/pendingactions/actions"
+	"github.com/portainer/portainer/api/pendingactions/handlers"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/pkg/featureflags"
 	"github.com/portainer/portainer/pkg/libhelm"
+	"github.com/portainer/portainer/pkg/libstack"
+	"github.com/portainer/portainer/pkg/libstack/compose"
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
@@ -119,11 +121,15 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 			log.Fatal().Err(err).Msg("failed generating instance id")
 		}
 
+		migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{})
+		migratorCount := migratorInstance.GetMigratorCountOfCurrentAPIVersion()
+
 		// from MigrateData
 		v := models.Version{
 			SchemaVersion: portainer.APIVersion,
 			Edition:       int(portainer.PortainerCE),
 			InstanceID:    instanceId.String(),
+			MigratorCount: migratorCount,
 		}
 		store.VersionService.UpdateVersion(&v)
 
@@ -152,7 +158,17 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	return store
 }
 
-func initComposeStackManager(composeDeployer libstack.Deployer, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+// checkDBSchemaServerVersionMatch checks if the server version matches the db scehma version
+func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersion string, serverEdition int) bool {
+	v, err := dbStore.Version().Version()
+	if err != nil {
+		return false
+	}
+
+	return v.SchemaVersion == serverVersion && v.Edition == serverEdition
+}
+
+func initComposeStackManager(composeDeployer libstack.Deployer, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating compose manager")
@@ -184,7 +200,7 @@ func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
 	return apikey.NewAPIKeyService(datastore.APIKeyRepository(), datastore.User())
 }
 
-func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (dataservices.JWTService, error) {
+func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (portainer.JWTService, error) {
 	if userSessionTimeout == "" {
 		userSessionTimeout = portainer.DefaultUserSessionTimeout
 	}
@@ -248,11 +264,12 @@ func initSnapshotService(
 	dockerClientFactory *dockerclient.ClientFactory,
 	kubernetesClientFactory *kubecli.ClientFactory,
 	shutdownCtx context.Context,
+	pendingActionsService *pendingactions.PendingActionsService,
 ) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
+	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx, pendingActionsService)
 	if err != nil {
 		return nil, err
 	}
@@ -383,6 +400,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("")
 	}
 
+	// check if the db schema version matches with server version
+	if !checkDBSchemaServerVersionMatch(dataStore, portainer.APIVersion, int(portainer.Edition)) {
+		log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
+	}
+
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed getting instance id")
@@ -429,17 +451,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed initializing snapshot service")
-	}
-	snapshotService.Start()
-
 	authorizationService := authorization.NewService(dataStore)
 	authorizationService.K8sClientFactory = kubernetesClientFactory
 
@@ -447,7 +463,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
 
-	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
+	proxyManager := proxy.NewManager(kubernetesClientFactory)
 
 	reverseTunnelService.ProxyManager = proxyManager
 
@@ -458,7 +474,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing compose deployer")
 	}
 
-	composeStackManager := initComposeStackManager(composeDeployer, reverseTunnelService, proxyManager)
+	composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
@@ -467,6 +483,19 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
+	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory)
+	pendingActionsService.RegisterHandler(actions.CleanNAPWithOverridePolicies, handlers.NewHandlerCleanNAPWithOverridePolicies(authorizationService, dataStore))
+	pendingActionsService.RegisterHandler(actions.DeletePortainerK8sRegistrySecrets, handlers.NewHandlerDeleteRegistrySecrets(authorizationService, dataStore, kubernetesClientFactory))
+	pendingActionsService.RegisterHandler(actions.PostInitMigrateEnvironment, handlers.NewHandlerPostInitMigrateEnvironment(authorizationService, dataStore, kubernetesClientFactory, dockerClientFactory, *flags.Assets, kubernetesDeployer))
+
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
+	}
+	snapshotService.Start()
+
+	proxyManager.NewProxyFactory(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService)
+
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing helm package manager")
@@ -479,14 +508,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	applicationStatus := initStatus(instanceID)
 
-	demoService := demo.NewService()
-	if *flags.DemoEnvironment {
-		err := demoService.Init(dataStore, cryptoService)
-		if err != nil {
-			log.Fatal().Err(err).Msg("failed initializing demo environment")
-		}
-	}
-
 	// channel to control when the admin user is created
 	adminCreationDone := make(chan struct{}, 1)
 
@@ -556,10 +577,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	// but some more complex migrations require access to a kubernetes or docker
 	// client. Therefore we run a separate migration process just before
 	// starting the server.
-	postInitMigrator := datastore.NewPostInitMigrator(
+	postInitMigrator := postinit.NewPostInitMigrator(
 		kubernetesClientFactory,
 		dockerClientFactory,
 		dataStore,
+		*flags.Assets,
+		kubernetesDeployer,
 	)
 	if err := postInitMigrator.PostInitMigrate(); err != nil {
 		log.Fatal().Err(err).Msg("failure during post init migrations")
@@ -599,15 +622,13 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
 		StackDeployer:               stackDeployer,
-		DemoService:                 demoService,
 		UpgradeService:              upgradeService,
 		AdminCreationDone:           adminCreationDone,
+		PendingActionsService:       pendingActionsService,
 	}
 }
 
 func main() {
-	rand.Seed(time.Now().UnixNano())
-
 	configureLogger()
 	setLoggingMode("PRETTY")
 
@@ -629,6 +650,7 @@ func main() {
 			Msg("starting Portainer")
 
 		err := server.Start()
+
 		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/crypto/aes.go

@@ -1,52 +1,216 @@
 package crypto
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
 	"io"
 
+	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/scrypt"
 )
 
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
+const (
+	// AES GCM settings
+	aesGcmHeader    = "AES256-GCM" // The encrypted file header
+	aesGcmBlockSize = 1024 * 1024  // 1MB block for aes gcm
 
-var emptySalt []byte = make([]byte, 0)
+	// Argon2 settings
+	// Recommded settings lower memory hardware according to current OWASP recommendations
+	// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
+	// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
+	argon2MemoryCost = 12 * 1024
+	argon2TimeCost   = 3
+	argon2Threads    = 1
+	argon2KeyLength  = 32
+)
 
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
+// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
 func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	err := aesEncryptGCM(input, output, passphrase)
 	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
+		return fmt.Errorf("error encrypting file: %w", err)
 	}
 
 	return nil
 }
 
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
 func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Read file header to determine how it was encrypted
+	inputReader := bufio.NewReader(input)
+	header, err := inputReader.Peek(len(aesGcmHeader))
+	if err != nil {
+		return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
+	}
+
+	if string(header) == aesGcmHeader {
+		reader, err := aesDecryptGCM(inputReader, passphrase)
+		if err != nil {
+			return nil, fmt.Errorf("error decrypting file: %w", err)
+		}
+
+		return reader, nil
+	}
+
+	// Use the previous decryption routine which has no header (to support older archives)
+	reader, err := aesDecryptOFB(inputReader, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
+	}
+
+	return reader, nil
+}
+
+// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
+func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
+	// Derive key using argon2 with a random salt
+	salt := make([]byte, 16) // 16 bytes salt
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		return err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return err
+	}
+
+	// Generate nonce
+	nonce, err := NewRandomNonce(aesgcm.NonceSize())
+	if err != nil {
+		return err
+	}
+
+	// write the header
+	if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
+		return err
+	}
+
+	// Write nonce and salt to the output file
+	if _, err := output.Write(salt); err != nil {
+		return err
+	}
+	if _, err := output.Write(nonce.Value()); err != nil {
+		return err
+	}
+
+	// Buffer for reading plaintext blocks
+	buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
+	ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
+
+	// Encrypt plaintext in blocks
+	for {
+		n, err := io.ReadFull(input, buf)
+		if n == 0 {
+			break // end of plaintext input
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return err
+		}
+
+		// Seal encrypts the plaintext using the nonce returning the updated slice.
+		ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
+
+		_, err = output.Write(ciphertext)
+		if err != nil {
+			return err
+		}
+
+		nonce.Increment()
+	}
+
+	return nil
+}
+
+// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
+func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Reader & verify header
+	header := make([]byte, len(aesGcmHeader))
+	if _, err := io.ReadFull(input, header); err != nil {
+		return nil, err
+	}
+
+	if string(header) != aesGcmHeader {
+		return nil, fmt.Errorf("invalid header")
+	}
+
+	// Read salt
+	salt := make([]byte, 16) // Salt size
+	if _, err := io.ReadFull(input, salt); err != nil {
+		return nil, err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+
+	// Initialize AES cipher block
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create GCM mode with the cipher block
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read nonce from the input reader
+	nonce := NewNonce(aesgcm.NonceSize())
+	if err := nonce.Read(input); err != nil {
+		return nil, err
+	}
+
+	// Initialize a buffer to store decrypted data
+	buf := bytes.Buffer{}
+	plaintext := make([]byte, aesGcmBlockSize)
+
+	// Decrypt the ciphertext in blocks
+	for {
+		// Read a block of ciphertext from the input reader
+		ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
+		n, err := io.ReadFull(input, ciphertextBlock)
+		if n == 0 {
+			break // end of ciphertext
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return nil, err
+		}
+
+		// Decrypt the block of ciphertext
+		plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = buf.Write(plaintext)
+		if err != nil {
+			return nil, err
+		}
+
+		nonce.Increment()
+	}
+
+	return &buf, nil
+}
+
+// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+// note: This function used to decrypt files that were encrypted without a header i.e. old archives
+func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
+	var emptySalt []byte = make([]byte, 0)
+
 	// making a 32 bytes key that would correspond to AES-256
 	// don't necessarily need a salt, so just kept in empty
 	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
@@ -59,11 +223,9 @@ func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
 		return nil, err
 	}
 
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
+	// If the key is unique for each ciphertext, then it's ok to use a zero IV.
 	var iv [aes.BlockSize]byte
 	stream := cipher.NewOFB(block, iv[:])
-
 	reader := &cipher.StreamReader{S: stream, R: input}
 
 	return reader, nil

api/crypto/aes_test.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -9,7 +10,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func randBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	const passphrase = "passphrase"
+
 	tmpdir := t.TempDir()
 
 	var (
@@ -18,17 +31,99 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024*1024*100 + 523)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withStrongPassphrase(t *testing.T) {
+	const passphrase = "A strong passphrase with special characters: !@#$%^&*()_+"
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withTheSamePasswordSmallFile(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
 
 	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
 	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
@@ -57,7 +152,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024 * 50)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -96,7 +191,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1034)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -117,11 +212,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	decryptedFileWriter, _ := os.Create(decryptedFilePath)
 	defer decryptedFileWriter.Close()
 
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := os.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+	_, err = AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.NotNil(t, err, "Should not allow decrypt with wrong passphrase")
 }

api/crypto/ecdsa.go

@@ -8,7 +8,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 
-	"github.com/portainer/libcrypto"
+	"github.com/portainer/portainer/pkg/libcrypto"
 )
 
 const (

api/crypto/nonce.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+type Nonce struct {
+	val []byte
+}
+
+func NewNonce(size int) *Nonce {
+	return &Nonce{val: make([]byte, size)}
+}
+
+// NewRandomNonce generates a new initial nonce with the lower byte set to a random value
+// This ensures there are plenty of nonce values availble before rolling over
+// Based on ideas from the Secure Programming Cookbook for C and C++ by John Viega, Matt Messier
+// https://www.oreilly.com/library/view/secure-programming-cookbook/0596003943/ch04s09.html
+func NewRandomNonce(size int) (*Nonce, error) {
+	randomBytes := 1
+	if size <= randomBytes {
+		return nil, errors.New("nonce size must be greater than the number of random bytes")
+	}
+
+	randomPart := make([]byte, randomBytes)
+	if _, err := rand.Read(randomPart); err != nil {
+		return nil, err
+	}
+
+	zeroPart := make([]byte, size-randomBytes)
+	nonceVal := append(randomPart, zeroPart...)
+	return &Nonce{val: nonceVal}, nil
+}
+
+func (n *Nonce) Read(stream io.Reader) error {
+	_, err := io.ReadFull(stream, n.val)
+	return err
+}
+
+func (n *Nonce) Value() []byte {
+	return n.val
+}
+
+func (n *Nonce) Increment() error {
+	// Start incrementing from the least significant byte
+	for i := len(n.val) - 1; i >= 0; i-- {
+		// Increment the current byte
+		n.val[i]++
+
+		// Check for overflow
+		if n.val[i] != 0 {
+			// No overflow, nonce is successfully incremented
+			return nil
+		}
+	}
+
+	// If we reach here, it means the nonce has overflowed
+	return errors.New("nonce overflow")
+}

api/crypto/tls.go

@@ -22,6 +22,12 @@ func CreateTLSConfiguration() *tls.Config {
 			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
 			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
 		},
 	}
 }

api/database/boltdb/db.go

@@ -8,6 +8,7 @@ import (
 	"math"
 	"os"
 	"path"
+	"strconv"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
@@ -73,7 +74,6 @@ func (connection *DbConnection) IsEncryptedStore() bool {
 // NeedsEncryptionMigration returns true if database encryption is enabled and
 // we have an un-encrypted DB that requires migration to an encrypted DB
 func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
-
 	// Cases:  Note, we need to check both portainer.db and portainer.edb
 	// to determine if it's a new store.   We only need to differentiate between cases 2,3 and 5
 
@@ -121,11 +121,11 @@ func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
 
 // Open opens and initializes the BoltDB database.
 func (connection *DbConnection) Open() error {
-
 	log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
 
 	// Now we open the db
 	databasePath := connection.GetDatabaseFilePath()
+
 	db, err := bolt.Open(databasePath, 0600, &bolt.Options{
 		Timeout:         1 * time.Second,
 		InitialMmapSize: connection.InitialMmapSize,
@@ -144,6 +144,8 @@ func (connection *DbConnection) Open() error {
 // Close closes the BoltDB database.
 // Safe to being called multiple times.
 func (connection *DbConnection) Close() error {
+	log.Info().Msg("closing PortainerDB")
+
 	if connection.DB != nil {
 		return connection.DB.Close()
 	}
@@ -176,6 +178,7 @@ func (connection *DbConnection) ViewTx(fn func(portainer.Transaction) error) err
 func (connection *DbConnection) BackupTo(w io.Writer) error {
 	return connection.View(func(tx *bolt.Tx) error {
 		_, err := tx.WriteTo(w)
+
 		return err
 	})
 }
@@ -190,6 +193,7 @@ func (connection *DbConnection) ExportRaw(filename string) error {
 	if err != nil {
 		return err
 	}
+
 	return os.WriteFile(filename, b, 0600)
 }
 
@@ -210,7 +214,7 @@ func keyToString(b []byte) string {
 
 	v := binary.BigEndian.Uint64(b)
 	if v <= math.MaxInt32 {
-		return fmt.Sprintf("%d", v)
+		return strconv.FormatUint(v, 10)
 	}
 
 	return string(b)
@@ -255,7 +259,7 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 			return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 		}
 
-		err := connection.UnmarshalObjectWithJsoniter(data, object)
+		err := connection.UnmarshalObject(data, object)
 		if err != nil {
 			return err
 		}
@@ -319,22 +323,22 @@ func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id [
 	})
 }
 
-func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (connection *DbConnection) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	return connection.ViewTx(func(tx portainer.Transaction) error {
-		return tx.GetAll(bucketName, obj, append)
+		return tx.GetAll(bucketName, obj, appendFn)
 	})
 }
 
 // TODO: decide which Unmarshal to use, and use one...
-func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	return connection.ViewTx(func(tx portainer.Transaction) error {
-		return tx.GetAllWithJsoniter(bucketName, obj, append)
+		return tx.GetAllWithJsoniter(bucketName, obj, appendFn)
 	})
 }
 
-func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	return connection.ViewTx(func(tx portainer.Transaction) error {
-		return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, append)
+		return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, appendFn)
 	})
 }
 
@@ -343,14 +347,13 @@ func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error)
 	buckets := map[string]interface{}{}
 
 	err := connection.View(func(tx *bolt.Tx) error {
-		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+		return tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
 			bucketName := string(name)
 			seqId := bucket.Sequence()
 			buckets[bucketName] = int(seqId)
+
 			return nil
 		})
-
-		return err
 	})
 
 	return buckets, err
@@ -364,6 +367,7 @@ func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error
 		id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
 		if !ok {
 			log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
+
 			continue
 		}
 

api/database/boltdb/export.go

@@ -1,10 +1,10 @@
 package boltdb
 
 import (
-	"encoding/json"
 	"time"
 
 	"github.com/rs/zerolog/log"
+	"github.com/segmentio/encoding/json"
 	bolt "go.etcd.io/bbolt"
 )
 

api/database/boltdb/json.go

@@ -1,34 +1,40 @@
 package boltdb
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
-	"encoding/json"
-	"fmt"
 	"io"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
+	"github.com/segmentio/encoding/json"
 )
 
-var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
+var errEncryptedStringTooShort = errors.New("encrypted string too short")
 
 // MarshalObject encodes an object to binary format
-func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
+func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error) {
+	buf := &bytes.Buffer{}
+
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		data = []byte(v)
+		buf.WriteString(v)
 	} else {
-		data, err = json.Marshal(object)
-		if err != nil {
-			return data, err
+		enc := json.NewEncoder(buf)
+		enc.SetSortMapKeys(false)
+		enc.SetAppendNewline(false)
+
+		if err := enc.Encode(object); err != nil {
+			return nil, err
 		}
 	}
+
 	if connection.getEncryptionKey() == nil {
-		return data, nil
+		return buf.Bytes(), nil
 	}
-	return encrypt(data, connection.getEncryptionKey())
+
+	return encrypt(buf.Bytes(), connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
@@ -54,31 +60,6 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
 	return err
 }
 
-// UnmarshalObjectWithJsoniter decodes an object from binary data
-// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
-// decoding at the moment.
-func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
-	if connection.getEncryptionKey() != nil {
-		var err error
-		data, err = decrypt(data, connection.getEncryptionKey())
-		if err != nil {
-			return err
-		}
-	}
-	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
-	err := jsoni.Unmarshal(data, &object)
-	if err != nil {
-		if s, ok := object.(*string); ok {
-			*s = string(data)
-			return nil
-		}
-
-		return err
-	}
-
-	return nil
-}
-
 // mmm, don't have a KMS .... aes GCM seems the most likely from
 // https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
 
@@ -88,22 +69,20 @@ func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error)
 	if err != nil {
 		return encrypted, err
 	}
+
 	nonce := make([]byte, gcm.NonceSize())
 	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
 		return encrypted, err
 	}
-	ciphertextByte := gcm.Seal(
-		nonce,
-		nonce,
-		plaintext,
-		nil)
-	return ciphertextByte, nil
+
+	return gcm.Seal(nonce, nonce, plaintext, nil), nil
 }
 
 func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
 	if string(encrypted) == "false" {
 		return []byte("false"), nil
 	}
+
 	block, err := aes.NewCipher(passphrase)
 	if err != nil {
 		return encrypted, errors.Wrap(err, "Error creating cypher block")
@@ -120,11 +99,8 @@ func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err err
 	}
 
 	nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
-	plaintextByte, err = gcm.Open(
-		nil,
-		nonce,
-		ciphertextByteClean,
-		nil)
+
+	plaintextByte, err = gcm.Open(nil, nonce, ciphertextByteClean, nil)
 	if err != nil {
 		return encrypted, errors.Wrap(err, "Error decrypting text")
 	}

api/database/boltdb/tx.go

@@ -28,10 +28,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	data := make([]byte, len(value))
-	copy(data, value)
-
-	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
+	return tx.conn.UnmarshalObject(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -49,7 +46,9 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
 	return bucket.Delete(key)
 }
 
-func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
+	var ids []int
+
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	cursor := bucket.Cursor()
@@ -59,11 +58,14 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 			return err
 		}
 
-		if id, ok := matching(obj); ok {
-			err := bucket.Delete(tx.conn.ConvertToKey(id))
-			if err != nil {
-				return err
-			}
+		if id, ok := matchingFn(obj); ok {
+			ids = append(ids, id)
+		}
+	}
+
+	for _, id := range ids {
+		if err := bucket.Delete(tx.conn.ConvertToKey(id)); err != nil {
+			return err
 		}
 	}
 
@@ -72,9 +74,10 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 
 func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
 	bucket := tx.tx.Bucket([]byte(bucketName))
+
 	id, err := bucket.NextSequence()
 	if err != nil {
-		log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifer")
+		log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifier")
 		return 0
 	}
 
@@ -115,54 +118,42 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
 	return bucket.Put(id, data)
 }
 
-func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
+		err := tx.conn.UnmarshalObject(v, obj)
+		if err == nil {
+			obj, err = appendFn(obj)
+		}
+
+		return err
+	})
+}
+
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	return bucket.ForEach(func(k []byte, v []byte) error {
+		err := tx.conn.UnmarshalObject(v, obj)
+		if err == nil {
+			obj, err = appendFn(obj)
+		}
+
+		return err
+	})
+}
+
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
+	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
+
+	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
 		err := tx.conn.UnmarshalObject(v, obj)
 		if err != nil {
 			return err
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
-	bucket := tx.tx.Bucket([]byte(bucketName))
-
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
-		if err != nil {
-			return err
-		}
-
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
-	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
-
-	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
-		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
-		if err != nil {
-			return err
-		}
-
-		obj, err = append(obj)
+		obj, err = appendFn(obj)
 		if err != nil {
 			return err
 		}

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -1,24 +1,22 @@
 package apikeyrepository
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "api_key"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "api_key"
 
 // Service represents a service for managing api-key data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]
 }
 
 // NewService creates a new instance of a service.
@@ -29,22 +27,25 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 // GetAPIKeysByUserID returns a slice containing all the APIKeys a user has access to.
 func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error) {
-	var result = make([]portainer.APIKey, 0)
+	result := make([]portainer.APIKey, 0)
 
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
 			}
 
 			if record.UserID == userID {
@@ -59,19 +60,19 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
 
 // GetAPIKeyByDigest returns the API key for the associated digest.
 // Note: there is a 1-to-1 mapping of api-key and digest
-func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error) {
+func (service *Service) GetAPIKeyByDigest(digest string) (*portainer.APIKey, error) {
 	var k *portainer.APIKey
 	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			key, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
 			}
-			if bytes.Equal(key.Digest, digest) {
+			if key.Digest == digest {
 				k = key
 				return nil, stop
 			}
@@ -90,9 +91,9 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 	return nil, err
 }
 
-// CreateAPIKey creates a new APIKey object.
-func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
-	return service.connection.CreateObject(
+// Create creates a new APIKey object.
+func (service *Service) Create(record *portainer.APIKey) error {
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.APIKeyID(id)
@@ -101,26 +102,3 @@ func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
 		},
 	)
 }
-
-// GetAPIKey retrieves an existing APIKey object by api key ID.
-func (service *Service) GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	var key portainer.APIKey
-	identifier := service.connection.ConvertToKey(int(keyID))
-
-	err := service.connection.GetObject(BucketName, identifier, &key)
-	if err != nil {
-		return nil, err
-	}
-
-	return &key, nil
-}
-
-func (service *Service) UpdateAPIKey(key *portainer.APIKey) error {
-	identifier := service.connection.ConvertToKey(int(key.ID))
-	return service.connection.UpdateObject(BucketName, identifier, key)
-}
-
-func (service *Service) DeleteAPIKey(ID portainer.APIKeyID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/base.go

@@ -0,0 +1,66 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseCRUD[T any, I constraints.Integer] interface {
+	Create(element *T) error
+	Read(ID I) (*T, error)
+	ReadAll() ([]T, error)
+	Update(ID I, element *T) error
+	Delete(ID I) error
+}
+
+type BaseDataService[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+}
+
+func (s *BaseDataService[T, I]) BucketName() string {
+	return s.Bucket
+}
+
+func (service *BaseDataService[T, I]) Tx(tx portainer.Transaction) BaseDataServiceTx[T, I] {
+	return BaseDataServiceTx[T, I]{
+		Bucket:     service.Bucket,
+		Connection: service.Connection,
+		Tx:         tx,
+	}
+}
+
+func (service BaseDataService[T, I]) Read(ID I) (*T, error) {
+	var element *T
+
+	return element, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		element, err = service.Tx(tx).Read(ID)
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		collection, err = service.Tx(tx).ReadAll()
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) Update(ID I, element *T) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Update(ID, element)
+	})
+}
+
+func (service BaseDataService[T, I]) Delete(ID I) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Delete(ID)
+	})
+}

api/dataservices/base_tx.go

@@ -0,0 +1,49 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseDataServiceTx[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+	Tx         portainer.Transaction
+}
+
+func (service BaseDataServiceTx[T, I]) BucketName() string {
+	return service.Bucket
+}
+
+func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
+	var element T
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	err := service.Tx.GetObject(service.Bucket, identifier, &element)
+	if err != nil {
+		return nil, err
+	}
+
+	return &element, nil
+}
+
+func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Tx.GetAllWithJsoniter(
+		service.Bucket,
+		new(T),
+		AppendFn(&collection),
+	)
+}
+
+func (service BaseDataServiceTx[T, I]) Update(ID I, element *T) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.UpdateObject(service.Bucket, identifier, element)
+}
+
+func (service BaseDataServiceTx[T, I]) Delete(ID I) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.DeleteObject(service.Bucket, identifier)
+}

api/dataservices/customtemplate/customtemplate.go

@@ -1,25 +1,16 @@
 package customtemplate
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "customtemplates"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "customtemplates"
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,64 +21,20 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// CustomTemplates return an array containing all the custom templates.
-func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
-	var customTemplates = make([]portainer.CustomTemplate, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.CustomTemplate{},
-		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
-			customTemplate, ok := obj.(*portainer.CustomTemplate)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
-				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
-			}
-			customTemplates = append(customTemplates, *customTemplate)
-
-			return &portainer.CustomTemplate{}, nil
-		})
-
-	return customTemplates, err
-}
-
-// CustomTemplate returns an custom template by ID.
-func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
-	var customTemplate portainer.CustomTemplate
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &customTemplate)
-	if err != nil {
-		return nil, err
-	}
-
-	return &customTemplate, nil
-}
-
-// UpdateCustomTemplate updates an custom template.
-func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, customTemplate)
-}
-
-// DeleteCustomTemplate deletes an custom template.
-func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // CreateCustomTemplate uses the existing id and saves it.
 // TODO: where does the ID come from, and is it safe?
 func (service *Service) Create(customTemplate *portainer.CustomTemplate) error {
-	return service.connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
+	return service.Connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
 }
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgegroup/edgegroup.go

@@ -2,6 +2,7 @@ package edgegroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,7 +10,7 @@ const BucketName = "edgegroups"
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 func (service *Service) BucketName() string {
@@ -24,69 +25,36 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeGroups return a slice containing all the Edge groups.
-func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups []portainer.EdgeGroup
-	var err error
-
-	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-		groups, err = service.Tx(tx).EdgeGroups()
-		return err
-	})
-
-	return groups, err
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group *portainer.EdgeGroup
-	var err error
-
-	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-		group, err = service.Tx(tx).EdgeGroup(ID)
-		return err
-	})
-
-	return group, err
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, group)
-}
-
 // Deprecated: UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	edgeGroup := &portainer.EdgeGroup{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
 		updateFunc(edgeGroup)
 	})
 }
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
-		return service.Tx(tx).DeleteEdgeGroup(ID)
-	})
-}
-
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) Create(group *portainer.EdgeGroup) error {
-	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
 		return service.Tx(tx).Create(group)
 	})
 }

api/dataservices/edgegroup/tx.go

@@ -2,60 +2,13 @@ package edgegroup
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeGroups return a slice containing all the Edge groups.
-func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups = make([]portainer.EdgeGroup, 0)
-
-	err := service.tx.GetAllWithJsoniter(
-		BucketName,
-		&portainer.EdgeGroup{},
-		func(obj interface{}) (interface{}, error) {
-			group, ok := obj.(*portainer.EdgeGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
-				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
-			}
-			groups = append(groups, *group)
-
-			return &portainer.EdgeGroup{}, nil
-		})
-
-	return groups, err
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service ServiceTx) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group portainer.EdgeGroup
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &group)
-	if err != nil {
-		return nil, err
-	}
-
-	return &group, nil
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service ServiceTx) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, group)
+	dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 // UpdateEdgeGroupFunc is a no-op inside a transaction.
@@ -63,14 +16,8 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service ServiceTx) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			group.ID = portainer.EdgeGroupID(id)

api/dataservices/edgejob/edgejob.go

@@ -1,11 +1,8 @@
 package edgejob
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "edgejobs"
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,86 +21,50 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeJobs returns a list of Edge jobs
-func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
-}
-
-// EdgeJob returns an Edge job by ID
-func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &edgeJob)
-	if err != nil {
-		return nil, err
-	}
-
-	return &edgeJob, nil
-}
-
 // Create creates a new EdgeJob
-func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+func (service *Service) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
+}
+
+// CreateWithID creates a new EdgeJob
+func (service *Service) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	edgeJob.ID = ID
 
-	return service.connection.CreateObjectWithId(
+	return service.Connection.CreateObjectWithId(
 		BucketName,
 		int(edgeJob.ID),
 		edgeJob,
 	)
 }
 
-// Deprecated: use UpdateEdgeJobFunc instead
-func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
-}
-
 // UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	edgeJob := &portainer.EdgeJob{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
 		updateFunc(edgeJob)
 	})
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgejob/tx.go

@@ -2,68 +2,25 @@ package edgejob
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeJobs returns a list of Edge jobs
-func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
-}
-
-// EdgeJob returns an Edge job by ID
-func (service ServiceTx) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &edgeJob)
-	if err != nil {
-		return nil, err
-	}
-
-	return &edgeJob, nil
+	dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // Create creates a new EdgeJob
-func (service ServiceTx) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	edgeJob.ID = ID
-
-	return service.tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+func (service ServiceTx) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
 }
 
-// UpdateEdgeJob updates an edge job
-func (service ServiceTx) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, edgeJob)
+// CreateWithID creates a new EdgeJob
+func (service ServiceTx) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.Tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
 }
 
 // UpdateEdgeJobFunc is a no-op inside a transaction.
@@ -71,14 +28,7 @@ func (service ServiceTx) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc fu
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service ServiceTx) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
-	return service.tx.GetNextIdentifier(BucketName)
+	return service.Tx.GetNextIdentifier(BucketName)
 }

api/dataservices/edgestack/edgestack.go

@@ -1,12 +1,10 @@
 package edgestack
 
 import (
-	"fmt"
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -64,22 +62,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.EdgeStack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.EdgeStack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
-				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.EdgeStack{}, nil
-		})
-
-	return stacks, err
+		dataservices.AppendFn(&stacks),
+	)
 }
 
 // EdgeStack returns an Edge stack by ID.

api/dataservices/endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -144,6 +145,23 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
+func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.connection.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -1,9 +1,8 @@
 package endpoint
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -28,6 +27,8 @@ func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 		return nil, err
 	}
 
+	endpoint.LastCheckInDate, _ = service.service.Heartbeat(ID)
+
 	return &endpoint, nil
 }
 
@@ -65,6 +66,7 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 	for edgeID, endpointID := range service.service.idxEdgeID {
 		if endpointID == ID {
 			delete(service.service.idxEdgeID, edgeID)
+
 			break
 		}
 	}
@@ -80,22 +82,11 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.tx.GetAllWithJsoniter(
+	return endpoints, service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Endpoint{},
-		func(obj interface{}) (interface{}, error) {
-			endpoint, ok := obj.(*portainer.Endpoint)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
-				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
-			}
-
-			endpoints = append(endpoints, *endpoint)
-
-			return &portainer.Endpoint{}, nil
-		})
-
-	return endpoints, err
+		dataservices.AppendFn(&endpoints),
+	)
 }
 
 func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {
@@ -131,6 +122,23 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
+func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.tx.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/endpointgroup/endpointgroup.go

@@ -1,25 +1,15 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "endpoint_groups"
-)
+const BucketName = "endpoint_groups"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,67 +20,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &endpointGroup)
-	if err != nil {
-		return nil, err
-	}
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, endpointGroup)
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
-}
-
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service *Service) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointgroup/tx.go

@@ -1,72 +1,17 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service ServiceTx) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &endpointGroup)
-	if err != nil {
-		return nil, err
-	}
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service ServiceTx) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, endpointGroup)
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
+	dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointrelation/endpointrelation.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -54,22 +53,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.connection.GetAll(
+	return all, service.connection.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID

api/dataservices/endpointrelation/tx.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -22,22 +21,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.tx.GetAll(
+	return all, service.tx.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns an Environment(Endpoint) relation object by EndpointID

api/dataservices/extension/extension.go

@@ -1,17 +1,12 @@
 package extension
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "extension"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "extension"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,22 +46,12 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.GetAll(
+	return extensions, service.connection.GetAll(
 		BucketName,
 		&portainer.Extension{},
-		func(obj interface{}) (interface{}, error) {
-			extension, ok := obj.(*portainer.Extension)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
-				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
-			}
+		dataservices.AppendFn(&extensions),
+	)
 
-			extensions = append(extensions, *extension)
-
-			return &portainer.Extension{}, nil
-		})
-
-	return extensions, err
 }
 
 // Persist persists a extension inside the database.

api/dataservices/fdoprofile/fdoprofile.go

@@ -1,25 +1,16 @@
 package fdoprofile
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "fdo_profiles"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "fdo_profiles"
 
 // Service represents a service for managingFDO Profiles data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,66 +21,23 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// FDOProfiles return an array containing all the FDO Profiles.
-func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
-	var fdoProfiles = make([]portainer.FDOProfile, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.FDOProfile{},
-		func(obj interface{}) (interface{}, error) {
-			fdoProfile, ok := obj.(*portainer.FDOProfile)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
-
-				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
-			}
-			fdoProfiles = append(fdoProfiles, *fdoProfile)
-			return &portainer.FDOProfile{}, nil
-		})
-
-	return fdoProfiles, err
-}
-
-// FDOProfile returns an FDO Profile by ID.
-func (service *Service) FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error) {
-	var FDOProfile portainer.FDOProfile
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &FDOProfile)
-	if err != nil {
-		return nil, err
-	}
-
-	return &FDOProfile, nil
-}
-
 // Create assign an ID to a new FDO Profile and saves it.
 func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
-	return service.connection.CreateObjectWithId(
+	return service.Connection.CreateObjectWithId(
 		BucketName,
 		int(FDOProfile.ID),
 		FDOProfile,
 	)
 }
 
-// Update updates an FDO Profile.
-func (service *Service) Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, FDOProfile)
-}
-
-// Delete deletes an FDO Profile.
-func (service *Service) Delete(ID portainer.FDOProfileID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for a FDO Profile.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -1,25 +1,16 @@
 package helmuserrepository
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "helm_user_repository"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "helm_user_repository"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,59 +21,29 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// HelmUserRepository returns an array of all HelmUserRepository
-func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
-	var repos = make([]portainer.HelmUserRepository, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			repos = append(repos, *r)
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return repos, err
-}
-
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
 func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
-	err := service.connection.GetAll(
+	return result, service.Connection.GetAll(
 		BucketName,
 		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			record, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			if record.UserID == userID {
-				result = append(result, *record)
-			}
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return result, err
+		dataservices.FilterFn(&result, func(e portainer.HelmUserRepository) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // CreateHelmUserRepository creates a new HelmUserRepository object.
 func (service *Service) Create(record *portainer.HelmUserRepository) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.HelmUserRepositoryID(id)
@@ -90,15 +51,3 @@ func (service *Service) Create(record *portainer.HelmUserRepository) error {
 		},
 	)
 }
-
-// UpdateHelmUserRepostory updates an registry.
-func (service *Service) UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, registry *portainer.HelmUserRepository) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteHelmUserRepository deletes an registry.
-func (service *Service) DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/helpers.go

@@ -0,0 +1,68 @@
+package dataservices
+
+import (
+	"errors"
+	"fmt"
+
+	perrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+// ErrStop signals the stop of computation when filtering results
+var ErrStop = errors.New("stop")
+
+func IsErrObjectNotFound(e error) bool {
+	return errors.Is(e, perrors.ErrObjectNotFound)
+}
+
+// AppendFn appends elements to the given collection slice
+func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		*collection = append(*collection, *element)
+
+		return new(T), nil
+	}
+}
+
+// FilterFn appends elements to the given collection when the predicate is true
+func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*element) {
+			*collection = append(*collection, *element)
+		}
+
+		return new(T), nil
+	}
+}
+
+// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
+// success
+func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		e, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*e) {
+			*element = *e
+			return new(T), ErrStop
+		}
+
+		return new(T), nil
+	}
+}

api/dataservices/interface.go

@@ -1,13 +1,8 @@
 package dataservices
 
 import (
-	"errors"
-	"io"
-	"time"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
 type (
@@ -37,10 +32,11 @@ type (
 		User() UserService
 		Version() VersionService
 		Webhook() WebhookService
+		PendingActions() PendingActionsService
 	}
 
-	// DataStore defines the interface to manage the data
 	DataStore interface {
+		Connection() portainer.Connection
 		Open() (newStore bool, err error)
 		Init() error
 		Close() error
@@ -49,7 +45,7 @@ type (
 		MigrateData() error
 		Rollback(force bool) error
 		CheckCurrentEdition() error
-		BackupTo(w io.Writer) error
+		Backup(path string) (string, error)
 		Export(filename string) (err error)
 
 		DataStoreTx
@@ -57,36 +53,28 @@ type (
 
 	// CustomTemplateService represents a service to manage custom templates
 	CustomTemplateService interface {
+		BaseCRUD[portainer.CustomTemplate, portainer.CustomTemplateID]
 		GetNextIdentifier() int
-		CustomTemplates() ([]portainer.CustomTemplate, error)
-		CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error)
-		Create(customTemplate *portainer.CustomTemplate) error
-		UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error
-		DeleteCustomTemplate(ID portainer.CustomTemplateID) error
-		BucketName() string
 	}
 
 	// EdgeGroupService represents a service to manage Edge groups
 	EdgeGroupService interface {
-		EdgeGroups() ([]portainer.EdgeGroup, error)
-		EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error)
-		Create(group *portainer.EdgeGroup) error
-		UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error
+		BaseCRUD[portainer.EdgeGroup, portainer.EdgeGroupID]
 		UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(group *portainer.EdgeGroup)) error
-		DeleteEdgeGroup(ID portainer.EdgeGroupID) error
-		BucketName() string
 	}
 
 	// EdgeJobService represents a service to manage Edge jobs
 	EdgeJobService interface {
-		EdgeJobs() ([]portainer.EdgeJob, error)
-		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
-		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
-		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		BaseCRUD[portainer.EdgeJob, portainer.EdgeJobID]
+		CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
-		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
-		BucketName() string
+	}
+
+	PendingActionsService interface {
+		BaseCRUD[portainer.PendingAction, portainer.PendingActionID]
+		GetNextIdentifier() int
+		DeleteByEndpointID(ID portainer.EndpointID) error
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks
@@ -95,7 +83,6 @@ type (
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
 		EdgeStackVersion(ID portainer.EdgeStackID) (int, bool)
 		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
-		// Deprecated: Use UpdateEdgeStackFunc instead.
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
@@ -107,6 +94,7 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
+		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)
@@ -119,12 +107,7 @@ type (
 
 	// EndpointGroupService represents a service for managing environment(endpoint) group data
 	EndpointGroupService interface {
-		EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error)
-		EndpointGroups() ([]portainer.EndpointGroup, error)
-		Create(group *portainer.EndpointGroup) error
-		UpdateEndpointGroup(ID portainer.EndpointGroupID, group *portainer.EndpointGroup) error
-		DeleteEndpointGroup(ID portainer.EndpointGroupID) error
-		BucketName() string
+		BaseCRUD[portainer.EndpointGroup, portainer.EndpointGroupID]
 	}
 
 	// EndpointRelationService represents a service for managing environment(endpoint) relations data
@@ -139,72 +122,37 @@ type (
 
 	// FDOProfileService represents a service to manage FDO Profiles
 	FDOProfileService interface {
-		FDOProfiles() ([]portainer.FDOProfile, error)
-		FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error)
-		Create(FDOProfile *portainer.FDOProfile) error
-		Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error
-		Delete(ID portainer.FDOProfileID) error
+		BaseCRUD[portainer.FDOProfile, portainer.FDOProfileID]
 		GetNextIdentifier() int
-		BucketName() string
 	}
 
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
+		BaseCRUD[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
-		Create(record *portainer.HelmUserRepository) error
-		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
-		DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error
-		BucketName() string
-	}
-
-	// JWTService represents a service for managing JWT tokens
-	JWTService interface {
-		GenerateToken(data *portainer.TokenData) (string, error)
-		GenerateTokenForOAuth(data *portainer.TokenData, expiryTime *time.Time) (string, error)
-		GenerateTokenForKubeconfig(data *portainer.TokenData) (string, error)
-		ParseAndVerifyToken(token string) (*portainer.TokenData, error)
-		SetUserSessionDuration(userSessionDuration time.Duration)
 	}
 
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
-		Registry(ID portainer.RegistryID) (*portainer.Registry, error)
-		Registries() ([]portainer.Registry, error)
-		Create(registry *portainer.Registry) error
-		UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error
-		DeleteRegistry(ID portainer.RegistryID) error
-		BucketName() string
+		BaseCRUD[portainer.Registry, portainer.RegistryID]
 	}
 
 	// ResourceControlService represents a service for managing resource control data
 	ResourceControlService interface {
-		ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error)
+		BaseCRUD[portainer.ResourceControl, portainer.ResourceControlID]
 		ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error)
-		ResourceControls() ([]portainer.ResourceControl, error)
-		Create(rc *portainer.ResourceControl) error
-		UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error
-		DeleteResourceControl(ID portainer.ResourceControlID) error
-		BucketName() string
 	}
 
 	// RoleService represents a service for managing user roles
 	RoleService interface {
-		Role(ID portainer.RoleID) (*portainer.Role, error)
-		Roles() ([]portainer.Role, error)
-		Create(role *portainer.Role) error
-		UpdateRole(ID portainer.RoleID, role *portainer.Role) error
-		BucketName() string
+		BaseCRUD[portainer.Role, portainer.RoleID]
 	}
 
 	// APIKeyRepositoryService
 	APIKeyRepository interface {
-		CreateAPIKey(key *portainer.APIKey) error
-		GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error)
-		UpdateAPIKey(key *portainer.APIKey) error
-		DeleteAPIKey(ID portainer.APIKeyID) error
+		BaseCRUD[portainer.APIKey, portainer.APIKeyID]
 		GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error)
-		GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error)
+		GetAPIKeyByDigest(digest string) (*portainer.APIKey, error)
 	}
 
 	// SettingsService represents a service for managing application settings
@@ -215,12 +163,7 @@ type (
 	}
 
 	SnapshotService interface {
-		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
-		Snapshots() ([]portainer.Snapshot, error)
-		UpdateSnapshot(snapshot *portainer.Snapshot) error
-		DeleteSnapshot(endpointID portainer.EndpointID) error
-		Create(snapshot *portainer.Snapshot) error
-		BucketName() string
+		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
 	}
 
 	// SSLSettingsService represents a service for managing application settings
@@ -232,53 +175,33 @@ type (
 
 	// StackService represents a service for managing stack data
 	StackService interface {
-		Stack(ID portainer.StackID) (*portainer.Stack, error)
+		BaseCRUD[portainer.Stack, portainer.StackID]
 		StackByName(name string) (*portainer.Stack, error)
 		StacksByName(name string) ([]portainer.Stack, error)
-		Stacks() ([]portainer.Stack, error)
-		Create(stack *portainer.Stack) error
-		UpdateStack(ID portainer.StackID, stack *portainer.Stack) error
-		DeleteStack(ID portainer.StackID) error
 		GetNextIdentifier() int
 		StackByWebhookID(ID string) (*portainer.Stack, error)
 		RefreshableStacks() ([]portainer.Stack, error)
-		BucketName() string
 	}
 
 	// TagService represents a service for managing tag data
 	TagService interface {
-		Tags() ([]portainer.Tag, error)
-		Tag(ID portainer.TagID) (*portainer.Tag, error)
-		Create(tag *portainer.Tag) error
-		UpdateTag(ID portainer.TagID, tag *portainer.Tag) error
+		BaseCRUD[portainer.Tag, portainer.TagID]
 		UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error
-		DeleteTag(ID portainer.TagID) error
-		BucketName() string
 	}
 
 	// TeamService represents a service for managing user data
 	TeamService interface {
-		Team(ID portainer.TeamID) (*portainer.Team, error)
+		BaseCRUD[portainer.Team, portainer.TeamID]
 		TeamByName(name string) (*portainer.Team, error)
-		Teams() ([]portainer.Team, error)
-		Create(team *portainer.Team) error
-		UpdateTeam(ID portainer.TeamID, team *portainer.Team) error
-		DeleteTeam(ID portainer.TeamID) error
-		BucketName() string
 	}
 
 	// TeamMembershipService represents a service for managing team membership data
 	TeamMembershipService interface {
-		TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error)
-		TeamMemberships() ([]portainer.TeamMembership, error)
+		BaseCRUD[portainer.TeamMembership, portainer.TeamMembershipID]
 		TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error)
 		TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error)
-		Create(membership *portainer.TeamMembership) error
-		UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error
-		DeleteTeamMembership(ID portainer.TeamMembershipID) error
 		DeleteTeamMembershipByUserID(userID portainer.UserID) error
 		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
-		BucketName() string
 		DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error
 	}
 
@@ -291,38 +214,24 @@ type (
 
 	// UserService represents a service for managing user data
 	UserService interface {
-		User(ID portainer.UserID) (*portainer.User, error)
+		BaseCRUD[portainer.User, portainer.UserID]
 		UserByUsername(username string) (*portainer.User, error)
-		Users() ([]portainer.User, error)
 		UsersByRole(role portainer.UserRole) ([]portainer.User, error)
-		Create(user *portainer.User) error
-		UpdateUser(ID portainer.UserID, user *portainer.User) error
-		DeleteUser(ID portainer.UserID) error
-		BucketName() string
 	}
 
 	// VersionService represents a service for managing version data
 	VersionService interface {
-		Edition() (portainer.SoftwareEdition, error)
 		InstanceID() (string, error)
 		UpdateInstanceID(ID string) error
+		Edition() (portainer.SoftwareEdition, error)
 		Version() (*models.Version, error)
 		UpdateVersion(*models.Version) error
 	}
 
 	// WebhookService represents a service for managing webhook data.
 	WebhookService interface {
-		Webhooks() ([]portainer.Webhook, error)
-		Webhook(ID portainer.WebhookID) (*portainer.Webhook, error)
-		Create(portainer *portainer.Webhook) error
-		UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error
+		BaseCRUD[portainer.Webhook, portainer.WebhookID]
 		WebhookByResourceID(resourceID string) (*portainer.Webhook, error)
 		WebhookByToken(token string) (*portainer.Webhook, error)
-		DeleteWebhook(ID portainer.WebhookID) error
-		BucketName() string
 	}
 )
-
-func IsErrObjectNotFound(e error) bool {
-	return errors.Is(e, dserrors.ErrObjectNotFound)
-}

api/dataservices/pendingactions/pendingactions.go

@@ -0,0 +1,105 @@
+package pendingactions
+
+import (
+	"fmt"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	BucketName = "pending_actions"
+)
+
+type Service struct {
+	dataservices.BaseDataService[portainer.PendingAction, portainer.PendingActionID]
+}
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.PendingAction, portainer.PendingActionID]
+}
+
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		BaseDataService: dataservices.BaseDataService[portainer.PendingAction, portainer.PendingActionID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
+	}, nil
+}
+
+func (s Service) Create(config *portainer.PendingAction) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Create(config)
+	})
+}
+
+func (s Service) Update(ID portainer.PendingActionID, config *portainer.PendingAction) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Update(ID, config)
+	})
+}
+
+func (s Service) DeleteByEndpointID(ID portainer.EndpointID) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).DeleteByEndpointID(ID)
+	})
+}
+
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingAction, portainer.PendingActionID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
+	}
+}
+
+func (s ServiceTx) Create(config *portainer.PendingAction) error {
+	return s.Tx.CreateObject(BucketName, func(id uint64) (int, interface{}) {
+		config.ID = portainer.PendingActionID(id)
+		config.CreatedAt = time.Now().Unix()
+
+		return int(config.ID), config
+	})
+}
+
+func (s ServiceTx) Update(ID portainer.PendingActionID, config *portainer.PendingAction) error {
+	return s.BaseDataServiceTx.Update(ID, config)
+}
+
+func (s ServiceTx) DeleteByEndpointID(ID portainer.EndpointID) error {
+	log.Debug().Int("endpointId", int(ID)).Msg("deleting pending actions for endpoint")
+	pendingActions, err := s.BaseDataServiceTx.ReadAll()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve pending-actions for endpoint (%d): %w", ID, err)
+	}
+
+	for _, pendingAction := range pendingActions {
+		if pendingAction.EndpointID == ID {
+			err := s.BaseDataServiceTx.Delete(pendingAction.ID)
+			if err != nil {
+				log.Debug().Int("endpointId", int(ID)).Msgf("failed to delete pending action: %v", err)
+			}
+		}
+	}
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
+}

api/dataservices/registry/registry.go

@@ -1,11 +1,8 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "registries"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,55 +21,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Registry returns a registry by ID.
-func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &registry)
-	if err != nil {
-		return nil, err
-	}
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service *Service) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
-}
-
 // Create creates a new registry.
 func (service *Service) Create(registry *portainer.Registry) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -84,15 +48,3 @@ func (service *Service) Create(registry *portainer.Registry) error {
 		},
 	)
 }
-
-// UpdateRegistry updates a registry.
-func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteRegistry deletes a registry.
-func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/registry/tx.go

@@ -1,59 +1,17 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Registry returns a registry by ID.
-func (service ServiceTx) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &registry)
-	if err != nil {
-		return nil, err
-	}
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service ServiceTx) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
+	dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]
 }
 
 // Create creates a new registry.
 func (service ServiceTx) Create(registry *portainer.Registry) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -61,15 +19,3 @@ func (service ServiceTx) Create(registry *portainer.Registry) error {
 		},
 	)
 }
-
-// UpdateRegistry updates a registry.
-func (service ServiceTx) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteRegistry deletes a registry.
-func (service ServiceTx) DeleteRegistry(ID portainer.RegistryID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
@@ -14,11 +15,7 @@ const BucketName = "resource_control"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // NewService creates a new instance of a service.
@@ -29,44 +26,37 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// ResourceControl returns a ResourceControl object by ID
-func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &resourceControl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &resourceControl, nil
-}
-
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
 // to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
 // if no ResourceControl was found.
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -90,31 +80,9 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 	return nil, err
 }
 
-// ResourceControls returns all the ResourceControl objects
-func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
-}
-
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -122,15 +90,3 @@ func (service *Service) Create(resourceControl *portainer.ResourceControl) error
 		},
 	)
 }
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, resourceControl)
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/resourcecontrol/tx.go

@@ -5,30 +5,13 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// ResourceControl returns a ResourceControl object by ID
-func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &resourceControl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &resourceControl, nil
+	dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
@@ -37,14 +20,14 @@ func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*porta
 func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.tx.GetAll(
+	err := service.Tx.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -68,31 +51,9 @@ func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, r
 	return nil, err
 }
 
-// ResourceControls returns all the ResourceControl objects
-func (service ServiceTx) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
-}
-
 // CreateResourceControl creates a new ResourceControl object
 func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -100,15 +61,3 @@ func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) erro
 		},
 	)
 }
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service ServiceTx) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, resourceControl)
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service ServiceTx) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/role/role.go

@@ -1,11 +1,8 @@
 package role
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "roles"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Role, portainer.RoleID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,55 +21,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Role returns a Role by ID
-func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &set)
-	if err != nil {
-		return nil, err
-	}
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service *Service) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Role{},
-		func(obj interface{}) (interface{}, error) {
-			set, ok := obj.(*portainer.Role)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-				return nil, fmt.Errorf("failed to convert to Role object: %s", obj)
-			}
-
-			sets = append(sets, *set)
-
-			return &portainer.Role{}, nil
-		})
-
-	return sets, err
-}
-
 // CreateRole creates a new Role.
 func (service *Service) Create(role *portainer.Role) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			role.ID = portainer.RoleID(id)
@@ -84,9 +48,3 @@ func (service *Service) Create(role *portainer.Role) error {
 		},
 	)
 }
-
-// UpdateRole updates a role.
-func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, role)
-}

api/dataservices/role/tx.go

@@ -1,61 +1,17 @@
 package role
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-// Service represents a service for managing environment(endpoint) data.
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Role returns a Role by ID
-func (service ServiceTx) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &set)
-	if err != nil {
-		return nil, err
-	}
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service ServiceTx) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Role{},
-		func(obj interface{}) (interface{}, error) {
-			set, ok := obj.(*portainer.Role)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-				return nil, fmt.Errorf("failed to convert to Role object: %s", obj)
-			}
-
-			sets = append(sets, *set)
-
-			return &portainer.Role{}, nil
-		})
-
-	return sets, err
+	dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]
 }
 
 // CreateRole creates a new Role.
 func (service ServiceTx) Create(role *portainer.Role) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			role.ID = portainer.RoleID(id)
@@ -63,9 +19,3 @@ func (service ServiceTx) Create(role *portainer.Role) error {
 		},
 	)
 }
-
-// UpdateRole updates a role.
-func (service ServiceTx) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, role)
-}

api/dataservices/schedule/schedule.go

@@ -1,17 +1,12 @@
 package schedule
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "schedules"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "schedules"
 
 // Service represents a service for managing schedule data.
 type Service struct {
@@ -63,22 +58,11 @@ func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			schedules = append(schedules, *schedule)
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.AppendFn(&schedules),
+	)
 }
 
 // SchedulesByJobType return a array containing all the schedules
@@ -86,24 +70,13 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			if schedule.JobType == jobType {
-				schedules = append(schedules, *schedule)
-			}
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.FilterFn(&schedules, func(e portainer.Schedule) bool {
+			return e.JobType == jobType
+		}),
+	)
 }
 
 // Create assign an ID to a new schedule and saves it.

api/dataservices/snapshot/snapshot.go

@@ -1,23 +1,14 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	BucketName = "snapshots"
-)
+const BucketName = "snapshots"
 
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]
 }
 
 func NewService(connection portainer.Connection) (*Service, error) {
@@ -27,58 +18,23 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-	identifier := service.connection.ConvertToKey(int(endpointID))
-
-	err := service.connection.GetObject(BucketName, identifier, &snapshot)
-	if err != nil {
-		return nil, err
-	}
-
-	return &snapshot, nil
-}
-
-func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	err := service.connection.GetAllWithJsoniter(
-		BucketName,
-		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
-}
-
-func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
-	return service.connection.UpdateObject(BucketName, identifier, snapshot)
-}
-
-func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	identifier := service.connection.ConvertToKey(int(endpointID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 func (service *Service) Create(snapshot *portainer.Snapshot) error {
-	return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/snapshot/tx.go

@@ -1,63 +1,14 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-	identifier := service.service.connection.ConvertToKey(int(endpointID))
-
-	err := service.tx.GetObject(BucketName, identifier, &snapshot)
-	if err != nil {
-		return nil, err
-	}
-
-	return &snapshot, nil
-}
-
-func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	err := service.tx.GetAllWithJsoniter(
-		BucketName,
-		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
-}
-
-func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	identifier := service.service.connection.ConvertToKey(int(snapshot.EndpointID))
-	return service.tx.UpdateObject(BucketName, identifier, snapshot)
-}
-
-func (service ServiceTx) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	identifier := service.service.connection.ConvertToKey(int(endpointID))
-	return service.tx.DeleteObject(BucketName, identifier)
+	dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]
 }
 
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
-	return service.tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/stack/stack.go

@@ -2,27 +2,19 @@ package stack
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "stacks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "stacks"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Stack, portainer.StackID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,35 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Stack, portainer.StackID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Stack returns a stack object by ID.
-func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
-	var stack portainer.Stack
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &stack)
-	if err != nil {
-		return nil, err
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: service.BaseDataService.Tx(tx),
 	}
-
-	return &stack, nil
 }
 
 // StackByName returns a stack object by name.
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
-	var s *portainer.Stack
+	var s portainer.Stack
 
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 
-			if stack.Name == name {
-				s = stack
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if errors.Is(err, stop) {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
@@ -86,125 +65,58 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.Name == name {
-				stacks = append(stacks, stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
-}
-
-// Stacks returns an array containing all the stacks.
-func (service *Service) Stacks() ([]portainer.Stack, error) {
-	var stacks = make([]portainer.Stack, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 }
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) Create(stack *portainer.Stack) error {
-	return service.connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
-}
-
-// UpdateStack updates a stack.
-func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, stack)
-}
-
-// DeleteStack deletes a stack.
-func (service *Service) DeleteStack(ID portainer.StackID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
+	return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
 }
 
 // StackByWebhookID returns a pointer to a stack object by webhook ID.
 // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
 func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	var s *portainer.Stack
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var s portainer.Stack
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			var ok bool
-			s, ok = obj.(*portainer.Stack)
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
 
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-
-				return &portainer.Stack{}, nil
-			}
-
-			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if errors.Is(err, stop) {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
 
 	return nil, err
-
 }
 
 // RefreshableStacks returns stacks that are configured for a periodic update
 func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 	stacks := make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
-				stacks = append(stacks, *stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
 }

api/dataservices/stack/tx.go

@@ -0,0 +1,98 @@
+package stack
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.Stack, portainer.StackID]
+}
+
+// StackByName returns a stack object by name.
+func (service ServiceTx) StackByName(name string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// Stacks returns an array containing all the stacks with same name
+func (service ServiceTx) StacksByName(name string) ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+}
+
+// GetNextIdentifier returns the next identifier for a stack.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
+// CreateStack creates a new stack.
+func (service ServiceTx) Create(stack *portainer.Stack) error {
+	return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack)
+}
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
+}

api/dataservices/tag/tag.go

@@ -1,25 +1,16 @@
 package tag
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "tags"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "tags"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Tag, portainer.TagID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,55 +21,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Tags return an array containing all the tags.
-func (service *Service) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
-}
-
-// Tag returns a tag by ID.
-func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &tag)
-	if err != nil {
-		return nil, err
-	}
-
-	return &tag, nil
-}
-
 // CreateTag creates a new tag.
 func (service *Service) Create(tag *portainer.Tag) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -87,24 +49,12 @@ func (service *Service) Create(tag *portainer.Tag) error {
 	)
 }
 
-// Deprecated: Use UpdateTagFunc instead.
-func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, tag)
-}
-
 // UpdateTagFunc updates a tag inside a transaction avoiding data races.
 func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	tag := &portainer.Tag{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, tag, func() {
 		updateFunc(tag)
 	})
 }
-
-// DeleteTag deletes a tag.
-func (service *Service) DeleteTag(ID portainer.TagID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/tag/tx.go

@@ -2,60 +2,18 @@ package tag
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Tags return an array containing all the tags.
-func (service ServiceTx) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
-}
-
-// Tag returns a tag by ID.
-func (service ServiceTx) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &tag)
-	if err != nil {
-		return nil, err
-	}
-
-	return &tag, nil
+	dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]
 }
 
 // CreateTag creates a new tag.
 func (service ServiceTx) Create(tag *portainer.Tag) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -64,19 +22,7 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTag updates a tag
-func (service ServiceTx) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, tag)
-}
-
-// UpdateTagFunc is a no-op inside a transaction
+// UpdateTagFunc is a no-op inside a transaction.
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }
-
-// DeleteTag deletes a tag.
-func (service ServiceTx) DeleteTag(ID portainer.TagID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/team/team.go

@@ -2,27 +2,19 @@ package team
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "teams"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "teams"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Team, portainer.TeamID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,29 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Team, portainer.TeamID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Team returns a Team by ID
-func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
-	var team portainer.Team
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &team)
-	if err != nil {
-		return nil, err
-	}
-
-	return &team, nil
-}
-
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
-	var t *portainer.Team
+	var t portainer.Team
 
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
+		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+			return strings.EqualFold(e.Name, name)
+		}),
+	)
 
-			if strings.EqualFold(team.Name, name) {
-				t = team
-				return nil, stop
-			}
-
-			return &portainer.Team{}, nil
-		})
-	if errors.Is(err, stop) {
-		return t, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &t, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
@@ -82,37 +55,9 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	return nil, err
 }
 
-// Teams return an array containing all the teams.
-func (service *Service) Teams() ([]portainer.Team, error) {
-	var teams = make([]portainer.Team, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
-
-			teams = append(teams, *team)
-
-			return &portainer.Team{}, nil
-		})
-
-	return teams, err
-}
-
-// UpdateTeam saves a Team.
-func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, team)
-}
-
 // CreateTeam creates a new Team.
 func (service *Service) Create(team *portainer.Team) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			team.ID = portainer.TeamID(id)
@@ -120,9 +65,3 @@ func (service *Service) Create(team *portainer.Team) error {
 		},
 	)
 }
-
-// DeleteTeam deletes a Team.
-func (service *Service) DeleteTeam(ID portainer.TeamID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/teammembership/teammembership.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
@@ -13,11 +14,7 @@ const BucketName = "team_membership"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,109 +25,52 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// TeamMembership returns a TeamMembership object by ID
-func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var membership portainer.TeamMembership
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &membership)
-	if err != nil {
-		return nil, err
-	}
-
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			memberships = append(memberships, *membership)
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.Connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.UserID == userID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.Connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.TeamID == teamID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, membership)
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) Create(membership *portainer.TeamMembership) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			membership.ID = portainer.TeamMembershipID(id)
@@ -139,15 +79,9 @@ func (service *Service) Create(membership *portainer.TeamMembership) error {
 	)
 }
 
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -168,7 +102,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -188,7 +122,7 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 }
 
 func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/teammembership/tx.go

@@ -4,111 +4,44 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// TeamMembership returns a TeamMembership object by ID
-func (service ServiceTx) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var membership portainer.TeamMembership
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &membership)
-	if err != nil {
-		return nil, err
-	}
-
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service ServiceTx) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			memberships = append(memberships, *membership)
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+	dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service ServiceTx) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.tx.GetAll(
+	return memberships, service.Tx.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.UserID == userID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.tx.GetAll(
+	return memberships, service.Tx.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.TeamID == teamID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service ServiceTx) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, membership)
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			membership.ID = portainer.TeamMembershipID(id)
@@ -117,15 +50,9 @@ func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
 	)
 }
 
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service ServiceTx) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -146,7 +73,7 @@ func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) e
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -166,7 +93,7 @@ func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) e
 }
 
 func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/user/tx.go

@@ -0,0 +1,63 @@
+package user
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]
+}
+
+// UserByUsername returns a user by username.
+func (service ServiceTx) UserByUsername(username string) (*portainer.User, error) {
+	var u portainer.User
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// UsersByRole return an array containing all the users with the specified role.
+func (service ServiceTx) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+
+	return users, service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
+}
+
+// CreateUser creates a new user.
+func (service ServiceTx) Create(user *portainer.User) error {
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			user.ID = portainer.UserID(id)
+			user.Username = strings.ToLower(user.Username)
+
+			return int(user.ID), user
+		},
+	)
+}

api/dataservices/user/user.go

@@ -2,27 +2,19 @@ package user
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "users"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "users"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.User, portainer.UserID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,37 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// User returns a user by ID
-func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
-	var user portainer.User
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &user)
-	if err != nil {
-		return nil, err
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
-
-	return &user, nil
 }
 
 // UserByUsername returns a user by username.
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
-	var u *portainer.User
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var u portainer.User
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if strings.EqualFold(user.Username, username) {
-				u = user
-				return nil, stop
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return u, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
 	}
 
 	if err == nil {
@@ -84,64 +65,22 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 	return nil, err
 }
 
-// Users return an array containing all the users.
-func (service *Service) Users() ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			users = append(users, *user)
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
-}
-
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.GetAll(
+	return users, service.Connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if user.Role == role {
-				users = append(users, *user)
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
-}
-
-// UpdateUser saves a user.
-func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	user.Username = strings.ToLower(user.Username)
-	return service.connection.UpdateObject(BucketName, identifier, user)
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
 }
 
 // CreateUser creates a new user.
 func (service *Service) Create(user *portainer.User) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			user.ID = portainer.UserID(id)
@@ -151,9 +90,3 @@ func (service *Service) Create(user *portainer.User) error {
 		},
 	)
 }
-
-// DeleteUser deletes a user.
-func (service *Service) DeleteUser(ID portainer.UserID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/version/version.go

@@ -73,6 +73,10 @@ func (service *Service) IsUpdating() (bool, error) {
 
 // StoreIsUpdating store the database updating status.
 func (service *Service) StoreIsUpdating(isUpdating bool) error {
+	if isUpdating {
+		return service.connection.UpdateObject(BucketName, []byte(updatingKey), isUpdating)
+	}
+
 	return service.connection.DeleteObject(BucketName, []byte(updatingKey))
 }
 

api/dataservices/webhook/webhook.go

@@ -2,26 +2,18 @@ package webhook
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "webhooks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "webhooks"
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]
 }
 
 // NewService creates a new instance of a service.
@@ -32,70 +24,27 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Webhooks returns an array of all webhooks
-func (service *Service) Webhooks() ([]portainer.Webhook, error) {
-	var webhooks = make([]portainer.Webhook, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			webhooks = append(webhooks, *webhook)
-
-			return &portainer.Webhook{}, nil
-		})
-
-	return webhooks, err
-}
-
-// Webhook returns a webhook by ID.
-func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
-	var webhook portainer.Webhook
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &webhook)
-	if err != nil {
-		return nil, err
-	}
-
-	return &webhook, nil
-}
-
 // WebhookByResourceID returns a webhook by the ResourceID it is associated with.
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var w portainer.Webhook
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.ResourceID == ID
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.ResourceID == ID {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
@@ -107,29 +56,18 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 
 // WebhookByToken returns a webhook by the random token it is associated with.
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var w portainer.Webhook
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.Token == token
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.Token == token {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
@@ -139,15 +77,9 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 	return nil, err
 }
 
-// DeleteWebhook deletes a webhook.
-func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) Create(webhook *portainer.Webhook) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			webhook.ID = portainer.WebhookID(id)
@@ -155,9 +87,3 @@ func (service *Service) Create(webhook *portainer.Webhook) error {
 		},
 	)
 }
-
-// UpdateWebhook update a webhook.
-func (service *Service) UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, webhook)
-}

api/datastore/backup.go

@@ -4,186 +4,89 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"time"
 
-	"github.com/portainer/portainer/api/database/models"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/rs/zerolog/log"
 )
 
-var backupDefaults = struct {
-	backupDir string
-	commonDir string
-}{
-	"backups",
-	"common",
+// Backup takes an optional output path and creates a backup of the database.
+// The database connection is stopped before running the backup to avoid any
+// corruption and if a path is not given a default is used.
+// The path or an error are returned.
+func (store *Store) Backup(path string) (string, error) {
+	if err := store.createBackupPath(); err != nil {
+		return "", err
+	}
+
+	backupFilename := store.backupFilename()
+	if path != "" {
+		backupFilename = path
+	}
+	log.Info().Str("from", store.connection.GetDatabaseFilePath()).Str("to", backupFilename).Msgf("Backing up database")
+
+	// Close the store before backing up
+	err := store.Close()
+	if err != nil {
+		return "", fmt.Errorf("failed to close store before backup: %w", err)
+	}
+
+	err = store.fileService.Copy(store.connection.GetDatabaseFilePath(), backupFilename, true)
+	if err != nil {
+		return "", fmt.Errorf("failed to create backup file: %w", err)
+	}
+
+	// reopen the store
+	_, err = store.Open()
+	if err != nil {
+		return "", fmt.Errorf("failed to reopen store after backup: %w", err)
+	}
+
+	return backupFilename, nil
 }
 
-//
-// Backup Helpers
-//
+func (store *Store) Restore() error {
+	backupFilename := store.backupFilename()
+	return store.RestoreFromFile(backupFilename)
+}
 
-// createBackupFolders create initial folders for backups
-func (store *Store) createBackupFolders() {
-	// create common dir
-	commonDir := store.commonBackupDir()
-	if exists, _ := store.fileService.FileExists(commonDir); !exists {
-		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			log.Error().Err(err).Msg("error while creating common backup folder")
+func (store *Store) RestoreFromFile(backupFilename string) error {
+	store.Close()
+	if err := store.fileService.Copy(backupFilename, store.connection.GetDatabaseFilePath(), true); err != nil {
+		return fmt.Errorf("unable to restore backup file %q. err: %w", backupFilename, err)
+	}
+
+	log.Info().Str("from", backupFilename).Str("to", store.connection.GetDatabaseFilePath()).Msgf("database restored")
+
+	_, err := store.Open()
+	if err != nil {
+		return fmt.Errorf("unable to determine version of restored portainer backup file: %w", err)
+	}
+
+	// determine the db version
+	version, err := store.VersionService.Version()
+	if err != nil {
+		return fmt.Errorf("unable to determine restored database version. err: %w", err)
+	}
+
+	editionLabel := portainer.SoftwareEdition(version.Edition).GetEditionLabel()
+	log.Info().Msgf("Restored database version: Portainer %s %s", editionLabel, version.SchemaVersion)
+	return nil
+}
+
+func (store *Store) createBackupPath() error {
+	backupDir := path.Join(store.connection.GetStorePath(), "backups")
+	if exists, _ := store.fileService.FileExists(backupDir); !exists {
+		if err := os.MkdirAll(backupDir, 0o700); err != nil {
+			return fmt.Errorf("unable to create backup folder: %w", err)
 		}
 	}
+	return nil
+}
+
+func (store *Store) backupFilename() string {
+	return path.Join(store.connection.GetStorePath(), "backups", store.connection.GetDatabaseFileName()+".bak")
 }
 
 func (store *Store) databasePath() string {
 	return store.connection.GetDatabaseFilePath()
 }
-
-func (store *Store) commonBackupDir() string {
-	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
-}
-
-func (store *Store) copyDBFile(from string, to string) error {
-	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
-
-	err := store.fileService.Copy(from, to, true)
-	if err != nil {
-		log.Error().Err(err).Msg("failed")
-	}
-
-	return err
-}
-
-// BackupOptions provide a helper to inject backup options
-type BackupOptions struct {
-	Version        string
-	BackupDir      string
-	BackupFileName string
-	BackupPath     string
-}
-
-// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// - db backup prior to version upgrade
-// - db rollback
-func getBackupRestoreOptions(backupDir string) *BackupOptions {
-	return &BackupOptions{
-		BackupDir:      backupDir, //connection.commonBackupDir(),
-		BackupFileName: beforePortainerVersionUpgradeBackup,
-	}
-}
-
-// Backup current database with default options
-func (store *Store) Backup(version *models.Version) (string, error) {
-	if version == nil {
-		return store.backupWithOptions(nil)
-	}
-
-	return store.backupWithOptions(&BackupOptions{
-		Version: version.SchemaVersion,
-	})
-}
-
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-	if options == nil {
-		options = &BackupOptions{}
-	}
-	if options.Version == "" {
-		v, err := store.VersionService.Version()
-		if err != nil {
-			options.Version = ""
-		}
-		options.Version = v.SchemaVersion
-	}
-	if options.BackupDir == "" {
-		options.BackupDir = store.commonBackupDir()
-	}
-	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
-	}
-	if options.BackupPath == "" {
-		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-	}
-	return options
-}
-
-// BackupWithOptions backup current database with options
-func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
-	log.Info().Msg("creating DB backup")
-
-	store.createBackupFolders()
-
-	options = store.setupOptions(options)
-	dbPath := store.databasePath()
-
-	if err := store.Close(); err != nil {
-		return options.BackupPath, fmt.Errorf(
-			"error closing datastore before creating backup: %w",
-			err,
-		)
-	}
-
-	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
-		return options.BackupPath, err
-	}
-
-	if _, err := store.Open(); err != nil {
-		return options.BackupPath, fmt.Errorf(
-			"error opening datastore after creating backup: %w",
-			err,
-		)
-	}
-
-	return options.BackupPath, nil
-}
-
-// RestoreWithOptions previously saved backup for the current Edition  with options
-// Restore strategies:
-// - default: restore latest from current edition
-// - restore a specific
-func (store *Store) restoreWithOptions(options *BackupOptions) error {
-	options = store.setupOptions(options)
-
-	// Check if backup file exist before restoring
-	_, err := os.Stat(options.BackupPath)
-	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
-
-		return err
-	}
-
-	err = store.Close()
-	if err != nil {
-		log.Error().Err(err).Msg("error while closing store before restore")
-
-		return err
-	}
-
-	log.Info().Msg("restoring DB backup")
-	err = store.copyDBFile(options.BackupPath, store.databasePath())
-	if err != nil {
-		return err
-	}
-
-	_, err = store.Open()
-	return err
-}
-
-// RemoveWithOptions removes backup database based on supplied options
-func (store *Store) removeWithOptions(options *BackupOptions) error {
-	log.Info().Msg("removing DB backup")
-
-	options = store.setupOptions(options)
-	_, err := os.Stat(options.BackupPath)
-
-	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
-		return err
-	}
-
-	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
-	err = os.Remove(options.BackupPath)
-	if err != nil {
-		log.Error().Err(err).Msg("failed")
-		return err
-	}
-
-	return nil
-}

api/datastore/backup_test.go

@@ -2,106 +2,79 @@ package datastore
 
 import (
 	"fmt"
-	"os"
-	"path"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
+
+	"github.com/rs/zerolog/log"
 )
 
-func TestCreateBackupFolders(t *testing.T) {
-	_, store := MustNewTestStore(t, true, true)
-
-	connection := store.GetConnection()
-	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
-
-	if isFileExist(backupPath) {
-		t.Error("Expect backups folder to not exist")
-	}
-
-	store.createBackupFolders()
-	if !isFileExist(backupPath) {
-		t.Error("Expect backups folder to exist")
-	}
-}
-
 func TestStoreCreation(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
 	if store == nil {
-		t.Error("Expect to create a store")
+		t.Fatal("Expect to create a store")
 	}
 
-	if store.CheckCurrentEdition() != nil {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	if portainer.SoftwareEdition(v.Edition) != portainer.PortainerCE {
 		t.Error("Expect to get CE Edition")
 	}
+
+	if v.SchemaVersion != portainer.APIVersion {
+		t.Error("Expect to get APIVersion")
+	}
 }
 
 func TestBackup(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
-	connection := store.GetConnection()
-
-	t.Run("Backup should create default db backup", func(t *testing.T) {
+	backupFileName := store.backupFilename()
+	t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) {
 		v := models.Version{
+			Edition:       int(portainer.PortainerCE),
 			SchemaVersion: portainer.APIVersion,
 		}
 		store.VersionService.UpdateVersion(&v)
-		store.backupWithOptions(nil)
+		store.Backup("")
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-
-	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-		store.backupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerVersionUpgradeBackup,
-			BackupDir:      store.commonBackupDir(),
-		})
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
 	})
 }
 
-func TestRemoveWithOptions(t *testing.T) {
-	_, store := MustNewTestStore(t, true, true)
+func TestRestore(t *testing.T) {
+	_, store := MustNewTestStore(t, true, false)
 
-	t.Run("successfully removes file if existent", func(t *testing.T) {
-		store.createBackupFolders()
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
+	t.Run("Basic Restore", func(t *testing.T) {
+		// override and set initial db version and edition
+		updateEdition(store, portainer.PortainerCE)
+		updateVersion(store, "2.4")
 
-		filePath := path.Join(options.BackupDir, options.BackupFileName)
-		f, err := os.Create(filePath)
-		if err != nil {
-			t.Fatalf("file should be created; err=%s", err)
-		}
-		f.Close()
+		store.Backup("")
+		updateVersion(store, "2.16")
+		testVersion(store, "2.16", t)
+		store.Restore()
 
-		err = store.removeWithOptions(options)
-		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
-		}
-
-		if isFileExist(f.Name()) {
-			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
-		}
+		// check if the restore is successful and the version is correct
+		testVersion(store, "2.4", t)
 	})
 
-	t.Run("fails to removes file if non-existent", func(t *testing.T) {
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
+	t.Run("Basic Restore After Multiple Backups", func(t *testing.T) {
+		// override and set initial db version and edition
+		updateEdition(store, portainer.PortainerCE)
+		updateVersion(store, "2.4")
+		store.Backup("")
+		updateVersion(store, "2.14")
+		updateVersion(store, "2.16")
+		testVersion(store, "2.16", t)
+		store.Restore()
 
-		err := store.removeWithOptions(options)
-		if err == nil {
-			t.Error("RemoveWithOptions should fail for non-existent file")
-		}
+		// check if the restore is successful and the version is correct
+		testVersion(store, "2.4", t)
 	})
 }

api/datastore/datastore.go

@@ -31,8 +31,14 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	if encryptionReq {
+		backupFilename, err := store.Backup("")
+		if err != nil {
+			return false, fmt.Errorf("failed to backup database prior to encrypting: %w", err)
+		}
+
 		err = store.encryptDB()
 		if err != nil {
+			store.RestoreFromFile(backupFilename) // restore from backup if encryption fails
 			return false, err
 		}
 	}

api/datastore/datastore_test.go

@@ -150,7 +150,7 @@ func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType porta
 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
 
 	case portainer.EdgeAgentOnKubernetesEnvironment:
-		cs := chisel.NewService(store, nil)
+		cs := chisel.NewService(store, nil, nil)
 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
 		edgeKey := cs.GenerateEdgeKey(URL, "", int(id))
 		expectedEndpoint.EdgeKey = edgeKey
@@ -316,7 +316,7 @@ func (store *Store) testCustomTemplates(t *testing.T) {
 
 	customTemplate.Create(expectedTemplate)
 
-	actualTemplate, err := customTemplate.CustomTemplate(expectedTemplate.ID)
+	actualTemplate, err := customTemplate.Read(expectedTemplate.ID)
 	is.NoError(err, "CustomTemplate should not return an error")
 	is.Equal(expectedTemplate, actualTemplate, "expected and actual template do not match")
 }
@@ -350,11 +350,11 @@ func (store *Store) testRegistries(t *testing.T) {
 	err = regService.Create(reg2)
 	is.NoError(err)
 
-	actualReg1, err := regService.Registry(reg1.ID)
+	actualReg1, err := regService.Read(reg1.ID)
 	is.NoError(err)
 	is.Equal(reg1, actualReg1, "registries differ")
 
-	actualReg2, err := regService.Registry(reg2.ID)
+	actualReg2, err := regService.Read(reg2.ID)
 	is.NoError(err)
 	is.Equal(reg2, actualReg2, "registries differ")
 }
@@ -406,11 +406,11 @@ func (store *Store) testTags(t *testing.T) {
 	err = tags.Create(tag2)
 	is.NoError(err, "Tags.Create should succeed")
 
-	actual, err := tags.Tag(tag1.ID)
+	actual, err := tags.Read(tag1.ID)
 	is.NoError(err, "tag1 should be found")
 	is.Equal(tag1, actual, "tags differ")
 
-	actual, err = tags.Tag(tag2.ID)
+	actual, err = tags.Read(tag2.ID)
 	is.NoError(err, "tag2 should be found")
 	is.Equal(tag2, actual, "tags differ")
 }

api/datastore/helpers_test.go

@@ -0,0 +1,58 @@
+package datastore
+
+import (
+	"path/filepath"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+// isFileExist is helper function to check for file existence
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
+func updateVersion(store *Store, v string) {
+	version, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	version.SchemaVersion = v
+
+	err = store.VersionService.UpdateVersion(version)
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+}
+
+func updateEdition(store *Store, edition portainer.SoftwareEdition) {
+	version, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	version.Edition = int(edition)
+
+	err = store.VersionService.UpdateVersion(version)
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+}
+
+// testVersion is a helper which tests current store version against wanted version
+func testVersion(store *Store, versionWant string, t *testing.T) {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+	if v.SchemaVersion != versionWant {
+		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
+	}
+}