fix(buildscripts): make build process more closely resemble EE (#8881)

* fix(editor): fix styles [EE-5369]

* rm hash

---------

Co-authored-by: testa113 <testa113>

.eslintrc.yml

@@ -23,8 +23,6 @@ parserOptions:
     modules: true
 
 rules:
-  no-console: warn
-  no-alert: error
   no-control-regex: 'off'
   no-empty: warn
   no-empty-function: warn
@@ -88,8 +86,8 @@ overrides:
       no-plusplus: off
       func-style: [error, 'declaration']
       import/prefer-default-export: off
-      no-use-before-define: 'off'
+      no-use-before-define: ['error', { functions: false }]
-      '@typescript-eslint/no-use-before-define': ['error', { functions: false, 'allowNamedExports': true }]
+      '@typescript-eslint/no-use-before-define': ['error', { functions: false }]
       no-shadow: 'off'
       '@typescript-eslint/no-shadow': off
       jsx-a11y/no-autofocus: warn

.github/DISCUSSION_TEMPLATE/help.yaml

@@ -1,11 +0,0 @@
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Before asking a question, make sure it hasn't been already asked and answered. You can search our [discussions](https://github.com/orgs/portainer/discussions) and [bug reports](https://github.com/portainer/portainer/issues) in GitHub.  Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io/) first.
-
-  - type: textarea
-    attributes:
-      label: Ask a Question!
-    validations:
-      required: true

.github/DISCUSSION_TEMPLATE/ideas.yaml

@@ -1,38 +0,0 @@
-body:
-  - type: markdown
-    attributes:
-      value: |
-        # Welcome!
-        
-        Thanks for suggesting an idea for Portainer!
-
-        Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion cagetory](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
-
-        Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io) as they may point you toward a solution.
-        
-        **DO NOT FILE DUPLICATE REQUESTS.**
-
-  - type: textarea
-    attributes:
-      label: Is your feature request related to a problem? Please describe
-      description: Short list of what the feature request aims to address.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Describe the solution you'd like
-      description: A clear and concise description of what you want to happen.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Describe alternatives you've considered
-      description: A clear and concise description of any alternative solutions or features you've considered.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Additional context
-      description: Add any other context or screenshots about the feature request here.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a bug report
+title: ''
+labels: bug/need-confirmation, kind/bug
+assignees: ''
+---
+
+<!--
+
+Thanks for reporting a bug for Portainer !
+
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
+
+Before opening a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
+-->
+
+**Bug description**
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Portainer Logs**
+Provide the logs of your Portainer container or Service.
+You can see how [here](https://documentation.portainer.io/r/portainer-logs)
+
+**Steps to reproduce the issue:**
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Technical details:**
+
+- Portainer version:
+- Docker version (managed by Portainer):
+- Kubernetes version (managed by Portainer):
+- Platform (windows/linux):
+- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
+- Browser:
+- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
+- Have you reviewed our technical documentation and knowledge base? Yes/No
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,166 +0,0 @@
-name: Bug Report
-description: Create a report to help us improve.
-labels: kind/bug,bug/need-confirmation
-body:
-
-  - type: markdown
-    attributes:
-      value: |
-        # Welcome!
-        
-        The issue tracker is for reporting bugs. If you have an [idea for a new feature](https://github.com/orgs/portainer/discussions/categories/ideas) or a [general question about Portainer](https://github.com/orgs/portainer/discussions/categories/help) please post in our [GitHub Discussions](https://github.com/orgs/portainer/discussions).
-        
-        You can also ask for help in our [community Slack channel](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA).
-        
-        **DO NOT FILE ISSUES FOR GENERAL SUPPORT QUESTIONS**.
-
-  - type: checkboxes
-    id: terms
-    attributes:
-      label: Before you start please confirm the following.
-      options:
-        - label: Yes, I've searched similar issues on [GitHub](https://github.com/portainer/portainer/issues).
-          required: true
-        - label: Yes, I've checked whether this issue is covered in the Portainer [documentation](https://docs.portainer.io) or [knowledge base](https://portal.portainer.io/knowledge).
-          required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        # About your issue
-
-        Tell us a bit about the issue you're having.
-
-        How to write a good bug report:
-
-        - Respect the issue template as much as possible.
-        - Summarize the issue so that we understand what is going wrong.
-        - Describe what you would have expected to have happened, and what actually happened instead.
-        - Provide easy to follow steps to reproduce the issue. 
-        - Remain clear and concise.
-        - Format your messages to help the reader focus on what matters and understand the structure of your message, use [Markdown syntax](https://help.github.com/articles/github-flavored-markdown).
-
-  - type: textarea
-    attributes:
-      label: Problem Description
-      description: A clear and concise description of what the bug is. 
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Expected Behavior
-      description: A clear and concise description of what you expected to happen.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Actual Behavior
-      description: A clear and concise description of what actually happens.
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Steps to Reproduce
-      description: Please be as detailed as possible when providing steps to reproduce.
-      placeholder: |
-        1. Go to '...'
-        2. Click on '....'
-        3. Scroll down to '....'
-        4. See error   
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Portainer logs or screenshots
-      description: Provide Portainer container logs or any screenshots related to the issue.
-    validations:
-      required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        # About your environment
-
-        Tell us a bit about your Portainer environment.
-
-  - type: dropdown
-    attributes:
-      label: Portainer version
-      description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
-      multiple: false
-      options:
-        - '2.19.3'
-        - '2.19.2'
-        - '2.19.1'
-        - '2.19.0'
-        - '2.18.4'
-        - '2.18.3'
-        - '2.18.2'
-        - '2.18.1'
-        - '2.17.1'
-        - '2.17.0'
-        - '2.16.2'
-        - '2.16.1'
-        - '2.16.0'
-    validations:
-      required: true
-
-  - type: dropdown
-    attributes:
-      label: Portainer Edition
-      multiple: false
-      options:
-        - 'Business Edition (BE/EE) with 5NF / 3NF license'
-        - 'Business Edition (BE/EE) with Home & Student license'
-        - 'Business Edition (BE/EE) with Starter license'
-        - 'Business Edition (BE/EE) with Professional or Enterprise license'
-        - 'Community Edition (CE)'
-    validations:
-      required: true
-
-  - type: input
-    attributes:
-      label: Platform and Version
-      description: |
-        Enter your container management platform (Docker | Swarm | Kubernetes) along with the version. 
-        Example: Docker 24.0.3 | Docker Swarm 24.0.3 | Kubernetes 1.26
-        You can find our supported platforms [in our documentation](https://docs.portainer.io/start/requirements-and-prerequisites).
-    validations:
-      required: true
-
-  - type: input
-    attributes:
-      label: OS and Architecture
-      description: |
-        Enter your Operating System, Version and Architecture. Example: Ubuntu 22.04, AMD64 | Raspbian OS, ARM64
-    validations:
-      required: true
-
-  - type: input
-    attributes:
-      label: Browser
-      description: | 
-        Enter your browser and version. Example: Google Chrome 114.0
-    validations:
-      required: false
-
-  - type: textarea
-    attributes:
-      label: What command did you use to deploy Portainer?
-      description: |
-        Example: `docker run -d -p 8000:8000 -p 9443:9443 --name portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce:latest`
-        If you deployed Portainer using a compose file or manifest you can provide this here as well.
-      render: bash
-    validations:
-      required: false
-
-  - type: textarea
-    attributes:
-      label: Additional Information
-      description: Any additional information about your environment, the bug, or anything else you think might be helpful.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +1,5 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Question
+  - name: Portainer Business Edition - Get 5 nodes free
-    url: https://github.com/orgs/portainer/discussions/new?category=help
+    url: https://portainer.io/pricing/take5 
-    about: Ask us a question about Portainer usage or deployment.
+    about: Portainer Business Edition has more features, more support and you can now get 5 nodes free for as long as you want.
-  - name: Idea or Feature Request
-    url: https://github.com/orgs/portainer/discussions/new?category=ideas
-    about: Suggest an idea or feature/enhancement that should be added in Portainer.
-  - name: Portainer Business Edition - Get 3 Nodes Free
-    url: https://www.portainer.io/take-3
-    about: Portainer Business Edition has more features, more support and you can now get 3 nodes free for as long as you want.

.github/workflows/ci.yaml

@@ -1,148 +0,0 @@
-name: ci
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'develop'
-      - '!release/*'
-  pull_request:
-    branches:
-      - 'develop'
-      - 'release/*'
-      - 'feat/*'
-      - 'fix/*'
-      - 'refactor/*'
-
-env:
-  DOCKER_HUB_REPO: portainerci/portainer
-  NODE_ENV: testing
-  GO_VERSION: 1.21.3
-  NODE_VERSION: 18.x
-
-jobs:
-  build_images:
-    strategy:
-      matrix:
-        config:
-          - { platform: linux, arch: amd64 }
-          - { platform: linux, arch: arm64 }
-          - { platform: windows, arch: amd64, version: 1809 }
-          - { platform: windows, arch: amd64, version: ltsc2022 }
-    runs-on: arc-runner-set
-    steps:
-      - name: '[preparation] checkout the current branch'
-        uses: actions/checkout@v3.5.3
-        with:
-          ref: ${{ github.event.inputs.branch }}
-      - name: '[preparation] set up golang'
-        uses: actions/setup-go@v4.0.1
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-      - name: '[preparation] cache paths'
-        id: cache-dir-path
-        run: |
-          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
-          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
-          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
-      - name: '[preparation] cache go'
-        uses: actions/cache@v3
-        with:
-          path: |
-            ${{ steps.cache-dir-path.outputs.go-build-dir }}
-            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
-          enableCrossOsArchive: true
-      - name: '[preparation] set up node.js'
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: ''
-      - name: '[preparation] cache yarn'
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
-          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
-          enableCrossOsArchive: true
-      - name: '[preparation] set up qemu'
-        uses: docker/setup-qemu-action@v2
-      - name: '[preparation] set up docker context for buildx'
-        run: docker context create builders
-      - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
-        with:
-          endpoint: builders
-      - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-      - name: '[preparation] set the container image tag'
-        run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
-          else
-            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
-          fi
-
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
-          else 
-            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
-          fi
-
-          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
-      - name: '[execution] build linux & windows portainer binaries'
-        run: |
-          export YARN_VERSION=$(yarn --version)
-          export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
-          export BUILDNUMBER=${GITHUB_RUN_NUMBER}
-          make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
-        env:
-          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
-      - name: '[execution] build and push docker images'
-        run: |
-          if [ "${{ matrix.config.platform }}" == "windows" ]; then
-            mv dist/portainer dist/portainer.exe
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
-          else 
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
-            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
-          fi
-        env:
-          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
-  build_manifests:
-    runs-on: arc-runner-set
-    needs: [build_images]
-    steps:
-      - name: '[preparation] docker login'
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-      - name: '[preparation] set up docker context for buildx'
-        run: docker version && docker context create builders
-      - name: '[preparation] set up docker buildx'
-        uses: docker/setup-buildx-action@v2
-        with:
-          endpoint: builders
-      - name: '[execution] build and push manifests'
-        run: |
-          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
-          else
-            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
-          fi
-
-          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
-            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"

.github/workflows/label-conflcts.yaml

@@ -11,5 +11,5 @@ jobs:
         with:
           CONFLICT_LABEL_NAME: 'has conflicts'
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MAX_RETRIES: 10
+          MAX_RETRIES: 5
-          WAIT_MS: 60000
+          WAIT_MS: 5000

.github/workflows/lint.yml

@@ -12,9 +12,6 @@ on:
       - develop
       - release/*
 
-env:
-  GO_VERSION: 1.21.3
-
 jobs:
   run-linters:
     name: Run linters
@@ -24,12 +21,13 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '18'
+          node-version: '14'
           cache: 'yarn'
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: 1.19.4
       - run: yarn --frozen-lockfile
+
       - name: Run linters
         uses: wearerequired/lint-action@v1
         with:
@@ -44,6 +42,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.54.1
+          version: latest
           working-directory: api
-          args: --timeout=10m -c .golangci.yaml
+          args: -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -2,25 +2,21 @@ name: Nightly Code Security Scan
 
 on:
   schedule:
-    - cron: '0 20 * * *'
+    - cron: '0 8 * * *'
   workflow_dispatch:
 
-env:
-  GO_VERSION: 1.21.3
-
 jobs:
   client-dependencies:
-    name: Client Dependency Check
+    name: Client dependency check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
       github.ref == 'refs/heads/develop'
     outputs:
       js: ${{ steps.set-matrix.outputs.js_result }}
     steps:
-      - name: checkout repository
+      - uses: actions/checkout@master
-        uses: actions/checkout@master
 
-      - name: scan vulnerabilities by Snyk
+      - name: Run Snyk to check for vulnerabilities
         uses: snyk/actions/node@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
@@ -28,148 +24,147 @@ jobs:
         with:
           json: true
 
-      - name: upload scan result as develop artifact
+      - name: Upload js security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-develop-result
           path: snyk.json
 
-      - name: develop scan report export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/js-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
 
-      - name: upload html file as artifact
+      - name: Upload js result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-js-result-${{github.run_id}}
           path: js-result.html
 
-      - name: analyse vulnerabilities
+      - name: Analyse the js result
         id: set-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
-          echo "js_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=js_result::${result}"
 
   server-dependencies:
-    name: Server Dependency Check
+    name: Server dependency check
     runs-on: ubuntu-latest
     if: >- # only run for develop branch
       github.ref == 'refs/heads/develop'
     outputs:
       go: ${{ steps.set-matrix.outputs.go_result }}
     steps:
-      - name: checkout repository
+      - uses: actions/checkout@master
-        uses: actions/checkout@master
 
-      - name: install Go
+      - uses: actions/setup-go@v3
-        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.19.4'
 
-      - name: download Go modules
+      - name: Download go modules
         run: cd ./api && go get -t -v -d ./...
 
-      - name: scan vulnerabilities by Snyk
+      - name: Run Snyk to check for vulnerabilities
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: upload scan result as develop artifact
+      - name: Upload go security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-develop-result
           path: snyk.json
 
-      - name: develop scan report export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/go-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
 
-      - name: upload html file as artifact
+      - name: Upload go result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-go-result-${{github.run_id}}
           path: go-result.html
 
-      - name: analyse vulnerabilities
+      - name: Analyse the go result
         id: set-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
-          echo "go_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=go_result::${result}"
 
   image-vulnerability:
-    name: Image Vulnerability Check
+    name: Build docker image and Image vulnerability check
     runs-on: ubuntu-latest
     if: >-
       github.ref == 'refs/heads/develop'
     outputs:
-      image-trivy: ${{ steps.set-trivy-matrix.outputs.image_trivy_result }}
+      image: ${{ steps.set-matrix.outputs.image_result }}
-      image-docker-scout: ${{ steps.set-docker-scout-matrix.outputs.image_docker_scout_result }}
     steps:
-      - name: scan vulnerabilities by Trivy
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Use golang 1.19.4
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.19.4'
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+
+      - name: Install packages
+        run: yarn --frozen-lockfile
+
+      - name: build
+        run: make build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: build/linux/Dockerfile
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+
+      - name: Load docker image
+        run: |
+          docker load --input /tmp/trivy-portainer-image.tar
+
+      - name: Run Trivy vulnerability scanner
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
-      - name: upload Trivy image security scan result as artifact
+      - name: Upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-develop-result
           path: image-trivy.json
 
-      - name: develop Trivy scan report export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-trivy-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
 
-      - name: upload html file as Trivy artifact
+      - name: Upload go result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-${{github.run_id}}
-          path: image-trivy-result.html
+          path: image-result.html
 
-      - name: analyse vulnerabilities from Trivy
+      - name: Analyse the trivy result
-        id: set-trivy-matrix
+        id: set-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
-          echo "image_trivy_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=image_result::${result}"
-
-      - name: scan vulnerabilities by Docker Scout
-        uses: docker/scout-action@v1
-        continue-on-error: true
-        with:
-          command: cves
-          image: portainerci/portainer:develop
-          sarif-file: image-docker-scout.json
-          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
-          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} 
-
-      - name: upload Docker Scout image security scan result as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: image-security-scan-develop-result
-          path: image-docker-scout.json
-
-      - name: develop Docker Scout scan report export to html
-        run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
-
-      - name: upload html file as Docker Scout artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: html-image-result-${{github.run_id}}
-          path: image-docker-scout-result.html
-
-      - name: analyse vulnerabilities from Docker Scout
-        id: set-docker-scout-matrix
-        run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=matrix)
-          echo "image_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
-    name: Analyse Scan Results
+    name: Analyse scan result
     needs: [client-dependencies, server-dependencies, image-vulnerability]
     runs-on: ubuntu-latest
     if: >-
@@ -178,27 +173,23 @@ jobs:
       matrix:
         js: ${{fromJson(needs.client-dependencies.outputs.js)}}
         go: ${{fromJson(needs.server-dependencies.outputs.go)}}
-        image-trivy: ${{fromJson(needs.image-vulnerability.outputs.image-trivy)}}
+        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
-        image-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.image-docker-scout)}}
     steps:
-      - name: display the results of js, Go, and image scan
+      - name: Display the results of js, go and image
         run: |
-          echo "${{ matrix.js.status }}"
+          echo ${{ matrix.js.status }}
-          echo "${{ matrix.go.status }}"
+          echo ${{ matrix.go.status }}
-          echo "${{ matrix.image-trivy.status }}"
+          echo ${{ matrix.image.status }}
-          echo "${{ matrix.image-docker-scout.status }}"
+          echo ${{ matrix.js.summary }}
-          echo "${{ matrix.js.summary }}"
+          echo ${{ matrix.go.summary }}
-          echo "${{ matrix.go.summary }}"
+          echo ${{ matrix.image.summary }}
-          echo "${{ matrix.image-trivy.summary }}"
-          echo "${{ matrix.image-docker-scout.summary }}"
 
-      - name: send message to Slack
+      - name: Send Slack message
         if: >-
           matrix.js.status == 'failure' ||
           matrix.go.status == 'failure' || 
-          matrix.image-trivy.status == 'failure' || 
+          matrix.image.status == 'failure'
-          matrix.image-docker-scout.status == 'failure' 
+        uses: slackapi/slack-github-action@v1.18.0
-        uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |
             {
@@ -233,14 +224,7 @@ jobs:
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "*Image Trivy vulnerability check*: *${{ matrix.image-trivy.status }}*\n${{ matrix.image-trivy.summary }}\n"
+                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
-                      }
-                    },
-                    {
-                      "type": "section",
-                      "text": {
-                        "type": "mrkdwn",
-                        "text": "*Image Docker Scout vulnerability check*: *${{ matrix.image-docker-scout.status }}*\n${{ matrix.image-docker-scout.summary }}\n"
                       }
                     }
                   ]

.github/workflows/pr-security.yml

@@ -7,19 +7,15 @@ on:
       - edited
     paths:
       - 'package.json'
-      - 'go.mod'
+      - 'api/go.mod'
+      - 'gruntfile.js'
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
       - 'build/windows/Dockerfile'
-      - '.github/workflows/pr-security.yml'
-
-env:
-  GO_VERSION: 1.21.3
-  NODE_VERSION: 18.x
 
 jobs:
   client-dependencies:
-    name: Client Dependency Check
+    name: Client dependency check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
@@ -27,10 +23,9 @@ jobs:
     outputs:
       jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
     steps:
-      - name: checkout repository
+      - uses: actions/checkout@master
-        uses: actions/checkout@master
 
-      - name: scan vulnerabilities by Snyk
+      - name: Run Snyk to check for vulnerabilities
         uses: snyk/actions/node@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
@@ -38,13 +33,13 @@ jobs:
         with:
           json: true
 
-      - name: upload scan result as pull-request artifact
+      - name: Upload js security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: js-security-scan-feat-result
           path: snyk.json
 
-      - name: download artifacts from develop branch built by nightly scan
+      - name: Download artifacts from develop branch
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -56,24 +51,24 @@ jobs:
             echo "null" > ./js-snyk-develop.json
           fi
 
-      - name: pr vs develop scan report comparison export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=table --export --export-filename="/data/js-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
 
-      - name: upload html file as artifact
+      - name: Upload js result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-js-result-compare-to-develop-${{github.run_id}}
           path: js-result.html
 
-      - name: analyse different vulnerabilities against develop branch
+      - name: Analyse the js diff result
         id: set-diff-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
-          echo "js_diff_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=js_diff_result::${result}"
 
   server-dependencies:
-    name: Server Dependency Check
+    name: Server dependency check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
@@ -81,32 +76,30 @@ jobs:
     outputs:
       godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
     steps:
-      - name: checkout repository
+      - uses: actions/checkout@master
-        uses: actions/checkout@master
 
-      - name: install Go
+      - uses: actions/setup-go@v3
-        uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.19.4'
 
-      - name: download Go modules
+      - name: Download go modules
         run: cd ./api && go get -t -v -d ./...
 
-      - name: scan vulnerabilities by Snyk
+      - name: Run Snyk to check for vulnerabilities
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
-      - name: upload scan result as pull-request artifact
+      - name: Upload go security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: go-security-scan-feature-result
           path: snyk.json
 
-      - name: download artifacts from develop branch built by nightly scan
+      - name: Download artifacts from develop branch
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -118,79 +111,78 @@ jobs:
             echo "null" > ./go-snyk-develop.json
           fi
 
-      - name: pr vs develop scan report comparison export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=table --export --export-filename="/data/go-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
 
-      - name: upload html file as artifact
+      - name: Upload go result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-go-result-compare-to-develop-${{github.run_id}}
           path: go-result.html
 
-      - name: analyse different vulnerabilities against develop branch
+      - name: Analyse the go diff result
         id: set-diff-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
-          echo "go_diff_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=go_diff_result::${result}"
 
   image-vulnerability:
-    name: Image Vulnerability Check
+    name: Build docker image and Image vulnerability check
     runs-on: ubuntu-latest
     if: >-
       github.event.pull_request &&
       github.event.review.body == '/scan'
     outputs:
-      imagediff-trivy: ${{ steps.set-diff-trivy-matrix.outputs.image_diff_trivy_result }}
+      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
-      imagediff-docker-scout: ${{ steps.set-diff-docker-scout-matrix.outputs.image_diff_docker_scout_result }}
     steps:
-      - name: checkout code
+      - name: Checkout code
         uses: actions/checkout@master
 
-      - name: install Go
+      - name: Use golang 1.19.4
         uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.19.4'
 
-      - name: install Node.js
+      - name: Use Node.js 18.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v1
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: 18.x
 
       - name: Install packages
         run: yarn --frozen-lockfile
 
       - name: build
-        run: make build-all
+        run: make build
 
-      - name: set up docker buildx
+      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
-      - name: build and compress image
+      - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v2
         with:
           context: .
           file: build/linux/Dockerfile
-          tags: local-portainer:${{ github.sha }}
+          tags: trivy-portainer:${{ github.sha }}
-          outputs: type=docker,dest=/tmp/local-portainer-image.tar
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
 
-      - name: load docker image
+      - name: Load docker image
         run: |
-          docker load --input /tmp/local-portainer-image.tar
+          docker load --input /tmp/trivy-portainer-image.tar
 
-      - name: scan vulnerabilities by Trivy
+      - name: Run Trivy vulnerability scanner
         uses: docker://docker.io/aquasec/trivy:latest
         continue-on-error: true
         with:
-          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress local-portainer:${{ github.sha }}
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
 
-      - name: upload Trivy image security scan result as artifact
+      - name: Upload image security scan result as artifact
         uses: actions/upload-artifact@v3
         with:
           name: image-security-scan-feature-result
           path: image-trivy.json
 
-      - name: download Trivy artifacts from develop branch built by nightly scan
+      - name: Download artifacts from develop branch
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -202,68 +194,24 @@ jobs:
             echo "null" > ./image-trivy-develop.json
           fi
 
-      - name: pr vs develop Trivy scan report comparison export to html
+      - name: Export scan result to html file
         run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-trivy-result")
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
 
-      - name: upload html file as Trivy artifact
+      - name: Upload image result html file
         uses: actions/upload-artifact@v3
         with:
           name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-trivy-result.html
+          path: image-result.html
 
-      - name: analyse different vulnerabilities against develop branch by Trivy
+      - name: Analyse the image diff result
-        id: set-diff-trivy-matrix
+        id: set-diff-matrix
         run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
-          echo "image_diff_trivy_result=${result}" >> $GITHUB_OUTPUT
+          echo "::set-output name=image_diff_result::${result}"
-
-      - name: scan vulnerabilities by Docker Scout
-        uses: docker/scout-action@v1
-        continue-on-error: true
-        with:
-          command: cves
-          image: local-portainer:${{ github.sha }}
-          sarif-file: image-docker-scout.json
-          dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
-          dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: upload Docker Scout image security scan result as artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: image-security-scan-feature-result
-          path: image-docker-scout.json
-
-      - name: download Docker Scout artifacts from develop branch built by nightly scan
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          mv ./image-docker-scout.json ./image-docker-scout-feature.json
-          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
-          if [[ -e ./image-docker-scout.json ]]; then
-            mv ./image-docker-scout.json ./image-docker-scout-develop.json
-          else
-            echo "null" > ./image-docker-scout-develop.json
-          fi
-
-      - name: pr vs develop Docker Scout scan report comparison export to html
-        run: |
-          $(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
-
-      - name: upload html file as Docker Scout artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: html-image-result-compare-to-develop-${{github.run_id}}
-          path: image-docker-scout-result.html
-
-      - name: analyse different vulnerabilities against develop branch by Docker Scout
-        id: set-diff-docker-scout-matrix
-        run: |
-          result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=matrix)
-          echo "image_diff_docker_scout_result=${result}" >> $GITHUB_OUTPUT
 
   result-analysis:
-    name: Analyse Scan Result Against develop Branch
+    name: Analyse scan result compared to develop
     needs: [client-dependencies, server-dependencies, image-vulnerability]
     runs-on: ubuntu-latest
     if: >-
@@ -273,22 +221,18 @@ jobs:
       matrix:
         jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
         godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
-        imagediff-trivy: ${{fromJson(needs.image-vulnerability.outputs.imagediff-trivy)}}
+        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
-        imagediff-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.imagediff-docker-scout)}}
     steps:
-      - name: check job status of diff result
+      - name: Check job status of diff result
         if: >-
           matrix.jsdiff.status == 'failure' ||
           matrix.godiff.status == 'failure' ||
-          matrix.imagediff-trivy.status == 'failure' ||
+          matrix.imagediff.status == 'failure'
-          matrix.imagediff-docker-scout.status == 'failure'
         run: |
-          echo "${{ matrix.jsdiff.status }}"
+          echo ${{ matrix.jsdiff.status }}
-          echo "${{ matrix.godiff.status }}"
+          echo ${{ matrix.godiff.status }}
-          echo "${{ matrix.imagediff-trivy.status }}"
+          echo ${{ matrix.imagediff.status }}
-          echo "${{ matrix.imagediff-docker-scout.status }}"
+          echo ${{ matrix.jsdiff.summary }}
-          echo "${{ matrix.jsdiff.summary }}"
+          echo ${{ matrix.godiff.summary }}
-          echo "${{ matrix.godiff.summary }}"
+          echo ${{ matrix.imagediff.summary }}
-          echo "${{ matrix.imagediff-trivy.summary }}"
-          echo "${{ matrix.imagediff-docker-scout.summary }}"
           exit 1

.github/workflows/stale.yml

@@ -1,8 +1,7 @@
 name: Close Stale Issues
 on:
   schedule:
-    - cron: '0 12 * * *'
+  - cron: '0 12 * * *'
-  workflow_dispatch:
 jobs:
   stale:
     runs-on: ubuntu-latest
@@ -10,7 +9,7 @@ jobs:
       issues: write
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v4.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/test.yaml

@@ -1,11 +1,5 @@
 name: Test
-
 on: push
-
-env:
-  GO_VERSION: 1.21.3
-  NODE_VERSION: 18.x
-
 jobs:
   test-client:
     runs-on: ubuntu-latest
@@ -14,25 +8,22 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 
       - name: Run tests
-        run: make test-client ARGS="--maxWorkers=2"
+        run: yarn jest --maxWorkers=2
-  test-server:
+  # test-server:
-    strategy:
+  #   runs-on: ubuntu-latest
-      matrix:
+  #   env:
-        config:
+  #     GOPRIVATE: "github.com/portainer"
-          - { platform: linux, arch: amd64 }
+  #   steps:
-          - { platform: linux, arch: arm64 }
+  #     - uses: actions/checkout@v3
-          - { platform: windows, arch: amd64, version: 1809 }
+  #     - uses: actions/setup-go@v3
-          - { platform: windows, arch: amd64, version: ltsc2022 }
+  #       with:
-    runs-on: ubuntu-latest
+  #         go-version: '1.18'
-    steps:
+  #     - name: Run tests
-      - uses: actions/checkout@v3
+  #       run: |
-      - uses: actions/setup-go@v3
+  #         cd api
-        with:
+  #         go test ./...
-          go-version: ${{ env.GO_VERSION }}
-      - name: Run tests
-        run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -7,10 +7,6 @@ on:
       - develop
       - 'release/*'
 
-env:
-  GO_VERSION: 1.21.3
-  NODE_VERSION: 18.x
-
 jobs:
   openapi-spec:
     runs-on: ubuntu-latest
@@ -19,13 +15,13 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: ${{ env.GO_VERSION }}
+          go-version: '1.18'
 
       - name: Download golang modules
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: ${{ env.NODE_VERSION }}
+          node-version: '14'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

.gitignore

@@ -11,10 +11,8 @@ storybook-static
 *.DS_Store
 
 .eslintcache
-__debug_bin*
+__debug_bin
 
 api/docs
 .idea
 .env
-go.work.sum
-

.husky/pre-commit

@@ -1,4 +0,0 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
-yarn lint-staged

.prettierrc

@@ -2,21 +2,15 @@
   "printWidth": 180,
   "singleQuote": true,
   "htmlWhitespaceSensitivity": "strict",
-  "trailingComma": "es5",
   "overrides": [
     {
-      "files": [
+      "files": ["*.html"],
-        "*.html"
-      ],
       "options": {
         "parser": "angular"
       }
     },
     {
-      "files": [
+      "files": ["*.{j,t}sx", "*.ts"],
-        "*.{j,t}sx",
-        "*.ts"
-      ],
       "options": {
         "printWidth": 80
       }

.storybook/main.js

@@ -0,0 +1,55 @@
+const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
+
+module.exports = {
+  stories: ['../app/**/*.stories.mdx', '../app/**/*.stories.@(ts|tsx)'],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    {
+      name: '@storybook/addon-postcss',
+      options: {
+        cssLoaderOptions: {
+          importLoaders: 1,
+          modules: {
+            localIdentName: '[path][name]__[local]',
+            auto: true,
+            exportLocalsConvention: 'camelCaseOnly',
+          },
+        },
+        postcssLoaderOptions: {
+          implementation: require('postcss'),
+        },
+      },
+    },
+  ],
+  webpackFinal: (config) => {
+    config.resolve.plugins = [
+      ...(config.resolve.plugins || []),
+      new TsconfigPathsPlugin({
+        extensions: config.resolve.extensions,
+      }),
+    ];
+
+    const svgRule = config.module.rules.find((rule) => rule.test && typeof rule.test.test === 'function' && rule.test.test('.svg'));
+    svgRule.test = new RegExp(svgRule.test.source.replace('svg|', ''));
+
+    config.module.rules.unshift({
+      test: /\.svg$/i,
+      type: 'asset',
+      resourceQuery: { not: [/c/] }, // exclude react component if *.svg?url
+    });
+
+    config.module.rules.unshift({
+      test: /\.svg$/i,
+      issuer: /\.(js|ts)(x)?$/,
+      resourceQuery: /c/, // *.svg?c
+      use: [{ loader: '@svgr/webpack', options: { icon: true } }],
+    });
+
+    return config;
+  },
+  core: {
+    builder: 'webpack5',
+  },
+  staticDirs: ['./public'],
+};

.storybook/main.ts

@@ -1,95 +0,0 @@
-import { StorybookConfig } from '@storybook/react-webpack5';
-
-import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
-import { Configuration } from 'webpack';
-import postcss from 'postcss';
-const config: StorybookConfig = {
-  stories: ['../app/**/*.stories.@(ts|tsx)'],
-  addons: [
-    '@storybook/addon-links',
-    '@storybook/addon-essentials',
-    {
-      name: '@storybook/addon-styling',
-      options: {
-        cssLoaderOptions: {
-          importLoaders: 1,
-          modules: {
-            localIdentName: '[path][name]__[local]',
-            auto: true,
-            exportLocalsConvention: 'camelCaseOnly',
-          },
-        },
-        postCss: {
-          implementation: postcss,
-        },
-      },
-    },
-  ],
-  webpackFinal: (config) => {
-    const rules = config?.module?.rules || [];
-
-    const imageRule = rules.find((rule) => {
-      const test = (rule as { test: RegExp }).test;
-
-      if (!test) {
-        return false;
-      }
-
-      return test.test('.svg');
-    }) as { [key: string]: any };
-
-    imageRule.exclude = /\.svg$/;
-
-    rules.unshift({
-      test: /\.svg$/i,
-      type: 'asset',
-      resourceQuery: {
-        not: [/c/],
-      }, // exclude react component if *.svg?url
-    });
-
-    rules.unshift({
-      test: /\.svg$/i,
-      issuer: /\.(js|ts)(x)?$/,
-      resourceQuery: /c/,
-      // *.svg?c
-      use: [
-        {
-          loader: '@svgr/webpack',
-          options: {
-            icon: true,
-          },
-        },
-      ],
-    });
-    return {
-      ...config,
-      resolve: {
-        ...config.resolve,
-        plugins: [
-          ...(config.resolve?.plugins || []),
-          new TsconfigPathsPlugin({
-            extensions: config.resolve?.extensions,
-          }),
-        ],
-      },
-      module: {
-        ...config.module,
-        rules,
-      },
-    } satisfies Configuration;
-  },
-  staticDirs: ['./public'],
-  typescript: {
-    reactDocgen: 'react-docgen-typescript',
-  },
-  framework: {
-    name: '@storybook/react-webpack5',
-    options: {},
-  },
-  docs: {
-    autodocs: true,
-  },
-};
-
-export default config;

.vscode.example/portainer.code-snippets

@@ -15,15 +15,6 @@
   // 	],
   // 	"description": "Log output to console"
   // }
-  "React Named Export Component": {
-    "prefix": "rnec",
-    "body": [
-      "export function $TM_FILENAME_BASE() {",
-      "  return <div>$TM_FILENAME_BASE</div>;",
-      "}"
-    ],
-    "description": "React Named Export Component"
-  },
   "Component": {
     "scope": "javascript",
     "prefix": "mycomponent",

CONTRIBUTING.md

@@ -79,33 +79,25 @@ The feature request process is similar to the bug report process but has an extr
 
 Ensure you have Docker, Node.js, yarn, and Golang installed in the correct versions.
 
-Install dependencies:
+Install dependencies with yarn:
 
 ```sh
-$ make deps
+$ yarn
 ```
 
 Then build and run the project in a Docker container:
 
 ```sh
-$ make dev
+$ yarn start
 ```
 
-Portainer server can now be accessed at <https://localhost:9443>. and UI dev server runs on <http://localhost:8999>.
+Portainer can now be accessed at <https://localhost:9443>.
-
-if you want to build the project you can run:
-
-```sh
-make build-all
-```
-
-For additional make commands, run `make help`.
 
 Find more detailed steps at <https://docs.portainer.io/contribute/build>.
 
-### Build customization
+### Build customisation
 
-You can customize the following settings:
+You can customise the following settings:
 
 - `PORTAINER_DATA`: The host dir or volume name used by portainer (default is `/tmp/portainer`, which won't persist over reboots).
 - `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`).

Makefile

@@ -4,89 +4,77 @@
 PLATFORM=$(shell go env GOOS)
 ARCH=$(shell go env GOARCH)
 
+TAG=latest
+SWAG_VERSION=v1.8.11
+
 # build target, can be one of "production", "testing", "development"
 ENV=development
 WEBPACK_CONFIG=webpack/webpack.$(ENV).js
-TAG=latest
 
-SWAG=go run github.com/swaggo/swag/cmd/swag@v1.8.11 
-GOTESTSUM=go run gotest.tools/gotestsum@latest
-
-# Don't change anything below this line unless you know what you're doing
 .DEFAULT_GOAL := help
 
+.PHONY: help build-storybook build-client devops download-binaries tidy clean client-deps
 
 ##@ Building
-.PHONY: init-dist build-storybook build build-client build-server build-image devops
+
 init-dist:
 	@mkdir -p dist
 
-build-all: deps build-server build-client ## Build the client, server and download external dependancies (doesn't build an image)
+build-storybook:
+	yarn storybook:build
 
-build-client: init-dist ## Build the client
+build: build-server build-client ## Build the server and client
+
+build-client: init-dist client-deps ## Build the client
 	export NODE_ENV=$(ENV) && yarn build --config $(WEBPACK_CONFIG)
 
 build-server: init-dist ## Build the server binary
 	./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
 
-build-image: build-all ## Build the Portainer image locally
+build-image: build ## Build the Portainer image
 	docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
 
-build-storybook: ## Build and serve the storybook files
+devops: clean init-dist download-binaries build-client ## Build the server binary for CI
-	yarn storybook:build
-
-devops: clean deps build-client ## Build the everything target specifically for CI
 	echo "Building the devops binary..."
 	@./build/build_binary_azuredevops.sh "$(PLATFORM)" "$(ARCH)"
 
-##@ Build dependencies
+##@ Dependencies
-.PHONY: deps server-deps client-deps tidy
-deps: server-deps client-deps ## Download all client and server build dependancies
 
-server-deps: init-dist ## Download dependant server binaries
+download-binaries: ## Download dependant binaries
 	@./build/download_binaries.sh $(PLATFORM) $(ARCH)
 
-client-deps: ## Install client dependencies
-	yarn
-
 tidy: ## Tidy up the go.mod file
 	cd api && go mod tidy
  
+client-deps: ## Install client dependencies
+	yarn
 
 ##@ Cleanup
-.PHONY: clean
+
 clean: ## Remove all build and download artifacts
 	@echo "Clearing the dist directory..."
 	@rm -rf dist/*
 
-
 ##@ Testing
-.PHONY: test test-client test-server
-test: test-server test-client ## Run all tests
 
 test-client: ## Run client tests
-	yarn test $(ARGS)
+	yarn test
 
 test-server:	## Run server tests
-	cd api && $(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...
+	cd api && go test -v ./...
+
+test: test-client test-server ## Run all tests
 
 ##@ Dev
-.PHONY: dev dev-client dev-server
-dev: ## Run both the client and server in development mode	
-	make dev-server
-	make dev-client
 
 dev-client: ## Run the client in development mode 
 	yarn dev
 
-dev-server: build-server ## Run the server in development mode
+dev-server: build-image ## Run the server in development mode
 	@./dev/run_container.sh
 
 
 ##@ Format
-.PHONY: format format-client format-server
-
-format: format-client format-server ## Format all code
 
 format-client: ## Format client code
 	yarn format
@@ -94,8 +82,10 @@ format-client: ## Format client code
 format-server: ## Format server code
 	cd api && go fmt ./...
 
+format: format-client format-server ## Format all code
+
 ##@ Lint
-.PHONY: lint lint-client lint-server
+
 lint: lint-client lint-server ## Lint all code
 
 lint-client: ## Lint client code
@@ -104,23 +94,29 @@ lint-client: ## Lint client code
 lint-server: ## Lint server code
 	cd api && go vet ./...
 
-
 ##@ Extension
-.PHONY: dev-extension
+
 dev-extension: build-server build-client ## Run the extension in development mode
 	make local -f build/docker-extension/Makefile
 
-
 ##@ Docs
-.PHONY: docs-build docs-validate docs-clean docs-validate-clean
+
-docs-build: init-dist ## Build docs
+docs-deps: ## Install docs dependencies
-	cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
+	go install github.com/swaggo/swag/cmd/swag@$(SWAG_VERSION)
+
+docs-build: docs-deps ## Build docs
+	cd api && swag init -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
 
 docs-validate: docs-build ## Validate docs
-	yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
+	yarn swagger2openapi --warnOnly api/docs/swagger.yaml -o api/docs/openapi.yaml
-	yarn swagger-cli validate dist/docs/openapi.yaml
+	yarn swagger-cli validate api/docs/openapi.yaml
+
+docs-clean: ## Clean docs
+	rm -rf api/docs
+
+docs-validate-clean: docs-validate docs-clean ## Validate and clean docs
 
 ##@ Helpers
-.PHONY: help
+
 help:  ## Display this help
-	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

README.md

@@ -9,7 +9,7 @@ Portainer consists of a single container that can run on any cluster. It can be
 **Portainer Business Edition** builds on the open-source base and includes a range of advanced features and functions (like RBAC and Support) that are specific to the needs of business users.
 
 - [Compare Portainer CE and Compare Portainer BE](https://portainer.io/products)
-- [Take3 – get 3 free nodes of Portainer Business for as long as you want them](https://www.portainer.io/take-3)
+- [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
 - [Portainer BE install guide](https://install.portainer.io)
 
 ## Latest Version
@@ -21,8 +21,8 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
 ## Getting started
 
 - [Deploy Portainer](https://docs.portainer.io/start/install)
-- [Documentation](https://docs.portainer.io)
+- [Documentation](https://documentation.portainer.io)
-- [Contribute to the project](https://docs.portainer.io/contribute/contribute)
+- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
 
 ## Features & Functions
 
@@ -30,22 +30,23 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 - [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
 - [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
+- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
 
 ## Getting help
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainer's community support channels [here.](https://www.portainer.io/get-support-for-portainer)
+Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
 
 - Issues: https://github.com/portainer/portainer/issues
 - Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
 
-You can join the Portainer Community by visiting [https://www.portainer.io/join-our-community](https://www.portainer.io/join-our-community). This will give you advance notice of events, content and other related Portainer content.
+You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
 
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
-- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
+- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://documentation.portainer.io/contributing/instructions/) to build it locally and make a pull request.
 
 ## Security
 
@@ -59,7 +60,7 @@ If you are a developer, and our code in this repo makes sense to you, we would l
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
 
-When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
+When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/documentation/in-app-analytics-and-privacy-policy/). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
 
 ## Limitations
 

api/.golangci.yaml

@@ -1,37 +1,26 @@
 linters:
-  # Disable all linters, the defaults don't pass on our code yet
+  # Disable all linters.
   disable-all: true
-
-  # Enable these for now
   enable:
     - depguard
-    - govet
-    - errorlint
-    - exportloopref
 linters-settings:
   depguard:
-    rules:
+    list-type: denylist
-      main:
+    include-go-root: true
-        deny:
+    packages:
-          - pkg: 'encoding/json'
+      - github.com/sirupsen/logrus
-            desc: 'use github.com/segmentio/encoding/json'
+      - golang.org/x/exp
-          - pkg: 'github.com/sirupsen/logrus'
+    packages-with-error-message:
-            desc: 'logging is allowed only by github.com/rs/zerolog'
+      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
-          - pkg: 'golang.org/x/exp'
+    ignore-file-rules:
-            desc: 'exp is not allowed'
+      - "**/*_test.go"
-          - pkg: 'github.com/portainer/libcrypto'
+    # Create additional guards that follow the same configuration pattern.
-            desc: 'use github.com/portainer/portainer/pkg/libcrypto'
+    # Results from all guards are aggregated together.
-          - pkg: 'github.com/portainer/libhttp'
+    # additional-guards:
-            desc: 'use github.com/portainer/portainer/pkg/libhttp'
+    #   - list-type: allowlist
-        files:
+    #     include-go-root: false
-          - '!**/*_test.go'
+    #     packages:
-          - '!**/base.go'
+    #       - github.com/sirupsen/logrus
-          - '!**/base_tx.go'
+    #     # Specify rules by which the linter ignores certain files for consideration.
-
+    #     ignore-file-rules:
-# errorlint is causing a typecheck error for some reason. The go compiler will report these
+    #       - "!**/*_test.go"
-# anyway, so ignore them from the linter
-issues:
-  exclude-rules:
-    - path: ./
-      linters:
-        - typecheck

api/adminmonitor/admin_monitor.go

@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
+	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/rs/zerolog/log"
 )

api/agent/version.go

@@ -4,7 +4,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"strconv"
 	"time"
@@ -43,9 +42,7 @@ func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (port
 	if err != nil {
 		return 0, "", err
 	}
-
+	defer resp.Body.Close()
-	io.Copy(io.Discard, resp.Body)
-	resp.Body.Close()
 
 	if resp.StatusCode != http.StatusNoContent {
 		return 0, "", fmt.Errorf("Failed request with status %d", resp.StatusCode)

api/apikey/apikey.go

@@ -1,6 +1,9 @@
 package apikey
 
 import (
+	"crypto/rand"
+	"io"
+
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -15,3 +18,13 @@ type APIKeyService interface {
 	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
 	InvalidateUserKeyCache(userId portainer.UserID) bool
 }
+
+// generateRandomKey generates a random key of specified length
+// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
+func generateRandomKey(length int) []byte {
+	k := make([]byte, length)
+	if _, err := io.ReadFull(rand.Reader, k); err != nil {
+		return nil
+	}
+	return k
+}

api/apikey/apikey_test.go

@@ -3,7 +3,6 @@ package apikey
 import (
 	"testing"
 
-	"github.com/portainer/portainer/api/internal/securecookie"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -34,7 +33,7 @@ func Test_generateRandomKey(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := securecookie.GenerateRandomKey(tt.wantLenth)
+			got := generateRandomKey(tt.wantLenth)
 			is.Equal(tt.wantLenth, len(got))
 		})
 	}
@@ -42,7 +41,7 @@ func Test_generateRandomKey(t *testing.T) {
 	t.Run("Generated keys are unique", func(t *testing.T) {
 		keys := make(map[string]bool)
 		for i := 0; i < 100; i++ {
-			key := securecookie.GenerateRandomKey(8)
+			key := generateRandomKey(8)
 			_, ok := keys[string(key)]
 			is.False(ok)
 			keys[string(key)] = true

api/apikey/service.go

@@ -8,7 +8,6 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/internal/securecookie"
 
 	"github.com/pkg/errors"
 )
@@ -40,7 +39,7 @@ func (a *apiKeyService) HashRaw(rawKey string) []byte {
 // GenerateApiKey generates a raw API key for a user (for one-time display).
 // The generated API key is stored in the cache and database.
 func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
-	randKey := securecookie.GenerateRandomKey(32)
+	randKey := generateRandomKey(32)
 	encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
 	prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
 
@@ -54,7 +53,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 		Digest:      hashDigest,
 	}
 
-	err := a.apiKeyRepository.Create(apiKey)
+	err := a.apiKeyRepository.CreateAPIKey(apiKey)
 	if err != nil {
 		return "", nil, errors.Wrap(err, "Unable to create API key")
 	}
@@ -67,7 +66,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 
 // GetAPIKey returns an API key by its ID.
 func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	return a.apiKeyRepository.Read(apiKeyID)
+	return a.apiKeyRepository.GetAPIKey(apiKeyID)
 }
 
 // GetAPIKeys returns all the API keys associated to a user.
@@ -89,7 +88,7 @@ func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, port
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve API key")
 	}
 
-	user, err := a.userRepository.Read(apiKey.UserID)
+	user, err := a.userRepository.User(apiKey.UserID)
 	if err != nil {
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve digest user")
 	}
@@ -107,20 +106,20 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
 		return errors.Wrap(err, "Unable to retrieve API key")
 	}
 	a.cache.Set(apiKey.Digest, user, *apiKey)
-	return a.apiKeyRepository.Update(apiKey.ID, apiKey)
+	return a.apiKeyRepository.UpdateAPIKey(apiKey)
 }
 
 // DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
 func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
 	// get api-key digest to remove from cache
-	apiKey, err := a.apiKeyRepository.Read(apiKeyID)
+	apiKey, err := a.apiKeyRepository.GetAPIKey(apiKeyID)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
 	}
 
 	// delete the user/api-key from cache
 	a.cache.Delete(apiKey.Digest)
-	return a.apiKeyRepository.Delete(apiKeyID)
+	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
 }
 
 func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {

api/apikey/service_test.go

@@ -22,7 +22,8 @@ func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
 func Test_GenerateApiKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -75,7 +76,8 @@ func Test_GenerateApiKey(t *testing.T) {
 func Test_GetAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -94,7 +96,8 @@ func Test_GetAPIKey(t *testing.T) {
 func Test_GetAPIKeys(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -114,7 +117,8 @@ func Test_GetAPIKeys(t *testing.T) {
 func Test_GetDigestUserAndKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -149,7 +153,8 @@ func Test_GetDigestUserAndKey(t *testing.T) {
 func Test_UpdateAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -194,7 +199,8 @@ func Test_UpdateAPIKey(t *testing.T) {
 func Test_DeleteAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 
@@ -234,7 +240,8 @@ func Test_DeleteAPIKey(t *testing.T) {
 func Test_InvalidateUserKeyCache(t *testing.T) {
 	is := assert.New(t)
 
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
 

api/archive/targz.go

@@ -3,7 +3,6 @@ package archive
 import (
 	"archive/tar"
 	"compress/gzip"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -85,7 +84,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
 	for {
 		header, err := tarReader.Next()
 
-		if errors.Is(err, io.EOF) {
+		if err == io.EOF {
 			break
 		}
 
@@ -110,7 +109,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
 			}
 			outFile.Close()
 		default:
-			return fmt.Errorf("tar: unknown type: %v in %s",
+			return fmt.Errorf("Tar: uknown type: %v in %s",
 				header.Typeflag,
 				header.Name)
 		}

api/backup/backup.go

@@ -30,7 +30,6 @@ var filesToBackup = []string{
 	"portainer.key",
 	"portainer.pub",
 	"tls",
-	"chisel",
 }
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.

api/chisel/crypto/crypto.go

@@ -1,61 +0,0 @@
-package crypto
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-	"io"
-	"math/big"
-
-	chshare "github.com/jpillora/chisel/share"
-)
-
-var one = new(big.Int).SetInt64(1)
-
-// GenerateGo119CompatibleKey This function is basically copied from chshare.GenerateKey.
-func GenerateGo119CompatibleKey(seed string) ([]byte, error) {
-	r := chshare.NewDetermRand([]byte(seed))
-	priv, err := ecdsaGenerateKey(elliptic.P256(), r)
-	if err != nil {
-		return nil, err
-	}
-	b, err := x509.MarshalECPrivateKey(priv)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to marshal ECDSA private key: %w", err)
-	}
-	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil
-}
-
-// This function is copied from Go1.19
-func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
-	params := c.Params()
-	// Note that for P-521 this will actually be 63 bits more than the order, as
-	// division rounds down, but the extra bit is inconsequential.
-	b := make([]byte, params.N.BitLen()/8+8)
-	_, err = io.ReadFull(rand, b)
-	if err != nil {
-		return
-	}
-
-	k = new(big.Int).SetBytes(b)
-	n := new(big.Int).Sub(params.N, one)
-	k.Mod(k, n)
-	k.Add(k, one)
-	return
-}
-
-// This function is copied from Go1.19
-func ecdsaGenerateKey(c elliptic.Curve, rand io.Reader) (*ecdsa.PrivateKey, error) {
-	k, err := randFieldElement(c, rand)
-	if err != nil {
-		return nil, err
-	}
-
-	priv := new(ecdsa.PrivateKey)
-	priv.PublicKey.Curve = c
-	priv.D = k
-	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
-	return priv, nil
-}

api/chisel/crypto/crypto_test.go

@@ -1,37 +0,0 @@
-package crypto
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestGenerateGo119CompatibleKey(t *testing.T) {
-	type args struct {
-		seed string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []byte
-		wantErr bool
-	}{
-		{
-			name:    "Generate Go 1.19 compatible private key with a given seed",
-			args:    args{seed: "94qh17MCIk8BOkiI"},
-			want:    []byte("-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIHeohwk0Gy3RHVVViaHz7pz/HOiqA7fkv1FTM3mGgfT3oAoGCCqGSM49\nAwEHoUQDQgAEN7riX06xDsLNPuUmOvYFluNEakcFwZZRVvOcIYk/9VYnanDzW0Km\n8/BUUiKyJDuuGdS4fj9SlQ4iL8yBK01uKg==\n-----END EC PRIVATE KEY-----\n"),
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := GenerateGo119CompatibleKey(tt.args.seed)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("GenerateGo119CompatibleKey() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("GenerateGo119CompatibleKey()\ngot: Z %v\nwant: %v", got, tt.want)
-			}
-		})
-	}
-}

api/chisel/schedules.go

@@ -18,7 +18,6 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	for idx, existingJob := range tunnel.Jobs {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
-
 			break
 		}
 	}

api/chisel/service.go

@@ -3,7 +3,6 @@ package chisel
 import (
 	"context"
 	"fmt"
-	"io"
 	"net/http"
 	"sync"
 	"time"
@@ -12,8 +11,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 
+	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
-	"github.com/jpillora/chisel/share/ccrypto"
 	"github.com/rs/zerolog/log"
 )
 
@@ -36,16 +35,14 @@ type Service struct {
 	shutdownCtx       context.Context
 	ProxyManager      *proxy.Manager
 	mu                sync.Mutex
-	fileService       portainer.FileService
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
+func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
 	return &Service{
 		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
 		dataStore:        dataStore,
 		shutdownCtx:      shutdownCtx,
-		fileService:      fileService,
 	}
 }
 
@@ -61,11 +58,7 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 	httpClient := &http.Client{
 		Timeout: 3 * time.Second,
 	}
-
+	_, err = httpClient.Do(req)
-	resp, err := httpClient.Do(req)
-	io.Copy(io.Discard, resp.Body)
-	resp.Body.Close()
-
 	return err
 }
 
@@ -75,11 +68,10 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
 			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("KeepTunnelAlive: start")
+			Msg("start")
 
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
-
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
 		defer pingTicker.Stop()
 
@@ -92,13 +84,13 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 					log.Debug().
 						Int("endpoint_id", int(endpointID)).
 						Err(err).
-						Msg("KeepTunnelAlive: ping agent")
+						Msg("ping agent")
 				}
 			case <-maxAliveTicker.C:
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("KeepTunnelAlive: tunnel keep alive timeout")
+					Msg("tunnel keep alive timeout")
 
 				return
 			case <-ctx.Done():
@@ -106,7 +98,7 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("KeepTunnelAlive: tunnel stop")
+					Msg("tunnel stop")
 
 				return
 			}
@@ -120,15 +112,14 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // It starts the tunnel status verification process in the background.
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
-	privateKeyFile, err := service.retrievePrivateKeyFile()
+	keySeed, err := service.retrievePrivateKeySeed()
-
 	if err != nil {
 		return err
 	}
 
 	config := &chserver.Config{
 		Reverse: true,
-		KeyFile: privateKeyFile,
+		KeySeed: keySeed,
 	}
 
 	chiselServer, err := chserver.NewServer(config)
@@ -164,41 +155,26 @@ func (service *Service) StopTunnelServer() error {
 	return service.chiselServer.Close()
 }
 
-func (service *Service) retrievePrivateKeyFile() (string, error) {
+func (service *Service) retrievePrivateKeySeed() (string, error) {
-	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
+	var serverInfo *portainer.TunnelServerInfo
 
-	exist, _ := service.fileService.FileExists(privateKeyFile)
+	serverInfo, err := service.dataStore.TunnelServer().Info()
-	if !exist {
+	if service.dataStore.IsErrObjectNotFound(err) {
-		log.Debug().
+		keySeed := uniuri.NewLen(16)
-			Str("private-key", privateKeyFile).
-			Msg("Chisel private key file does not exist")
 
-		privateKey, err := ccrypto.GenerateKey("")
+		serverInfo = &portainer.TunnelServerInfo{
-		if err != nil {
+			PrivateKeySeed: keySeed,
-			log.Error().
-				Err(err).
-				Msg("Failed to generate chisel private key")
-			return "", err
 		}
 
-		err = service.fileService.StoreChiselPrivateKey(privateKey)
+		err := service.dataStore.TunnelServer().UpdateInfo(serverInfo)
 		if err != nil {
-			log.Error().
-				Err(err).
-				Msg("Failed to save Chisel private key to disk")
 			return "", err
-		} else {
-			log.Info().
-				Str("private-key", privateKeyFile).
-				Msg("Generated a new Chisel private key file")
 		}
-	} else {
+	} else if err != nil {
-		log.Info().
+		return "", err
-			Str("private-key", privateKeyFile).
-			Msg("Found Chisel private key file on disk")
 	}
 
-	return privateKeyFile, nil
+	return serverInfo.PrivateKeySeed, nil
 }
 
 func (service *Service) startTunnelVerificationLoop() {
@@ -290,7 +266,14 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 		return err
 	}
 
-	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
+	endpointURL := endpoint.URL
 
-	return service.snapshotService.SnapshotEndpoint(endpoint)
+	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
+	err = service.snapshotService.SnapshotEndpoint(endpoint)
+	if err != nil {
+		return err
+	}
+
+	endpoint.URL = endpointURL
+	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 }

api/chisel/tunnel.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/portainer/libcrypto"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge/cache"
-	"github.com/portainer/portainer/pkg/libcrypto"
 
 	"github.com/dchest/uniuri"
 )
@@ -126,10 +126,7 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 	credentials := tunnel.Credentials
 	if credentials != "" {
 		tunnel.Credentials = ""
-
+		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
-		if service.chiselServer != nil {
-			service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
-		}
 	}
 
 	service.ProxyManager.DeleteEndpointProxy(endpointID)
@@ -164,12 +161,9 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
 
 		username, password := generateRandomCredentials()
 		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-
+		err = service.chiselServer.AddUser(username, password, authorizedRemote)
-		if service.chiselServer != nil {
+		if err != nil {
-			err = service.chiselServer.AddUser(username, password, authorizedRemote)
+			return err
-			if err != nil {
-				return err
-			}
 		}
 
 		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)

api/cli/cli.go

@@ -49,7 +49,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
@@ -72,7 +72,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		if err != nil {
 			panic(err)
 		}
-
 		*flags.Assets = filepath.Join(filepath.Dir(ex), *flags.Assets)
 	}
 
@@ -81,6 +80,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 // ValidateFlags validates the values of the flags.
 func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
+
 	displayDeprecationWarnings(flags)
 
 	err := validateEndpointURL(*flags.EndpointURL)
@@ -111,38 +111,31 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 }
 
 func validateEndpointURL(endpointURL string) error {
-	if endpointURL == "" {
+	if endpointURL != "" {
-		return nil
+		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
-	}
+			return errInvalidEndpointProtocol
+		}
 
-	if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
+		if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
-		return errInvalidEndpointProtocol
+			socketPath := strings.TrimPrefix(endpointURL, "unix://")
-	}
+			socketPath = strings.TrimPrefix(socketPath, "npipe://")
-
+			if _, err := os.Stat(socketPath); err != nil {
-	if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
+				if os.IsNotExist(err) {
-		socketPath := strings.TrimPrefix(endpointURL, "unix://")
+					return errSocketOrNamedPipeNotFound
-		socketPath = strings.TrimPrefix(socketPath, "npipe://")
+				}
-		if _, err := os.Stat(socketPath); err != nil {
+				return err
-			if os.IsNotExist(err) {
-				return errSocketOrNamedPipeNotFound
 			}
-
-			return err
 		}
 	}
-
 	return nil
 }
 
 func validateSnapshotInterval(snapshotInterval string) error {
-	if snapshotInterval == "" {
+	if snapshotInterval != "" {
-		return nil
+		_, err := time.ParseDuration(snapshotInterval)
+		if err != nil {
+			return errInvalidSnapshotInterval
+		}
 	}
-
-	_, err := time.ParseDuration(snapshotInterval)
-	if err != nil {
-		return errInvalidSnapshotInterval
-	}
-
 	return nil
 }

api/cli/confirm.go

@@ -12,14 +12,13 @@ func Confirm(message string) (bool, error) {
 	fmt.Printf("%s [y/N]", message)
 
 	reader := bufio.NewReader(os.Stdin)
-
 	answer, err := reader.ReadString('\n')
 	if err != nil {
 		return false, err
 	}
-
+	answer = strings.Replace(answer, "\n", "", -1)
-	answer = strings.ReplaceAll(answer, "\n", "")
 	answer = strings.ToLower(answer)
 
 	return answer == "y" || answer == "yes", nil
+
 }

api/cmd/portainer/log.go

@@ -39,9 +39,9 @@ func setLoggingMode(mode string) {
 	case "PRETTY":
 		log.Logger = log.Output(zerolog.ConsoleWriter{
 			Out:           os.Stderr,
+			NoColor:       true,
 			TimeFormat:    "2006/01/02 03:04PM",
-			FormatMessage: formatMessage,
+			FormatMessage: formatMessage})
-		})
 	case "JSON":
 		log.Logger = log.Output(os.Stderr)
 	}
@@ -51,6 +51,5 @@ func formatMessage(i interface{}) string {
 	if i == nil {
 		return ""
 	}
-
 	return fmt.Sprintf("%s |", i)
 }

api/cmd/portainer/main.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"time"
 
+	libstack "github.com/portainer/docker-compose-wrapper"
+	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/build"
@@ -20,21 +22,19 @@ import (
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
-	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
-	dockerclient "github.com/portainer/portainer/api/docker/client"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/hostmanagement/openamt"
 	"github.com/portainer/portainer/api/http"
+	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/edgestacks"
-	"github.com/portainer/portainer/api/internal/endpointutils"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/internal/upgrade"
@@ -43,13 +43,10 @@ import (
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
-	"github.com/portainer/portainer/api/pendingactions"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/pkg/featureflags"
 	"github.com/portainer/portainer/pkg/libhelm"
-	"github.com/portainer/portainer/pkg/libstack"
-	"github.com/portainer/portainer/pkg/libstack/compose"
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
@@ -121,15 +118,11 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 			log.Fatal().Err(err).Msg("failed generating instance id")
 		}
 
-		migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{})
-		migratorCount := migratorInstance.GetMigratorCountOfCurrentAPIVersion()
-
 		// from MigrateData
 		v := models.Version{
 			SchemaVersion: portainer.APIVersion,
 			Edition:       int(portainer.PortainerCE),
 			InstanceID:    instanceId.String(),
-			MigratorCount: migratorCount,
 		}
 		store.VersionService.UpdateVersion(&v)
 
@@ -158,17 +151,7 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	return store
 }
 
-// checkDBSchemaServerVersionMatch checks if the server version matches the db scehma version
+func initComposeStackManager(composeDeployer libstack.Deployer, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersion string, serverEdition int) bool {
-	v, err := dbStore.Version().Version()
-	if err != nil {
-		return false
-	}
-
-	return v.SchemaVersion == serverVersion && v.Edition == serverEdition
-}
-
-func initComposeStackManager(composeDeployer libstack.Deployer, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating compose manager")
@@ -200,7 +183,7 @@ func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
 	return apikey.NewAPIKeyService(datastore.APIKeyRepository(), datastore.User())
 }
 
-func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (portainer.JWTService, error) {
+func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (dataservices.JWTService, error) {
 	if userSessionTimeout == "" {
 		userSessionTimeout = portainer.DefaultUserSessionTimeout
 	}
@@ -250,8 +233,8 @@ func initSSLService(addr, certPath, keyPath string, fileService portainer.FileSe
 	return sslService, nil
 }
 
-func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *dockerclient.ClientFactory {
+func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
-	return dockerclient.NewClientFactory(signatureService, reverseTunnelService)
+	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
 func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
@@ -261,15 +244,14 @@ func initKubernetesClientFactory(signatureService portainer.DigitalSignatureServ
 func initSnapshotService(
 	snapshotIntervalFromFlag string,
 	dataStore dataservices.DataStore,
-	dockerClientFactory *dockerclient.ClientFactory,
+	dockerClientFactory *docker.ClientFactory,
 	kubernetesClientFactory *kubecli.ClientFactory,
 	shutdownCtx context.Context,
-	pendingActionsService *pendingactions.PendingActionsService,
 ) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx, pendingActionsService)
+	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -364,6 +346,147 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
+	tlsConfiguration := portainer.TLSConfiguration{
+		TLS:           *flags.TLS,
+		TLSSkipVerify: *flags.TLSSkipVerify,
+	}
+
+	if *flags.TLS {
+		tlsConfiguration.TLSCACertPath = *flags.TLSCacert
+		tlsConfiguration.TLSCertPath = *flags.TLSCert
+		tlsConfiguration.TLSKeyPath = *flags.TLSKey
+	} else if !*flags.TLS && *flags.TLSSkipVerify {
+		tlsConfiguration.TLS = true
+	}
+
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                *flags.EndpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          tlsConfiguration,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             []portainer.TagID{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
+	}
+
+	if strings.HasPrefix(endpoint.URL, "tcp://") {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
+		if err != nil {
+			return err
+		}
+
+		agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig)
+		if err != nil {
+			return err
+		}
+
+		if agentOnDockerEnvironment {
+			endpoint.Type = portainer.AgentOnDockerEnvironment
+		}
+	}
+
+	err := snapshotService.SnapshotEndpoint(endpoint)
+	if err != nil {
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).
+			Err(err).
+			Msg("environment snapshot error")
+	}
+
+	return dataStore.Endpoint().Create(endpoint)
+}
+
+func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
+	if strings.HasPrefix(endpointURL, "tcp://") {
+		_, err := client.ExecutePingOperation(endpointURL, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                endpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          portainer.TLSConfiguration{},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             []portainer.TagID{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
+	}
+
+	err := snapshotService.SnapshotEndpoint(endpoint)
+	if err != nil {
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).Err(err).
+			Msg("environment snapshot error")
+	}
+
+	return dataStore.Endpoint().Create(endpoint)
+}
+
+func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
+	if *flags.EndpointURL == "" {
+		return nil
+	}
+
+	endpoints, err := dataStore.Endpoint().Endpoints()
+	if err != nil {
+		return err
+	}
+
+	if len(endpoints) > 0 {
+		log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
+
+		return nil
+	}
+
+	if *flags.TLS || *flags.TLSSkipVerify {
+		return createTLSSecuredEndpoint(flags, dataStore, snapshotService)
+	}
+	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
+}
+
 func loadEncryptionSecretKey(keyfilename string) []byte {
 	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
 	if err != nil {
@@ -400,11 +523,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("")
 	}
 
-	// check if the db schema version matches with server version
-	if !checkDBSchemaServerVersionMatch(dataStore, portainer.APIVersion, int(portainer.Edition)) {
-		log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
-	}
-
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed getting instance id")
@@ -451,22 +569,20 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
-	authorizationService := authorization.NewService(dataStore)
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
-	authorizationService.K8sClientFactory = kubernetesClientFactory
-
-	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
-
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing snapshot service")
 	}
 	snapshotService.Start()
 
+	authorizationService := authorization.NewService(dataStore)
+	authorizationService.K8sClientFactory = kubernetesClientFactory
+
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
@@ -482,7 +598,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing compose deployer")
 	}
 
-	composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
+	composeStackManager := initComposeStackManager(composeDeployer, reverseTunnelService, proxyManager)
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
@@ -511,10 +627,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		}
 	}
 
-	// channel to control when the admin user is created
+	err = initEndpoint(flags, dataStore, snapshotService)
-	adminCreationDone := make(chan struct{}, 1)
+	if err != nil {
-
+		log.Fatal().Err(err).Msg("failed initializing environment")
-	go endpointutils.InitEndpoint(shutdownCtx, adminCreationDone, flags, dataStore, snapshotService)
+	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
@@ -549,9 +665,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 			if err != nil {
 				log.Fatal().Err(err).Msg("failed creating admin user")
 			}
-
-			// notify the admin user is created, the endpoint initialization can start
-			adminCreationDone <- struct{}{}
 		} else {
 			log.Info().Msg("instance already has an administrator user defined, skipping admin password related flags.")
 		}
@@ -563,7 +676,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer, dockerClientFactory, dataStore)
+	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
 	deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
@@ -571,7 +684,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
-	upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer, kubernetesClientFactory)
+	upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing upgrade service")
 	}
@@ -625,8 +738,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		StackDeployer:               stackDeployer,
 		DemoService:                 demoService,
 		UpgradeService:              upgradeService,
-		AdminCreationDone:           adminCreationDone,
-		PendingActionsService:       pendingActionsService,
 	}
 }
 

api/crypto/ecdsa.go

@@ -7,8 +7,9 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/hex"
+	"math/big"
 
-	"github.com/portainer/portainer/pkg/libcrypto"
+	"github.com/portainer/libcrypto"
 )
 
 const (
@@ -114,6 +115,9 @@ func (service *ECDSAService) CreateSignature(message string) (string, error) {
 
 	hash := libcrypto.HashFromBytes([]byte(message))
 
+	r := big.NewInt(0)
+	s := big.NewInt(0)
+
 	r, s, err := ecdsa.Sign(rand.Reader, service.privateKey, hash)
 	if err != nil {
 		return "", err

api/crypto/tls.go

@@ -20,8 +20,6 @@ func CreateTLSConfiguration() *tls.Config {
 			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
 			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
 			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
-			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
-			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
 		},
 	}
 }

api/database/boltdb/db.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"os"
 	"path"
 	"time"
@@ -183,7 +182,7 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
 func (connection *DbConnection) ExportRaw(filename string) error {
 	databasePath := connection.GetDatabaseFilePath()
 	if _, err := os.Stat(databasePath); err != nil {
-		return fmt.Errorf("stat on %s failed, error: %w", databasePath, err)
+		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
 	b, err := connection.ExportJSON(databasePath, true)
@@ -202,20 +201,6 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 	return b
 }
 
-// keyToString Converts a key to a string value suitable for logging
-func keyToString(b []byte) string {
-	if len(b) != 8 {
-		return string(b)
-	}
-
-	v := binary.BigEndian.Uint64(b)
-	if v <= math.MaxInt32 {
-		return fmt.Sprintf("%d", v)
-	}
-
-	return string(b)
-}
-
 // CreateBucket is a generic function used to create a bucket inside a database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
 	return connection.UpdateTx(func(tx portainer.Transaction) error {
@@ -252,10 +237,10 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 
 		data := bucket.Get(key)
 		if data == nil {
-			return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
+			return dserrors.ErrObjectNotFound
 		}
 
-		err := connection.UnmarshalObject(data, object)
+		err := connection.UnmarshalObjectWithJsoniter(data, object)
 		if err != nil {
 			return err
 		}

api/database/boltdb/export.go

@@ -1,10 +1,10 @@
 package boltdb
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 	bolt "go.etcd.io/bbolt"
 )
 

api/database/boltdb/json.go

@@ -1,41 +1,34 @@
 package boltdb
 
 import (
-	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
-	"github.com/segmentio/encoding/json"
 )
 
 var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
 
 // MarshalObject encodes an object to binary format
-func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error) {
+func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
-	buf := &bytes.Buffer{}
-
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		buf.WriteString(v)
+		data = []byte(v)
 	} else {
-		enc := json.NewEncoder(buf)
+		data, err = json.Marshal(object)
-		enc.SetSortMapKeys(false)
+		if err != nil {
-		enc.SetAppendNewline(false)
+			return data, err
-
-		if err := enc.Encode(object); err != nil {
-			return nil, err
 		}
 	}
-
 	if connection.getEncryptionKey() == nil {
-		return buf.Bytes(), nil
+		return data, nil
 	}
-
+	return encrypt(data, connection.getEncryptionKey())
-	return encrypt(buf.Bytes(), connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
@@ -61,6 +54,31 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
 	return err
 }
 
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// decoding at the moment.
+func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	if connection.getEncryptionKey() != nil {
+		var err error
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return err
+		}
+	}
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	err := jsoni.Unmarshal(data, &object)
+	if err != nil {
+		if s, ok := object.(*string); ok {
+			*s = string(data)
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
+}
+
 // mmm, don't have a KMS .... aes GCM seems the most likely from
 // https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
 

api/database/boltdb/json_test.go

@@ -129,7 +129,7 @@ func Test_UnMarshalObjectUnencrypted(t *testing.T) {
 			var object string
 			err := conn.UnmarshalObject(test.object, &object)
 			is.NoError(err)
-			is.Equal(test.expected, object)
+			is.Equal(test.expected, string(object))
 		})
 	}
 }

api/database/boltdb/tx.go

@@ -2,7 +2,6 @@ package boltdb
 
 import (
 	"bytes"
-	"fmt"
 
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
@@ -25,10 +24,13 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 
 	value := bucket.Get(key)
 	if value == nil {
-		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
+		return dserrors.ErrObjectNotFound
 	}
 
-	return tx.conn.UnmarshalObject(value, object)
+	data := make([]byte, len(value))
+	copy(data, value)
+
+	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -46,9 +48,7 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
 	return bucket.Delete(key)
 }
 
-func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
-	var ids []int
-
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	cursor := bucket.Cursor()
@@ -58,14 +58,11 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 			return err
 		}
 
-		if id, ok := matchingFn(obj); ok {
+		if id, ok := matching(obj); ok {
-			ids = append(ids, id)
+			err := bucket.Delete(tx.conn.ConvertToKey(id))
-		}
+			if err != nil {
-	}
+				return err
-
+			}
-	for _, id := range ids {
-		if err := bucket.Delete(tx.conn.ConvertToKey(id)); err != nil {
-			return err
 		}
 	}
 
@@ -77,6 +74,7 @@ func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
 	id, err := bucket.NextSequence()
 	if err != nil {
 		log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifer")
+
 		return 0
 	}
 
@@ -94,7 +92,7 @@ func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, i
 		return err
 	}
 
-	return bucket.Put(tx.conn.ConvertToKey(id), data)
+	return bucket.Put(tx.conn.ConvertToKey(int(id)), data)
 }
 
 func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
@@ -117,42 +115,54 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
 	return bucket.Put(id, data)
 }
 
-func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	return bucket.ForEach(func(k []byte, v []byte) error {
+	cursor := bucket.Cursor()
-		err := tx.conn.UnmarshalObject(v, obj)
+	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-		if err == nil {
-			obj, err = appendFn(obj)
-		}
-
-		return err
-	})
-}
-
-func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
-	bucket := tx.tx.Bucket([]byte(bucketName))
-
-	return bucket.ForEach(func(k []byte, v []byte) error {
-		err := tx.conn.UnmarshalObject(v, obj)
-		if err == nil {
-			obj, err = appendFn(obj)
-		}
-
-		return err
-	})
-}
-
-func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
-	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
-
-	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
 		err := tx.conn.UnmarshalObject(v, obj)
 		if err != nil {
 			return err
 		}
 
-		obj, err = appendFn(obj)
+		obj, err = append(obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	cursor := bucket.Cursor()
+	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		if err != nil {
+			return err
+		}
+
+		obj, err = append(obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
+
+	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		if err != nil {
+			return err
+		}
+
+		obj, err = append(obj)
 		if err != nil {
 			return err
 		}

api/database/boltdb/tx_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
 const testBucketName = "test-bucket"
@@ -97,7 +97,7 @@ func TestTxs(t *testing.T) {
 	err = conn.ViewTx(func(tx portainer.Transaction) error {
 		return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
 	})
-	if !dataservices.IsErrObjectNotFound(err) {
+	if err != dserrors.ErrObjectNotFound {
 		t.Fatal(err)
 	}
 

api/database/database.go

@@ -9,7 +9,8 @@ import (
 
 // NewDatabase should use config options to return a connection to the requested database
 func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
-	if storeType == "boltdb" {
+	switch storeType {
+	case "boltdb":
 		return &boltdb.DbConnection{
 			Path:          storePath,
 			EncryptionKey: encryptionKey,

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -2,22 +2,22 @@ package apikeyrepository
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/dataservices/errors"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "api_key"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "api_key"
+)
 
 // Service represents a service for managing api-key data.
 type Service struct {
-	dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]
+	connection portainer.Connection
 }
 
 // NewService creates a new instance of a service.
@@ -28,10 +28,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
@@ -39,14 +36,14 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error) {
 	var result = make([]portainer.APIKey, 0)
 
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
 
 			if record.UserID == userID {
@@ -64,14 +61,14 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
 func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error) {
 	var k *portainer.APIKey
 	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			key, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
 			if bytes.Equal(key.Digest, digest) {
 				k = key
@@ -81,20 +78,20 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 			return &portainer.APIKey{}, nil
 		})
 
-	if errors.Is(err, stop) {
+	if err == stop {
 		return k, nil
 	}
 
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
 }
 
-// Create creates a new APIKey object.
+// CreateAPIKey creates a new APIKey object.
-func (service *Service) Create(record *portainer.APIKey) error {
+func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.APIKeyID(id)
@@ -103,3 +100,26 @@ func (service *Service) Create(record *portainer.APIKey) error {
 		},
 	)
 }
+
+// GetAPIKey retrieves an existing APIKey object by api key ID.
+func (service *Service) GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error) {
+	var key portainer.APIKey
+	identifier := service.connection.ConvertToKey(int(keyID))
+
+	err := service.connection.GetObject(BucketName, identifier, &key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &key, nil
+}
+
+func (service *Service) UpdateAPIKey(key *portainer.APIKey) error {
+	identifier := service.connection.ConvertToKey(int(key.ID))
+	return service.connection.UpdateObject(BucketName, identifier, key)
+}
+
+func (service *Service) DeleteAPIKey(ID portainer.APIKeyID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/base.go

@@ -1,66 +0,0 @@
-package dataservices
-
-import (
-	portainer "github.com/portainer/portainer/api"
-
-	"golang.org/x/exp/constraints"
-)
-
-type BaseCRUD[T any, I constraints.Integer] interface {
-	Create(element *T) error
-	Read(ID I) (*T, error)
-	ReadAll() ([]T, error)
-	Update(ID I, element *T) error
-	Delete(ID I) error
-}
-
-type BaseDataService[T any, I constraints.Integer] struct {
-	Bucket     string
-	Connection portainer.Connection
-}
-
-func (s *BaseDataService[T, I]) BucketName() string {
-	return s.Bucket
-}
-
-func (service *BaseDataService[T, I]) Tx(tx portainer.Transaction) BaseDataServiceTx[T, I] {
-	return BaseDataServiceTx[T, I]{
-		Bucket:     service.Bucket,
-		Connection: service.Connection,
-		Tx:         tx,
-	}
-}
-
-func (service BaseDataService[T, I]) Read(ID I) (*T, error) {
-	var element *T
-
-	return element, service.Connection.ViewTx(func(tx portainer.Transaction) error {
-		var err error
-		element, err = service.Tx(tx).Read(ID)
-
-		return err
-	})
-}
-
-func (service BaseDataService[T, I]) ReadAll() ([]T, error) {
-	var collection = make([]T, 0)
-
-	return collection, service.Connection.ViewTx(func(tx portainer.Transaction) error {
-		var err error
-		collection, err = service.Tx(tx).ReadAll()
-
-		return err
-	})
-}
-
-func (service BaseDataService[T, I]) Update(ID I, element *T) error {
-	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
-		return service.Tx(tx).Update(ID, element)
-	})
-}
-
-func (service BaseDataService[T, I]) Delete(ID I) error {
-	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
-		return service.Tx(tx).Delete(ID)
-	})
-}

api/dataservices/base_tx.go

@@ -1,49 +0,0 @@
-package dataservices
-
-import (
-	portainer "github.com/portainer/portainer/api"
-
-	"golang.org/x/exp/constraints"
-)
-
-type BaseDataServiceTx[T any, I constraints.Integer] struct {
-	Bucket     string
-	Connection portainer.Connection
-	Tx         portainer.Transaction
-}
-
-func (service BaseDataServiceTx[T, I]) BucketName() string {
-	return service.Bucket
-}
-
-func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
-	var element T
-	identifier := service.Connection.ConvertToKey(int(ID))
-
-	err := service.Tx.GetObject(service.Bucket, identifier, &element)
-	if err != nil {
-		return nil, err
-	}
-
-	return &element, nil
-}
-
-func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
-	var collection = make([]T, 0)
-
-	return collection, service.Tx.GetAllWithJsoniter(
-		service.Bucket,
-		new(T),
-		AppendFn(&collection),
-	)
-}
-
-func (service BaseDataServiceTx[T, I]) Update(ID I, element *T) error {
-	identifier := service.Connection.ConvertToKey(int(ID))
-	return service.Tx.UpdateObject(service.Bucket, identifier, element)
-}
-
-func (service BaseDataServiceTx[T, I]) Delete(ID I) error {
-	identifier := service.Connection.ConvertToKey(int(ID))
-	return service.Tx.DeleteObject(service.Bucket, identifier)
-}

api/dataservices/customtemplate/customtemplate.go

@@ -1,16 +1,25 @@
 package customtemplate
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "customtemplates"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "customtemplates"
+)
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,20 +30,64 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
+// CustomTemplates return an array containing all the custom templates.
+func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
+	var customTemplates = make([]portainer.CustomTemplate, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.CustomTemplate{},
+		func(obj interface{}) (interface{}, error) {
+			//var tag portainer.Tag
+			customTemplate, ok := obj.(*portainer.CustomTemplate)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
+				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
+			}
+			customTemplates = append(customTemplates, *customTemplate)
+
+			return &portainer.CustomTemplate{}, nil
+		})
+
+	return customTemplates, err
+}
+
+// CustomTemplate returns an custom template by ID.
+func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
+	var customTemplate portainer.CustomTemplate
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &customTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	return &customTemplate, nil
+}
+
+// UpdateCustomTemplate updates an custom template.
+func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, customTemplate)
+}
+
+// DeleteCustomTemplate deletes an custom template.
+func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
 // CreateCustomTemplate uses the existing id and saves it.
 // TODO: where does the ID come from, and is it safe?
 func (service *Service) Create(customTemplate *portainer.CustomTemplate) error {
-	return service.Connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
+	return service.connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
 }
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return service.Connection.GetNextIdentifier(BucketName)
+	return service.connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgegroup/edgegroup.go

@@ -2,7 +2,6 @@ package edgegroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -10,7 +9,7 @@ const BucketName = "edgegroups"
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -25,36 +24,69 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]{
+		service: service,
-			Bucket:     BucketName,
+		tx:      tx,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
 	}
 }
 
+// EdgeGroups return a slice containing all the Edge groups.
+func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
+	var groups []portainer.EdgeGroup
+	var err error
+
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		groups, err = service.Tx(tx).EdgeGroups()
+		return err
+	})
+
+	return groups, err
+}
+
+// EdgeGroup returns an Edge group by ID.
+func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
+	var group *portainer.EdgeGroup
+	var err error
+
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		group, err = service.Tx(tx).EdgeGroup(ID)
+		return err
+	})
+
+	return group, err
+}
+
+// UpdateEdgeGroup updates an edge group.
+func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, group)
+}
+
 // Deprecated: UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
-	id := service.Connection.ConvertToKey(int(ID))
+	id := service.connection.ConvertToKey(int(ID))
 	edgeGroup := &portainer.EdgeGroup{}
 
-	return service.Connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
 		updateFunc(edgeGroup)
 	})
 }
 
+// DeleteEdgeGroup deletes an Edge group.
+func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).DeleteEdgeGroup(ID)
+	})
+}
+
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) Create(group *portainer.EdgeGroup) error {
-	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
 		return service.Tx(tx).Create(group)
 	})
 }

api/dataservices/edgegroup/tx.go

@@ -2,13 +2,60 @@ package edgegroup
 
 import (
 	"errors"
+	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EdgeGroups return a slice containing all the Edge groups.
+func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
+	var groups = make([]portainer.EdgeGroup, 0)
+
+	err := service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.EdgeGroup{},
+		func(obj interface{}) (interface{}, error) {
+			group, ok := obj.(*portainer.EdgeGroup)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
+				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
+			}
+			groups = append(groups, *group)
+
+			return &portainer.EdgeGroup{}, nil
+		})
+
+	return groups, err
+}
+
+// EdgeGroup returns an Edge group by ID.
+func (service ServiceTx) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
+	var group portainer.EdgeGroup
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &group)
+	if err != nil {
+		return nil, err
+	}
+
+	return &group, nil
+}
+
+// UpdateEdgeGroup updates an edge group.
+func (service ServiceTx) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, group)
 }
 
 // UpdateEdgeGroupFunc is a no-op inside a transaction.
@@ -16,8 +63,14 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
 	return errors.New("cannot be called inside a transaction")
 }
 
+// DeleteEdgeGroup deletes an Edge group.
+func (service ServiceTx) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
 func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
-	return service.Tx.CreateObject(
+	return service.tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			group.ID = portainer.EdgeGroupID(id)

api/dataservices/edgejob/edgejob.go

@@ -1,8 +1,11 @@
 package edgejob
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -10,7 +13,11 @@ const BucketName = "edgejobs"
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,50 +28,86 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]{
+		service: service,
-			Bucket:     BucketName,
+		tx:      tx,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
 	}
 }
 
-// Create creates a new EdgeJob
+// EdgeJobs returns a list of Edge jobs
-func (service *Service) Create(edgeJob *portainer.EdgeJob) error {
+func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
-	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.EdgeJob{},
+		func(obj interface{}) (interface{}, error) {
+			job, ok := obj.(*portainer.EdgeJob)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
+				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
+			}
+
+			edgeJobs = append(edgeJobs, *job)
+
+			return &portainer.EdgeJob{}, nil
+		})
+
+	return edgeJobs, err
 }
 
-// CreateWithID creates a new EdgeJob
+// EdgeJob returns an Edge job by ID
-func (service *Service) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
+}
+
+// Create creates a new EdgeJob
+func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	edgeJob.ID = ID
 
-	return service.Connection.CreateObjectWithId(
+	return service.connection.CreateObjectWithId(
 		BucketName,
 		int(edgeJob.ID),
 		edgeJob,
 	)
 }
 
+// Deprecated: use UpdateEdgeJobFunc instead
+func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
+}
+
 // UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
-	id := service.Connection.ConvertToKey(int(ID))
+	id := service.connection.ConvertToKey(int(ID))
 	edgeJob := &portainer.EdgeJob{}
 
-	return service.Connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
 		updateFunc(edgeJob)
 	})
 }
 
+// DeleteEdgeJob deletes an Edge job
+func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
-	return service.Connection.GetNextIdentifier(BucketName)
+	return service.connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgejob/tx.go

@@ -2,25 +2,68 @@ package edgejob
 
 import (
 	"errors"
+	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EdgeJobs returns a list of Edge jobs
+func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EdgeJob{},
+		func(obj interface{}) (interface{}, error) {
+			job, ok := obj.(*portainer.EdgeJob)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
+				return nil, fmt.Errorf("failed to convert to EdgeJob object: %s", obj)
+			}
+
+			edgeJobs = append(edgeJobs, *job)
+
+			return &portainer.EdgeJob{}, nil
+		})
+
+	return edgeJobs, err
+}
+
+// EdgeJob returns an Edge job by ID
+func (service ServiceTx) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
 }
 
 // Create creates a new EdgeJob
-func (service ServiceTx) Create(edgeJob *portainer.EdgeJob) error {
+func (service ServiceTx) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
-}
-
-// CreateWithID creates a new EdgeJob
-func (service ServiceTx) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	edgeJob.ID = ID
 
-	return service.Tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+	return service.tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+}
+
+// UpdateEdgeJob updates an edge job
+func (service ServiceTx) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, edgeJob)
 }
 
 // UpdateEdgeJobFunc is a no-op inside a transaction.
@@ -28,7 +71,14 @@ func (service ServiceTx) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc fu
 	return errors.New("cannot be called inside a transaction")
 }
 
+// DeleteEdgeJob deletes an Edge job
+func (service ServiceTx) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
-	return service.Tx.GetNextIdentifier(BucketName)
+	return service.tx.GetNextIdentifier(BucketName)
 }

api/dataservices/edgestack/edgestack.go

@@ -1,10 +1,12 @@
 package edgestack
 
 import (
+	"fmt"
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -62,11 +64,22 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	return stacks, service.connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.EdgeStack{},
-		dataservices.AppendFn(&stacks),
+		func(obj interface{}) (interface{}, error) {
-	)
+			stack, ok := obj.(*portainer.EdgeStack)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
+				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
+			}
+
+			stacks = append(stacks, *stack)
+
+			return &portainer.EdgeStack{}, nil
+		})
+
+	return stacks, err
 }
 
 // EdgeStack returns an Edge stack by ID.
@@ -146,11 +159,6 @@ func (service *Service) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc
 	})
 }
 
-// UpdateEdgeStackFuncTx is a helper function used to call UpdateEdgeStackFunc inside a transaction.
-func (service *Service) UpdateEdgeStackFuncTx(tx portainer.Transaction, ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
-	return service.Tx(tx).UpdateEdgeStackFunc(ID, updateFunc)
-}
-
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	service.mu.Lock()

api/dataservices/edgestack/tx.go

@@ -1,6 +1,7 @@
 package edgestack
 
 import (
+	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
@@ -28,7 +29,7 @@ func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
 			stack, ok := obj.(*portainer.EdgeStack)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
-				return nil, fmt.Errorf("failed to convert to EdgeStack object: %s", obj)
+				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
 			}
 
 			stacks = append(stacks, *stack)
@@ -100,16 +101,9 @@ func (service ServiceTx) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *po
 	return nil
 }
 
-// Deprecated: use UpdateEdgeStack inside a transaction instead.
+// UpdateEdgeStackFunc is a no-op inside a transaction.
 func (service ServiceTx) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
-	edgeStack, err := service.EdgeStack(ID)
+	return errors.New("cannot be called inside a transaction")
-	if err != nil {
-		return err
-	}
-
-	updateFunc(edgeStack)
-
-	return service.UpdateEdgeStack(ID, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.

api/dataservices/endpoint/endpoint.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -35,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 		idxEdgeID:  make(map[string]portainer.EndpointID),
 	}
 
-	es, err := s.endpoints()
+	es, err := s.Endpoints()
 	if err != nil {
 		return nil, err
 	}
@@ -90,7 +89,8 @@ func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	})
 }
 
-func (service *Service) endpoints() ([]portainer.Endpoint, error) {
+// Endpoints return an array containing all the environments(endpoints).
+func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints []portainer.Endpoint
 	var err error
 
@@ -99,14 +99,8 @@ func (service *Service) endpoints() ([]portainer.Endpoint, error) {
 		return err
 	})
 
-	return endpoints, err
-}
-
-// Endpoints return an array containing all the environments(endpoints).
-func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
-	endpoints, err := service.endpoints()
 	if err != nil {
-		return nil, err
+		return endpoints, err
 	}
 
 	for i, e := range endpoints {
@@ -145,23 +139,6 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
-func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
-
-	return endpoints, service.connection.GetAll(
-		BucketName,
-		&portainer.Endpoint{},
-		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
-			for t := range e.TeamAccessPolicies {
-				if t == teamID {
-					return true
-				}
-			}
-			return false
-		}),
-	)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -1,8 +1,9 @@
 package endpoint
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -27,8 +28,6 @@ func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 		return nil, err
 	}
 
-	endpoint.LastCheckInDate, _ = service.service.Heartbeat(ID)
-
 	return &endpoint, nil
 }
 
@@ -66,7 +65,6 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 	for edgeID, endpointID := range service.service.idxEdgeID {
 		if endpointID == ID {
 			delete(service.service.idxEdgeID, edgeID)
-
 			break
 		}
 	}
@@ -82,11 +80,22 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	return endpoints, service.tx.GetAllWithJsoniter(
+	err := service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Endpoint{},
-		dataservices.AppendFn(&endpoints),
+		func(obj interface{}) (interface{}, error) {
-	)
+			endpoint, ok := obj.(*portainer.Endpoint)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
+				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
+			}
+
+			endpoints = append(endpoints, *endpoint)
+
+			return &portainer.Endpoint{}, nil
+		})
+
+	return endpoints, err
 }
 
 func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {
@@ -122,23 +131,6 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
-func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
-
-	return endpoints, service.tx.GetAll(
-		BucketName,
-		&portainer.Endpoint{},
-		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
-			for t := range e.TeamAccessPolicies {
-				if t == teamID {
-					return true
-				}
-			}
-			return false
-		}),
-	)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/endpointgroup/endpointgroup.go

@@ -1,15 +1,25 @@
 package endpointgroup
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-const BucketName = "endpoint_groups"
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_groups"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -20,26 +30,67 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]{
+		service: service,
-			Bucket:     BucketName,
+		tx:      tx,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
 	}
 }
 
+// EndpointGroup returns an environment(endpoint) group by ID.
+func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
+	var endpointGroup portainer.EndpointGroup
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &endpointGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointGroup, nil
+}
+
+// UpdateEndpointGroup updates an environment(endpoint) group.
+func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, endpointGroup)
+}
+
+// DeleteEndpointGroup deletes an environment(endpoint) group.
+func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+// EndpointGroups return an array containing all the environment(endpoint) groups.
+func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
+	var endpointGroups = make([]portainer.EndpointGroup, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.EndpointGroup{},
+		func(obj interface{}) (interface{}, error) {
+			endpointGroup, ok := obj.(*portainer.EndpointGroup)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
+				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
+			}
+
+			endpointGroups = append(endpointGroups, *endpointGroup)
+
+			return &portainer.EndpointGroup{}, nil
+		})
+
+	return endpointGroups, err
+}
+
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service *Service) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointgroup/tx.go

@@ -1,17 +1,72 @@
 package endpointgroup
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EndpointGroup returns an environment(endpoint) group by ID.
+func (service ServiceTx) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
+	var endpointGroup portainer.EndpointGroup
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &endpointGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointGroup, nil
+}
+
+// UpdateEndpointGroup updates an environment(endpoint) group.
+func (service ServiceTx) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, endpointGroup)
+}
+
+// DeleteEndpointGroup deletes an environment(endpoint) group.
+func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+// EndpointGroups return an array containing all the environment(endpoint) groups.
+func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
+	var endpointGroups = make([]portainer.EndpointGroup, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EndpointGroup{},
+		func(obj interface{}) (interface{}, error) {
+			endpointGroup, ok := obj.(*portainer.EndpointGroup)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
+				return nil, fmt.Errorf("failed to convert to EndpointGroup object: %s", obj)
+			}
+
+			endpointGroups = append(endpointGroups, *endpointGroup)
+
+			return &portainer.EndpointGroup{}, nil
+		})
+
+	return endpointGroups, err
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.Tx.CreateObject(
+	return service.tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointrelation/endpointrelation.go

@@ -1,8 +1,9 @@
 package endpointrelation
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -13,21 +14,16 @@ const BucketName = "endpoint_relations"
 
 // Service represents a service for managing environment(endpoint) relation data.
 type Service struct {
-	connection      portainer.Connection
+	connection    portainer.Connection
-	updateStackFn   func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
+	updateStackFn func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
-	updateStackFnTx func(tx portainer.Transaction, ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 }
 
 func (service *Service) BucketName() string {
 	return BucketName
 }
 
-func (service *Service) RegisterUpdateStackFunction(
+func (service *Service) RegisterUpdateStackFunction(updateFunc func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error) {
-	updateFunc func(portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
-	updateFuncTx func(portainer.Transaction, portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
-) {
 	service.updateStackFn = updateFunc
-	service.updateStackFnTx = updateFuncTx
 }
 
 // NewService creates a new instance of a service.
@@ -53,11 +49,22 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	return all, service.connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		dataservices.AppendFn(&all),
+		func(obj interface{}) (interface{}, error) {
-	)
+			r, ok := obj.(*portainer.EndpointRelation)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
+				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
+			}
+
+			all = append(all, *r)
+
+			return &portainer.EndpointRelation{}, nil
+		})
+
+	return all, err
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID

api/dataservices/endpointrelation/tx.go

@@ -1,8 +1,9 @@
 package endpointrelation
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -21,11 +22,22 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	return all, service.tx.GetAll(
+	err := service.tx.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		dataservices.AppendFn(&all),
+		func(obj interface{}) (interface{}, error) {
-	)
+			r, ok := obj.(*portainer.EndpointRelation)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
+				return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
+			}
+
+			all = append(all, *r)
+
+			return &portainer.EndpointRelation{}, nil
+		})
+
+	return all, err
 }
 
 // EndpointRelation returns an Environment(Endpoint) relation object by EndpointID
@@ -139,7 +151,7 @@ func (service ServiceTx) updateEdgeStacksAfterRelationChange(previousRelationSta
 				}
 			}
 
-			service.service.updateStackFnTx(service.tx, refStackId, func(edgeStack *portainer.EdgeStack) {
+			service.service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
 				edgeStack.NumDeployments = numDeployments
 			})
 		}

api/dataservices/errors/errors.go

@@ -1,10 +1,9 @@
 package errors
 
-import (
+import "errors"
-	"errors"
-)
 
 var (
+	// TODO: i'm pretty sure this needs wrapping at several levels
 	ErrObjectNotFound     = errors.New("object not found inside the database")
 	ErrWrongDBEdition     = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 	ErrDBImportFailed     = errors.New("importing backup failed")

api/dataservices/extension/extension.go

@@ -1,12 +1,17 @@
 package extension
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "extension"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "extension"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -46,12 +51,22 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	return extensions, service.connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Extension{},
-		dataservices.AppendFn(&extensions),
+		func(obj interface{}) (interface{}, error) {
-	)
+			extension, ok := obj.(*portainer.Extension)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
+				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
+			}
 
+			extensions = append(extensions, *extension)
+
+			return &portainer.Extension{}, nil
+		})
+
+	return extensions, err
 }
 
 // Persist persists a extension inside the database.

api/dataservices/fdoprofile/fdoprofile.go

@@ -1,16 +1,25 @@
 package fdoprofile
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "fdo_profiles"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "fdo_profiles"
+)
 
 // Service represents a service for managingFDO Profiles data.
 type Service struct {
-	dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,23 +30,66 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
+// FDOProfiles return an array containing all the FDO Profiles.
+func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
+	var fdoProfiles = make([]portainer.FDOProfile, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.FDOProfile{},
+		func(obj interface{}) (interface{}, error) {
+			fdoProfile, ok := obj.(*portainer.FDOProfile)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
+
+				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
+			}
+			fdoProfiles = append(fdoProfiles, *fdoProfile)
+			return &portainer.FDOProfile{}, nil
+		})
+
+	return fdoProfiles, err
+}
+
+// FDOProfile returns an FDO Profile by ID.
+func (service *Service) FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error) {
+	var FDOProfile portainer.FDOProfile
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &FDOProfile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FDOProfile, nil
+}
+
 // Create assign an ID to a new FDO Profile and saves it.
 func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
-	return service.Connection.CreateObjectWithId(
+	return service.connection.CreateObjectWithId(
 		BucketName,
 		int(FDOProfile.ID),
 		FDOProfile,
 	)
 }
 
+// Update updates an FDO Profile.
+func (service *Service) Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, FDOProfile)
+}
+
+// Delete deletes an FDO Profile.
+func (service *Service) Delete(ID portainer.FDOProfileID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
 // GetNextIdentifier returns the next identifier for a FDO Profile.
 func (service *Service) GetNextIdentifier() int {
-	return service.Connection.GetNextIdentifier(BucketName)
+	return service.connection.GetNextIdentifier(BucketName)
 }

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -1,16 +1,25 @@
 package helmuserrepository
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "helm_user_repository"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "helm_user_repository"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,29 +30,59 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
+// HelmUserRepository returns an array of all HelmUserRepository
+func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
+	var repos = make([]portainer.HelmUserRepository, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.HelmUserRepository{},
+		func(obj interface{}) (interface{}, error) {
+			r, ok := obj.(*portainer.HelmUserRepository)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
+				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
+			}
+
+			repos = append(repos, *r)
+
+			return &portainer.HelmUserRepository{}, nil
+		})
+
+	return repos, err
+}
+
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
 func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
-	return result, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.HelmUserRepository{},
-		dataservices.FilterFn(&result, func(e portainer.HelmUserRepository) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.UserID == userID
+			record, ok := obj.(*portainer.HelmUserRepository)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
+				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
+			}
+
+			if record.UserID == userID {
+				result = append(result, *record)
+			}
+
+			return &portainer.HelmUserRepository{}, nil
+		})
+
+	return result, err
 }
 
 // CreateHelmUserRepository creates a new HelmUserRepository object.
 func (service *Service) Create(record *portainer.HelmUserRepository) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.HelmUserRepositoryID(id)
@@ -51,3 +90,15 @@ func (service *Service) Create(record *portainer.HelmUserRepository) error {
 		},
 	)
 }
+
+// UpdateHelmUserRepostory updates an registry.
+func (service *Service) UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, registry *portainer.HelmUserRepository) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, registry)
+}
+
+// DeleteHelmUserRepository deletes an registry.
+func (service *Service) DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/helpers.go

@@ -1,68 +0,0 @@
-package dataservices
-
-import (
-	"errors"
-	"fmt"
-
-	perrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
-)
-
-// ErrStop signals the stop of computation when filtering results
-var ErrStop = errors.New("stop")
-
-func IsErrObjectNotFound(e error) bool {
-	return errors.Is(e, perrors.ErrObjectNotFound)
-}
-
-// AppendFn appends elements to the given collection slice
-func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
-	return func(obj interface{}) (interface{}, error) {
-		element, ok := obj.(*T)
-		if !ok {
-			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
-			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
-		}
-
-		*collection = append(*collection, *element)
-
-		return new(T), nil
-	}
-}
-
-// FilterFn appends elements to the given collection when the predicate is true
-func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
-	return func(obj interface{}) (interface{}, error) {
-		element, ok := obj.(*T)
-		if !ok {
-			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
-			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
-		}
-
-		if predicate(*element) {
-			*collection = append(*collection, *element)
-		}
-
-		return new(T), nil
-	}
-}
-
-// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
-// success
-func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
-	return func(obj interface{}) (interface{}, error) {
-		e, ok := obj.(*T)
-		if !ok {
-			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
-			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
-		}
-
-		if predicate(*e) {
-			*element = *e
-			return new(T), ErrStop
-		}
-
-		return new(T), nil
-	}
-}

api/dataservices/interface.go

@@ -1,10 +1,15 @@
 package dataservices
 
+// 	"github.com/portainer/portainer/api/dataservices"
+
 import (
 	"io"
+	"time"
+
+	"github.com/portainer/portainer/api/database/models"
+	"github.com/portainer/portainer/api/dataservices/errors"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database/models"
 )
 
 type (
@@ -34,9 +39,9 @@ type (
 		User() UserService
 		Version() VersionService
 		Webhook() WebhookService
-		PendingActions() PendingActionsService
 	}
 
+	// DataStore defines the interface to manage the data
 	DataStore interface {
 		Open() (newStore bool, err error)
 		Init() error
@@ -54,27 +59,36 @@ type (
 
 	// CustomTemplateService represents a service to manage custom templates
 	CustomTemplateService interface {
-		BaseCRUD[portainer.CustomTemplate, portainer.CustomTemplateID]
 		GetNextIdentifier() int
+		CustomTemplates() ([]portainer.CustomTemplate, error)
+		CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error)
+		Create(customTemplate *portainer.CustomTemplate) error
+		UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error
+		DeleteCustomTemplate(ID portainer.CustomTemplateID) error
+		BucketName() string
 	}
 
 	// EdgeGroupService represents a service to manage Edge groups
 	EdgeGroupService interface {
-		BaseCRUD[portainer.EdgeGroup, portainer.EdgeGroupID]
+		EdgeGroups() ([]portainer.EdgeGroup, error)
+		EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error)
+		Create(group *portainer.EdgeGroup) error
+		UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error
 		UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(group *portainer.EdgeGroup)) error
+		DeleteEdgeGroup(ID portainer.EdgeGroupID) error
+		BucketName() string
 	}
 
 	// EdgeJobService represents a service to manage Edge jobs
 	EdgeJobService interface {
-		BaseCRUD[portainer.EdgeJob, portainer.EdgeJobID]
+		EdgeJobs() ([]portainer.EdgeJob, error)
-		CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
+		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
+		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
-	}
+		BucketName() string
-
-	PendingActionsService interface {
-		BaseCRUD[portainer.PendingActions, portainer.PendingActionsID]
-		GetNextIdentifier() int
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks
@@ -94,7 +108,6 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
-		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)
@@ -107,7 +120,12 @@ type (
 
 	// EndpointGroupService represents a service for managing environment(endpoint) group data
 	EndpointGroupService interface {
-		BaseCRUD[portainer.EndpointGroup, portainer.EndpointGroupID]
+		EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error)
+		EndpointGroups() ([]portainer.EndpointGroup, error)
+		Create(group *portainer.EndpointGroup) error
+		UpdateEndpointGroup(ID portainer.EndpointGroupID, group *portainer.EndpointGroup) error
+		DeleteEndpointGroup(ID portainer.EndpointGroupID) error
+		BucketName() string
 	}
 
 	// EndpointRelationService represents a service for managing environment(endpoint) relations data
@@ -122,35 +140,70 @@ type (
 
 	// FDOProfileService represents a service to manage FDO Profiles
 	FDOProfileService interface {
-		BaseCRUD[portainer.FDOProfile, portainer.FDOProfileID]
+		FDOProfiles() ([]portainer.FDOProfile, error)
+		FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error)
+		Create(FDOProfile *portainer.FDOProfile) error
+		Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error
+		Delete(ID portainer.FDOProfileID) error
 		GetNextIdentifier() int
+		BucketName() string
 	}
 
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		BaseCRUD[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
+		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
+		Create(record *portainer.HelmUserRepository) error
+		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
+		DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error
+		BucketName() string
+	}
+
+	// JWTService represents a service for managing JWT tokens
+	JWTService interface {
+		GenerateToken(data *portainer.TokenData) (string, error)
+		GenerateTokenForOAuth(data *portainer.TokenData, expiryTime *time.Time) (string, error)
+		GenerateTokenForKubeconfig(data *portainer.TokenData) (string, error)
+		ParseAndVerifyToken(token string) (*portainer.TokenData, error)
+		SetUserSessionDuration(userSessionDuration time.Duration)
 	}
 
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
-		BaseCRUD[portainer.Registry, portainer.RegistryID]
+		Registry(ID portainer.RegistryID) (*portainer.Registry, error)
+		Registries() ([]portainer.Registry, error)
+		Create(registry *portainer.Registry) error
+		UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error
+		DeleteRegistry(ID portainer.RegistryID) error
+		BucketName() string
 	}
 
 	// ResourceControlService represents a service for managing resource control data
 	ResourceControlService interface {
-		BaseCRUD[portainer.ResourceControl, portainer.ResourceControlID]
+		ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error)
 		ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error)
+		ResourceControls() ([]portainer.ResourceControl, error)
+		Create(rc *portainer.ResourceControl) error
+		UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error
+		DeleteResourceControl(ID portainer.ResourceControlID) error
+		BucketName() string
 	}
 
 	// RoleService represents a service for managing user roles
 	RoleService interface {
-		BaseCRUD[portainer.Role, portainer.RoleID]
+		Role(ID portainer.RoleID) (*portainer.Role, error)
+		Roles() ([]portainer.Role, error)
+		Create(role *portainer.Role) error
+		UpdateRole(ID portainer.RoleID, role *portainer.Role) error
+		BucketName() string
 	}
 
 	// APIKeyRepositoryService
 	APIKeyRepository interface {
-		BaseCRUD[portainer.APIKey, portainer.APIKeyID]
+		CreateAPIKey(key *portainer.APIKey) error
+		GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error)
+		UpdateAPIKey(key *portainer.APIKey) error
+		DeleteAPIKey(ID portainer.APIKeyID) error
 		GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error)
 		GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error)
 	}
@@ -163,7 +216,12 @@ type (
 	}
 
 	SnapshotService interface {
-		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
+		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
+		Snapshots() ([]portainer.Snapshot, error)
+		UpdateSnapshot(snapshot *portainer.Snapshot) error
+		DeleteSnapshot(endpointID portainer.EndpointID) error
+		Create(snapshot *portainer.Snapshot) error
+		BucketName() string
 	}
 
 	// SSLSettingsService represents a service for managing application settings
@@ -175,33 +233,53 @@ type (
 
 	// StackService represents a service for managing stack data
 	StackService interface {
-		BaseCRUD[portainer.Stack, portainer.StackID]
+		Stack(ID portainer.StackID) (*portainer.Stack, error)
 		StackByName(name string) (*portainer.Stack, error)
 		StacksByName(name string) ([]portainer.Stack, error)
+		Stacks() ([]portainer.Stack, error)
+		Create(stack *portainer.Stack) error
+		UpdateStack(ID portainer.StackID, stack *portainer.Stack) error
+		DeleteStack(ID portainer.StackID) error
 		GetNextIdentifier() int
 		StackByWebhookID(ID string) (*portainer.Stack, error)
 		RefreshableStacks() ([]portainer.Stack, error)
+		BucketName() string
 	}
 
 	// TagService represents a service for managing tag data
 	TagService interface {
-		BaseCRUD[portainer.Tag, portainer.TagID]
+		Tags() ([]portainer.Tag, error)
+		Tag(ID portainer.TagID) (*portainer.Tag, error)
+		Create(tag *portainer.Tag) error
+		UpdateTag(ID portainer.TagID, tag *portainer.Tag) error
 		UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error
+		DeleteTag(ID portainer.TagID) error
+		BucketName() string
 	}
 
 	// TeamService represents a service for managing user data
 	TeamService interface {
-		BaseCRUD[portainer.Team, portainer.TeamID]
+		Team(ID portainer.TeamID) (*portainer.Team, error)
 		TeamByName(name string) (*portainer.Team, error)
+		Teams() ([]portainer.Team, error)
+		Create(team *portainer.Team) error
+		UpdateTeam(ID portainer.TeamID, team *portainer.Team) error
+		DeleteTeam(ID portainer.TeamID) error
+		BucketName() string
 	}
 
 	// TeamMembershipService represents a service for managing team membership data
 	TeamMembershipService interface {
-		BaseCRUD[portainer.TeamMembership, portainer.TeamMembershipID]
+		TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error)
+		TeamMemberships() ([]portainer.TeamMembership, error)
 		TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error)
 		TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error)
+		Create(membership *portainer.TeamMembership) error
+		UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error
+		DeleteTeamMembership(ID portainer.TeamMembershipID) error
 		DeleteTeamMembershipByUserID(userID portainer.UserID) error
 		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
+		BucketName() string
 		DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error
 	}
 
@@ -214,24 +292,38 @@ type (
 
 	// UserService represents a service for managing user data
 	UserService interface {
-		BaseCRUD[portainer.User, portainer.UserID]
+		User(ID portainer.UserID) (*portainer.User, error)
 		UserByUsername(username string) (*portainer.User, error)
+		Users() ([]portainer.User, error)
 		UsersByRole(role portainer.UserRole) ([]portainer.User, error)
+		Create(user *portainer.User) error
+		UpdateUser(ID portainer.UserID, user *portainer.User) error
+		DeleteUser(ID portainer.UserID) error
+		BucketName() string
 	}
 
 	// VersionService represents a service for managing version data
 	VersionService interface {
+		Edition() (portainer.SoftwareEdition, error)
 		InstanceID() (string, error)
 		UpdateInstanceID(ID string) error
-		Edition() (portainer.SoftwareEdition, error)
 		Version() (*models.Version, error)
 		UpdateVersion(*models.Version) error
 	}
 
 	// WebhookService represents a service for managing webhook data.
 	WebhookService interface {
-		BaseCRUD[portainer.Webhook, portainer.WebhookID]
+		Webhooks() ([]portainer.Webhook, error)
+		Webhook(ID portainer.WebhookID) (*portainer.Webhook, error)
+		Create(portainer *portainer.Webhook) error
+		UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error
 		WebhookByResourceID(resourceID string) (*portainer.Webhook, error)
 		WebhookByToken(token string) (*portainer.Webhook, error)
+		DeleteWebhook(ID portainer.WebhookID) error
+		BucketName() string
 	}
 )
+
+func IsErrObjectNotFound(e error) bool {
+	return e == errors.ErrObjectNotFound
+}

api/dataservices/pendingactions/pendingactions.go

@@ -1,74 +0,0 @@
-package pendingactions
-
-import (
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-)
-
-const (
-	BucketName = "pending_actions"
-)
-
-type Service struct {
-	dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]
-}
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]
-}
-
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]{
-			Bucket:     BucketName,
-			Connection: connection,
-		},
-	}, nil
-}
-
-func (s Service) Create(config *portainer.PendingActions) error {
-	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
-		return s.Tx(tx).Create(config)
-	})
-}
-
-func (s Service) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
-	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
-		return s.Tx(tx).Update(ID, config)
-	})
-}
-
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
-	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]{
-			Bucket:     BucketName,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
-	}
-}
-
-func (s ServiceTx) Create(config *portainer.PendingActions) error {
-	return s.Tx.CreateObject(BucketName, func(id uint64) (int, interface{}) {
-		config.ID = portainer.PendingActionsID(id)
-		config.CreatedAt = time.Now().Unix()
-
-		return int(config.ID), config
-	})
-}
-
-func (s ServiceTx) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
-	return s.BaseDataServiceTx.Update(ID, config)
-}
-
-// GetNextIdentifier returns the next identifier for a custom template.
-func (service *Service) GetNextIdentifier() int {
-	return service.Connection.GetNextIdentifier(BucketName)
-}

api/dataservices/registry/registry.go

@@ -1,16 +1,25 @@
 package registry
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "registries"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "registries"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,26 +30,48 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// Registry returns an registry by ID.
-	return ServiceTx{
+func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]{
+	var registry portainer.Registry
-			Bucket:     BucketName,
+	identifier := service.connection.ConvertToKey(int(ID))
-			Connection: service.Connection,
+
-			Tx:         tx,
+	err := service.connection.GetObject(BucketName, identifier, &registry)
-		},
+	if err != nil {
+		return nil, err
 	}
+
+	return &registry, nil
 }
 
-// Create creates a new registry.
+// Registries returns an array containing all the registries.
+func (service *Service) Registries() ([]portainer.Registry, error) {
+	var registries = make([]portainer.Registry, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Registry{},
+		func(obj interface{}) (interface{}, error) {
+			registry, ok := obj.(*portainer.Registry)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
+				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
+			}
+
+			registries = append(registries, *registry)
+
+			return &portainer.Registry{}, nil
+		})
+
+	return registries, err
+}
+
+// CreateRegistry creates a new registry.
 func (service *Service) Create(registry *portainer.Registry) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -48,3 +79,15 @@ func (service *Service) Create(registry *portainer.Registry) error {
 		},
 	)
 }
+
+// UpdateRegistry updates an registry.
+func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, registry)
+}
+
+// DeleteRegistry deletes an registry.
+func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/registry/tx.go

@@ -1,21 +0,0 @@
-package registry
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]
-}
-
-// Create creates a new registry.
-func (service ServiceTx) Create(registry *portainer.Registry) error {
-	return service.Tx.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			registry.ID = portainer.RegistryID(id)
-			return int(registry.ID), registry
-		},
-	)
-}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -1,21 +1,25 @@
 package resourcecontrol
 
 import (
-	"errors"
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "resource_control"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "resource_control"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -26,21 +30,21 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// ResourceControl returns a ResourceControl object by ID
-	return ServiceTx{
+func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]{
+	var resourceControl portainer.ResourceControl
-			Bucket:     BucketName,
+	identifier := service.connection.ConvertToKey(int(ID))
-			Connection: service.Connection,
+
-			Tx:         tx,
+	err := service.connection.GetObject(BucketName, identifier, &resourceControl)
-		},
+	if err != nil {
+		return nil, err
 	}
+
+	return &resourceControl, nil
 }
 
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
@@ -49,14 +53,14 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -73,16 +77,38 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 
 			return &portainer.ResourceControl{}, nil
 		})
-	if errors.Is(err, stop) {
+	if err == stop {
 		return resourceControl, nil
 	}
 
 	return nil, err
 }
 
+// ResourceControls returns all the ResourceControl objects
+func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
+	var rcs = make([]portainer.ResourceControl, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.ResourceControl{},
+		func(obj interface{}) (interface{}, error) {
+			rc, ok := obj.(*portainer.ResourceControl)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
+				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+			}
+
+			rcs = append(rcs, *rc)
+
+			return &portainer.ResourceControl{}, nil
+		})
+
+	return rcs, err
+}
+
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -90,3 +116,15 @@ func (service *Service) Create(resourceControl *portainer.ResourceControl) error
 		},
 	)
 }
+
+// UpdateResourceControl saves a ResourceControl object.
+func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, resourceControl)
+}
+
+// DeleteResourceControl deletes a ResourceControl object by ID
+func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/resourcecontrol/tx.go

@@ -1,63 +0,0 @@
-package resourcecontrol
-
-import (
-	"errors"
-	"fmt"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-
-	"github.com/rs/zerolog/log"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]
-}
-
-// ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
-// to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
-// if no ResourceControl was found.
-func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
-	var resourceControl *portainer.ResourceControl
-	stop := fmt.Errorf("ok")
-	err := service.Tx.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
-			}
-
-			if rc.ResourceID == resourceID && rc.Type == resourceType {
-				resourceControl = rc
-				return nil, stop
-			}
-
-			for _, subResourceID := range rc.SubResourceIDs {
-				if subResourceID == resourceID {
-					resourceControl = rc
-					return nil, stop
-				}
-			}
-
-			return &portainer.ResourceControl{}, nil
-		})
-	if errors.Is(err, stop) {
-		return resourceControl, nil
-	}
-
-	return nil, err
-}
-
-// CreateResourceControl creates a new ResourceControl object
-func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
-	return service.Tx.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			resourceControl.ID = portainer.ResourceControlID(id)
-			return int(resourceControl.ID), resourceControl
-		},
-	)
-}

api/dataservices/role/role.go

@@ -1,16 +1,25 @@
 package role
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "roles"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "roles"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Role, portainer.RoleID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,26 +30,48 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Role, portainer.RoleID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// Role returns a Role by ID
-	return ServiceTx{
+func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]{
+	var set portainer.Role
-			Bucket:     BucketName,
+	identifier := service.connection.ConvertToKey(int(ID))
-			Connection: service.Connection,
+
-			Tx:         tx,
+	err := service.connection.GetObject(BucketName, identifier, &set)
-		},
+	if err != nil {
+		return nil, err
 	}
+
+	return &set, nil
+}
+
+// Roles return an array containing all the sets.
+func (service *Service) Roles() ([]portainer.Role, error) {
+	var sets = make([]portainer.Role, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Role{},
+		func(obj interface{}) (interface{}, error) {
+			set, ok := obj.(*portainer.Role)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
+				return nil, fmt.Errorf("Failed to convert to Role object: %s", obj)
+			}
+
+			sets = append(sets, *set)
+
+			return &portainer.Role{}, nil
+		})
+
+	return sets, err
 }
 
 // CreateRole creates a new Role.
 func (service *Service) Create(role *portainer.Role) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			role.ID = portainer.RoleID(id)
@@ -48,3 +79,9 @@ func (service *Service) Create(role *portainer.Role) error {
 		},
 	)
 }
+
+// UpdateRole updates a role.
+func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, role)
+}

api/dataservices/role/tx.go

@@ -1,21 +0,0 @@
-package role
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]
-}
-
-// CreateRole creates a new Role.
-func (service ServiceTx) Create(role *portainer.Role) error {
-	return service.Tx.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			role.ID = portainer.RoleID(id)
-			return int(role.ID), role
-		},
-	)
-}

api/dataservices/schedule/schedule.go

@@ -1,12 +1,17 @@
 package schedule
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "schedules"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "schedules"
+)
 
 // Service represents a service for managing schedule data.
 type Service struct {
@@ -58,11 +63,22 @@ func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	return schedules, service.connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		dataservices.AppendFn(&schedules),
+		func(obj interface{}) (interface{}, error) {
-	)
+			schedule, ok := obj.(*portainer.Schedule)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
+				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
+			}
+
+			schedules = append(schedules, *schedule)
+
+			return &portainer.Schedule{}, nil
+		})
+
+	return schedules, err
 }
 
 // SchedulesByJobType return a array containing all the schedules
@@ -70,13 +86,24 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	return schedules, service.connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		dataservices.FilterFn(&schedules, func(e portainer.Schedule) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.JobType == jobType
+			schedule, ok := obj.(*portainer.Schedule)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
+				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
+			}
+
+			if schedule.JobType == jobType {
+				schedules = append(schedules, *schedule)
+			}
+
+			return &portainer.Schedule{}, nil
+		})
+
+	return schedules, err
 }
 
 // Create assign an ID to a new schedule and saves it.

api/dataservices/settings/settings.go

@@ -31,13 +31,6 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
-	return ServiceTx{
-		service: service,
-		tx:      tx,
-	}
-}
-
 // Settings retrieve the settings object.
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings

api/dataservices/settings/tx.go

@@ -1,31 +0,0 @@
-package settings
-
-import (
-	portainer "github.com/portainer/portainer/api"
-)
-
-type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Settings retrieve the settings object.
-func (service ServiceTx) Settings() (*portainer.Settings, error) {
-	var settings portainer.Settings
-
-	err := service.tx.GetObject(BucketName, []byte(settingsKey), &settings)
-	if err != nil {
-		return nil, err
-	}
-
-	return &settings, nil
-}
-
-// UpdateSettings persists a Settings object.
-func (service ServiceTx) UpdateSettings(settings *portainer.Settings) error {
-	return service.tx.UpdateObject(BucketName, []byte(settingsKey), settings)
-}

api/dataservices/snapshot/snapshot.go

@@ -1,14 +1,23 @@
 package snapshot
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-const BucketName = "snapshots"
+const (
+	BucketName = "snapshots"
+)
 
 type Service struct {
-	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 func NewService(connection portainer.Connection) (*Service, error) {
@@ -18,23 +27,58 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]{
+		service: service,
-			Bucket:     BucketName,
+		tx:      tx,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
 	}
 }
 
-func (service *Service) Create(snapshot *portainer.Snapshot) error {
+func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	var snapshot portainer.Snapshot
+	identifier := service.connection.ConvertToKey(int(endpointID))
+
+	err := service.connection.GetObject(BucketName, identifier, &snapshot)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
+	var snapshots = make([]portainer.Snapshot, 0)
+
+	err := service.connection.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Snapshot{},
+		func(obj interface{}) (interface{}, error) {
+			snapshot, ok := obj.(*portainer.Snapshot)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
+				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
+			}
+			snapshots = append(snapshots, *snapshot)
+			return &portainer.Snapshot{}, nil
+		})
+
+	return snapshots, err
+}
+
+func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
+	identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
+	return service.connection.UpdateObject(BucketName, identifier, snapshot)
+}
+
+func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
+	identifier := service.connection.ConvertToKey(int(endpointID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+func (service *Service) Create(snapshot *portainer.Snapshot) error {
+	return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/snapshot/tx.go

@@ -1,14 +1,63 @@
 package snapshot
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot portainer.Snapshot
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+
+	err := service.tx.GetObject(BucketName, identifier, &snapshot)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
+	var snapshots = make([]portainer.Snapshot, 0)
+
+	err := service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Snapshot{},
+		func(obj interface{}) (interface{}, error) {
+			snapshot, ok := obj.(*portainer.Snapshot)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
+				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
+			}
+			snapshots = append(snapshots, *snapshot)
+			return &portainer.Snapshot{}, nil
+		})
+
+	return snapshots, err
+}
+
+func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {
+	identifier := service.service.connection.ConvertToKey(int(snapshot.EndpointID))
+	return service.tx.UpdateObject(BucketName, identifier, snapshot)
+}
+
+func (service ServiceTx) DeleteSnapshot(endpointID portainer.EndpointID) error {
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	return service.tx.DeleteObject(BucketName, identifier)
 }
 
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
-	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	return service.tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/stack/stack.go

@@ -1,20 +1,27 @@
 package stack
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/dataservices/errors"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "stacks"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "stacks"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Stack, portainer.StackID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -25,37 +32,50 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Stack, portainer.StackID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// Stack returns a stack object by ID.
-	return ServiceTx{
+func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
-		BaseDataServiceTx: service.BaseDataService.Tx(tx),
+	var stack portainer.Stack
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
 	}
+
+	return &stack, nil
 }
 
 // StackByName returns a stack object by name.
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
-	var s portainer.Stack
+	var s *portainer.Stack
 
-	err := service.Connection.GetAll(
+	stop := fmt.Errorf("ok")
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.Name == name
+			stack, ok := obj.(*portainer.Stack)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
+			}
 
-	if errors.Is(err, dataservices.ErrStop) {
+			if stack.Name == name {
-		return &s, nil
+				s = stack
+				return nil, stop
+			}
+
+			return &portainer.Stack{}, nil
+		})
+	if err == stop {
+		return s, nil
 	}
-
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -65,58 +85,125 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	return stacks, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.Name == name
+			stack, ok := obj.(portainer.Stack)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
+			}
+
+			if stack.Name == name {
+				stacks = append(stacks, stack)
+			}
+
+			return &portainer.Stack{}, nil
+		})
+
+	return stacks, err
+}
+
+// Stacks returns an array containing all the stacks.
+func (service *Service) Stacks() ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		func(obj interface{}) (interface{}, error) {
+			stack, ok := obj.(*portainer.Stack)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
+			}
+
+			stacks = append(stacks, *stack)
+
+			return &portainer.Stack{}, nil
+		})
+
+	return stacks, err
 }
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return service.Connection.GetNextIdentifier(BucketName)
+	return service.connection.GetNextIdentifier(BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) Create(stack *portainer.Stack) error {
-	return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
+	return service.connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
+}
+
+// UpdateStack updates a stack.
+func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, stack)
+}
+
+// DeleteStack deletes a stack.
+func (service *Service) DeleteStack(ID portainer.StackID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
 }
 
 // StackByWebhookID returns a pointer to a stack object by webhook ID.
 // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
 func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	var s portainer.Stack
+	var s *portainer.Stack
-
+	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+			var ok bool
-		}),
+			s, ok = obj.(*portainer.Stack)
-	)
 
-	if errors.Is(err, dataservices.ErrStop) {
+			if !ok {
-		return &s, nil
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+
+				return &portainer.Stack{}, nil
+			}
+
+			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
+				return nil, stop
+			}
+
+			return &portainer.Stack{}, nil
+		})
+	if err == stop {
+		return s, nil
 	}
-
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
+
 }
 
 // RefreshableStacks returns stacks that are configured for a periodic update
 func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 	stacks := make([]portainer.Stack, 0)
 
-	return stacks, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+			stack, ok := obj.(*portainer.Stack)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
+			}
+
+			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+				stacks = append(stacks, *stack)
+			}
+
+			return &portainer.Stack{}, nil
+		})
+
+	return stacks, err
 }

api/dataservices/stack/tests/stack_test.go

@@ -29,7 +29,8 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
 	b.createNewStack(newGuidString(t))
@@ -58,7 +59,7 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 		Type:         portainer.DockerComposeStack,
 		EndpointID:   2,
 		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Env:          []portainer.Pair{{Name: "Name1", Value: "Value1"}},
+		Env:          []portainer.Pair{{"Name1", "Value1"}},
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 		ProjectPath:  "/tmp/project",
@@ -86,7 +87,8 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store := datastore.MustNewTestStore(t, true, true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}
 	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}

api/dataservices/stack/tx.go

@@ -1,98 +0,0 @@
-package stack
-
-import (
-	"errors"
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.Stack, portainer.StackID]
-}
-
-// StackByName returns a stack object by name.
-func (service ServiceTx) StackByName(name string) (*portainer.Stack, error) {
-	var s portainer.Stack
-
-	err := service.Tx.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
-			return e.Name == name
-		}),
-	)
-
-	if errors.Is(err, dataservices.ErrStop) {
-		return &s, nil
-	}
-
-	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
-	}
-
-	return nil, err
-}
-
-// Stacks returns an array containing all the stacks with same name
-func (service ServiceTx) StacksByName(name string) ([]portainer.Stack, error) {
-	var stacks = make([]portainer.Stack, 0)
-
-	return stacks, service.Tx.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
-			return e.Name == name
-		}),
-	)
-}
-
-// GetNextIdentifier returns the next identifier for a stack.
-func (service ServiceTx) GetNextIdentifier() int {
-	return service.Tx.GetNextIdentifier(BucketName)
-}
-
-// CreateStack creates a new stack.
-func (service ServiceTx) Create(stack *portainer.Stack) error {
-	return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack)
-}
-
-// StackByWebhookID returns a pointer to a stack object by webhook ID.
-// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
-func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
-	var s portainer.Stack
-
-	err := service.Tx.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
-			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
-		}),
-	)
-
-	if errors.Is(err, dataservices.ErrStop) {
-		return &s, nil
-	}
-
-	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
-	}
-
-	return nil, err
-
-}
-
-// RefreshableStacks returns stacks that are configured for a periodic update
-func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
-	stacks := make([]portainer.Stack, 0)
-
-	return stacks, service.Tx.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
-			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
-		}),
-	)
-}

api/dataservices/tag/tag.go

@@ -1,16 +1,25 @@
 package tag
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "tags"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "tags"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Tag, portainer.TagID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -21,26 +30,55 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Tag, portainer.TagID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]{
+		service: service,
-			Bucket:     BucketName,
+		tx:      tx,
-			Connection: service.Connection,
-			Tx:         tx,
-		},
 	}
 }
 
+// Tags return an array containing all the tags.
+func (service *Service) Tags() ([]portainer.Tag, error) {
+	var tags = make([]portainer.Tag, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Tag{},
+		func(obj interface{}) (interface{}, error) {
+			tag, ok := obj.(*portainer.Tag)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
+				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
+			}
+
+			tags = append(tags, *tag)
+
+			return &portainer.Tag{}, nil
+		})
+
+	return tags, err
+}
+
+// Tag returns a tag by ID.
+func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
+	var tag portainer.Tag
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &tag)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tag, nil
+}
+
 // CreateTag creates a new tag.
 func (service *Service) Create(tag *portainer.Tag) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -49,12 +87,24 @@ func (service *Service) Create(tag *portainer.Tag) error {
 	)
 }
 
+// Deprecated: Use UpdateTagFunc instead.
+func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, tag)
+}
+
 // UpdateTagFunc updates a tag inside a transaction avoiding data races.
 func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
-	id := service.Connection.ConvertToKey(int(ID))
+	id := service.connection.ConvertToKey(int(ID))
 	tag := &portainer.Tag{}
 
-	return service.Connection.UpdateObjectFunc(BucketName, id, tag, func() {
+	return service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
 		updateFunc(tag)
 	})
 }
+
+// DeleteTag deletes a tag.
+func (service *Service) DeleteTag(ID portainer.TagID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/tag/tx.go

@@ -2,18 +2,60 @@ package tag
 
 import (
 	"errors"
+	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Tags return an array containing all the tags.
+func (service ServiceTx) Tags() ([]portainer.Tag, error) {
+	var tags = make([]portainer.Tag, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.Tag{},
+		func(obj interface{}) (interface{}, error) {
+			tag, ok := obj.(*portainer.Tag)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
+				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
+			}
+
+			tags = append(tags, *tag)
+
+			return &portainer.Tag{}, nil
+		})
+
+	return tags, err
+}
+
+// Tag returns a tag by ID.
+func (service ServiceTx) Tag(ID portainer.TagID) (*portainer.Tag, error) {
+	var tag portainer.Tag
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &tag)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tag, nil
 }
 
 // CreateTag creates a new tag.
 func (service ServiceTx) Create(tag *portainer.Tag) error {
-	return service.Tx.CreateObject(
+	return service.tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -22,7 +64,19 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTagFunc is a no-op inside a transaction.
+// UpdateTag updates a tag
+func (service ServiceTx) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, tag)
+}
+
+// UpdateTagFunc is a no-op inside a transaction
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }
+
+// DeleteTag deletes a tag.
+func (service ServiceTx) DeleteTag(ID portainer.TagID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}

api/dataservices/team/team.go

@@ -1,20 +1,27 @@
 package team
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/dataservices/errors"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "teams"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "teams"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Team, portainer.TeamID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -25,39 +32,86 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Team, portainer.TeamID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
+// Team returns a Team by ID
+func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
+	var team portainer.Team
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &team)
+	if err != nil {
+		return nil, err
+	}
+
+	return &team, nil
+}
+
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
-	var t portainer.Team
+	var t *portainer.Team
 
-	err := service.Connection.GetAll(
+	stop := fmt.Errorf("ok")
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Team{},
-		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+		func(obj interface{}) (interface{}, error) {
-			return strings.EqualFold(e.Name, name)
+			team, ok := obj.(*portainer.Team)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
+				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
+			}
 
-	if errors.Is(err, dataservices.ErrStop) {
+			if strings.EqualFold(team.Name, name) {
-		return &t, nil
+				t = team
+				return nil, stop
+			}
+
+			return &portainer.Team{}, nil
+		})
+	if err == stop {
+		return t, nil
 	}
-
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
 }
 
+// Teams return an array containing all the teams.
+func (service *Service) Teams() ([]portainer.Team, error) {
+	var teams = make([]portainer.Team, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Team{},
+		func(obj interface{}) (interface{}, error) {
+			team, ok := obj.(*portainer.Team)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
+				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
+			}
+
+			teams = append(teams, *team)
+
+			return &portainer.Team{}, nil
+		})
+
+	return teams, err
+}
+
+// UpdateTeam saves a Team.
+func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, team)
+}
+
 // CreateTeam creates a new Team.
 func (service *Service) Create(team *portainer.Team) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			team.ID = portainer.TeamID(id)
@@ -65,3 +119,9 @@ func (service *Service) Create(team *portainer.Team) error {
 		},
 	)
 }
+
+// DeleteTeam deletes a Team.
+func (service *Service) DeleteTeam(ID portainer.TeamID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/team/tests/team_test.go

@@ -10,7 +10,8 @@ import (
 
 func Test_teamByName(t *testing.T) {
 	t.Run("When store is empty should return ErrObjectNotFound", func(t *testing.T) {
-		_, store := datastore.MustNewTestStore(t, true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
 
 		_, err := store.Team().TeamByName("name")
 		assert.ErrorIs(t, err, errors.ErrObjectNotFound)
@@ -18,7 +19,8 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is no object with the same name should return ErrObjectNotFound", func(t *testing.T) {
-		_, store := datastore.MustNewTestStore(t, true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
 
 		teamBuilder := teamBuilder{
 			t:     t,
@@ -33,7 +35,8 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is an object with the same name should return the object", func(t *testing.T) {
-		_, store := datastore.MustNewTestStore(t, true, true)
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
 
 		teamBuilder := teamBuilder{
 			t:     t,

api/dataservices/teammembership/teammembership.go

@@ -4,17 +4,22 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "team_membership"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "team_membership"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -25,52 +30,102 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// TeamMembership returns a TeamMembership object by ID
-	return ServiceTx{
+func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]{
+	var membership portainer.TeamMembership
-			Bucket:     BucketName,
+	identifier := service.connection.ConvertToKey(int(ID))
-			Connection: service.Connection,
+
-			Tx:         tx,
+	err := service.connection.GetObject(BucketName, identifier, &membership)
-		},
+	if err != nil {
+		return nil, err
 	}
+
+	return &membership, nil
+}
+
+// TeamMemberships return an array containing all the TeamMembership objects.
+func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (interface{}, error) {
+			membership, ok := obj.(*portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+			}
+
+			memberships = append(memberships, *membership)
+
+			return &portainer.TeamMembership{}, nil
+		})
+
+	return memberships, err
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	return memberships, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.UserID == userID
+			membership, ok := obj.(*portainer.TeamMembership)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+			}
+
+			if membership.UserID == userID {
+				memberships = append(memberships, *membership)
+			}
+
+			return &portainer.TeamMembership{}, nil
+		})
+
+	return memberships, err
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	return memberships, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.TeamID == teamID
+			membership, ok := obj.(*portainer.TeamMembership)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+			}
+
+			if membership.TeamID == teamID {
+				memberships = append(memberships, *membership)
+			}
+
+			return &portainer.TeamMembership{}, nil
+		})
+
+	return memberships, err
+}
+
+// UpdateTeamMembership saves a TeamMembership object.
+func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, membership)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) Create(membership *portainer.TeamMembership) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			membership.ID = portainer.TeamMembershipID(id)
@@ -79,9 +134,15 @@ func (service *Service) Create(membership *portainer.TeamMembership) error {
 	)
 }
 
+// DeleteTeamMembership deletes a TeamMembership object.
+func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.Connection.DeleteAllObjects(
+	return service.connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -102,7 +163,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.Connection.DeleteAllObjects(
+	return service.connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -122,7 +183,7 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 }
 
 func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.Connection.DeleteAllObjects(
+	return service.connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/teammembership/tx.go

@@ -1,113 +0,0 @@
-package teammembership
-
-import (
-	"fmt"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-
-	"github.com/rs/zerolog/log"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]
-}
-
-// TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
-func (service ServiceTx) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	return memberships, service.Tx.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
-			return e.UserID == userID
-		}),
-	)
-}
-
-// TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
-func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	return memberships, service.Tx.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
-			return e.TeamID == teamID
-		}),
-	)
-}
-
-// CreateTeamMembership creates a new TeamMembership object.
-func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
-	return service.Tx.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			membership.ID = portainer.TeamMembershipID(id)
-			return int(membership.ID), membership
-		},
-	)
-}
-
-// DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
-func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.Tx.DeleteAllObjects(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (id int, ok bool) {
-			membership, ok := obj.(portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-				return -1, false
-			}
-
-			if membership.UserID == userID {
-				return int(membership.ID), true
-			}
-
-			return -1, false
-		})
-}
-
-// DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
-func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.Tx.DeleteAllObjects(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (id int, ok bool) {
-			membership, ok := obj.(portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-				return -1, false
-			}
-
-			if membership.TeamID == teamID {
-				return int(membership.ID), true
-			}
-
-			return -1, false
-		})
-}
-
-func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.Tx.DeleteAllObjects(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (id int, ok bool) {
-			membership, ok := obj.(portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-				return -1, false
-			}
-
-			if membership.TeamID == teamID && membership.UserID == userID {
-				return int(membership.ID), true
-			}
-
-			return -1, false
-		})
-}

api/dataservices/user/tx.go

@@ -1,63 +0,0 @@
-package user
-
-import (
-	"errors"
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-)
-
-type ServiceTx struct {
-	dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]
-}
-
-// UserByUsername returns a user by username.
-func (service ServiceTx) UserByUsername(username string) (*portainer.User, error) {
-	var u portainer.User
-
-	err := service.Tx.GetAll(
-		BucketName,
-		&portainer.User{},
-		dataservices.FirstFn(&u, func(e portainer.User) bool {
-			return strings.EqualFold(e.Username, username)
-		}),
-	)
-
-	if errors.Is(err, dataservices.ErrStop) {
-		return &u, nil
-	}
-
-	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
-	}
-
-	return nil, err
-}
-
-// UsersByRole return an array containing all the users with the specified role.
-func (service ServiceTx) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-
-	return users, service.Tx.GetAll(
-		BucketName,
-		&portainer.User{},
-		dataservices.FilterFn(&users, func(e portainer.User) bool {
-			return e.Role == role
-		}),
-	)
-}
-
-// CreateUser creates a new user.
-func (service ServiceTx) Create(user *portainer.User) error {
-	return service.Tx.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			user.ID = portainer.UserID(id)
-			user.Username = strings.ToLower(user.Username)
-
-			return int(user.ID), user
-		},
-	)
-}

api/dataservices/user/user.go

@@ -1,20 +1,27 @@
 package user
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/dataservices/errors"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "users"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "users"
+)
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	dataservices.BaseDataService[portainer.User, portainer.UserID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -25,62 +32,115 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.User, portainer.UserID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
-func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+// User returns a user by ID
-	return ServiceTx{
+func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
-		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]{
+	var user portainer.User
-			Bucket:     BucketName,
+	identifier := service.connection.ConvertToKey(int(ID))
-			Connection: service.Connection,
+
-			Tx:         tx,
+	err := service.connection.GetObject(BucketName, identifier, &user)
-		},
+	if err != nil {
+		return nil, err
 	}
+
+	return &user, nil
 }
 
 // UserByUsername returns a user by username.
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
-	var u portainer.User
+	var u *portainer.User
-
+	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		dataservices.FirstFn(&u, func(e portainer.User) bool {
+		func(obj interface{}) (interface{}, error) {
-			return strings.EqualFold(e.Username, username)
+			user, ok := obj.(*portainer.User)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
 
-	if errors.Is(err, dataservices.ErrStop) {
+				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-		return &u, nil
+			}
+
+			if strings.EqualFold(user.Username, username) {
+				u = user
+				return nil, stop
+			}
+
+			return &portainer.User{}, nil
+		})
+
+	if err == stop {
+		return u, nil
 	}
 
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
 }
 
+// Users return an array containing all the users.
+func (service *Service) Users() ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.User{},
+		func(obj interface{}) (interface{}, error) {
+			user, ok := obj.(*portainer.User)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
+				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
+			}
+
+			users = append(users, *user)
+
+			return &portainer.User{}, nil
+		})
+
+	return users, err
+}
+
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	return users, service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		dataservices.FilterFn(&users, func(e portainer.User) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.Role == role
+			user, ok := obj.(*portainer.User)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
+				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
+			}
+
+			if user.Role == role {
+				users = append(users, *user)
+			}
+
+			return &portainer.User{}, nil
+		})
+
+	return users, err
+}
+
+// UpdateUser saves a user.
+func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	user.Username = strings.ToLower(user.Username)
+	return service.connection.UpdateObject(BucketName, identifier, user)
 }
 
 // CreateUser creates a new user.
 func (service *Service) Create(user *portainer.User) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			user.ID = portainer.UserID(id)
@@ -90,3 +150,9 @@ func (service *Service) Create(user *portainer.User) error {
 		},
 	)
 }
+
+// DeleteUser deletes a user.
+func (service *Service) DeleteUser(ID portainer.UserID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}

api/dataservices/version/version.go

@@ -73,10 +73,6 @@ func (service *Service) IsUpdating() (bool, error) {
 
 // StoreIsUpdating store the database updating status.
 func (service *Service) StoreIsUpdating(isUpdating bool) error {
-	if isUpdating {
-		return service.connection.UpdateObject(BucketName, []byte(updatingKey), isUpdating)
-	}
-
 	return service.connection.DeleteObject(BucketName, []byte(updatingKey))
 }
 

api/dataservices/webhook/webhook.go

@@ -1,19 +1,26 @@
 package webhook
 
 import (
-	"errors"
+	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/dataservices/errors"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
-// BucketName represents the name of the bucket where this service stores data.
+const (
-const BucketName = "webhooks"
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "webhooks"
+)
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
 }
 
 // NewService creates a new instance of a service.
@@ -24,31 +31,74 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		BaseDataService: dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]{
+		connection: connection,
-			Bucket:     BucketName,
-			Connection: connection,
-		},
 	}, nil
 }
 
+// Webhooks returns an array of all webhooks
+func (service *Service) Webhooks() ([]portainer.Webhook, error) {
+	var webhooks = make([]portainer.Webhook, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.Webhook{},
+		func(obj interface{}) (interface{}, error) {
+			webhook, ok := obj.(*portainer.Webhook)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
+			}
+
+			webhooks = append(webhooks, *webhook)
+
+			return &portainer.Webhook{}, nil
+		})
+
+	return webhooks, err
+}
+
+// Webhook returns a webhook by ID.
+func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
+	var webhook portainer.Webhook
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &webhook)
+	if err != nil {
+		return nil, err
+	}
+
+	return &webhook, nil
+}
+
 // WebhookByResourceID returns a webhook by the ResourceID it is associated with.
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
-	var w portainer.Webhook
+	var w *portainer.Webhook
-
+	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.ResourceID == ID
+			webhook, ok := obj.(*portainer.Webhook)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
 
-	if errors.Is(err, dataservices.ErrStop) {
+				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-		return &w, nil
+			}
+
+			if webhook.ResourceID == ID {
+				w = webhook
+				return nil, stop
+			}
+
+			return &portainer.Webhook{}, nil
+		})
+
+	if err == stop {
+		return w, nil
 	}
 
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
@@ -56,30 +106,47 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 
 // WebhookByToken returns a webhook by the random token it is associated with.
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
-	var w portainer.Webhook
+	var w *portainer.Webhook
-
+	stop := fmt.Errorf("ok")
-	err := service.Connection.GetAll(
+	err := service.connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+		func(obj interface{}) (interface{}, error) {
-			return e.Token == token
+			webhook, ok := obj.(*portainer.Webhook)
-		}),
+			if !ok {
-	)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
 
-	if errors.Is(err, dataservices.ErrStop) {
+				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-		return &w, nil
+			}
+
+			if webhook.Token == token {
+				w = webhook
+				return nil, stop
+			}
+
+			return &portainer.Webhook{}, nil
+		})
+
+	if err == stop {
+		return w, nil
 	}
 
 	if err == nil {
-		return nil, dserrors.ErrObjectNotFound
+		return nil, errors.ErrObjectNotFound
 	}
 
 	return nil, err
 }
 
+// DeleteWebhook deletes a webhook.
+func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) Create(webhook *portainer.Webhook) error {
-	return service.Connection.CreateObject(
+	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			webhook.ID = portainer.WebhookID(id)
@@ -87,3 +154,9 @@ func (service *Service) Create(webhook *portainer.Webhook) error {
 		},
 	)
 }
+
+// UpdateWebhook update a webhook.
+func (service *Service) UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, webhook)
+}

api/datastore/backup.go

@@ -65,7 +65,7 @@ type BackupOptions struct {
 // - db rollback
 func getBackupRestoreOptions(backupDir string) *BackupOptions {
 	return &BackupOptions{
-		BackupDir:      backupDir,
+		BackupDir:      backupDir, //connection.commonBackupDir(),
 		BackupFileName: beforePortainerVersionUpgradeBackup,
 	}
 }
@@ -76,12 +76,12 @@ func (store *Store) Backup(version *models.Version) (string, error) {
 		return store.backupWithOptions(nil)
 	}
 
-	backupOptions := getBackupRestoreOptions(store.commonBackupDir())
+	return store.backupWithOptions(&BackupOptions{
-	backupOptions.Version = version.SchemaVersion
+		Version: version.SchemaVersion,
-	return store.backupWithOptions(backupOptions)
+	})
 }
 
-func (store *Store) setDefaultBackupOptions(options *BackupOptions) *BackupOptions {
+func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}
 	}
@@ -110,12 +110,12 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 
 	store.createBackupFolders()
 
-	options = store.setDefaultBackupOptions(options)
+	options = store.setupOptions(options)
 	dbPath := store.databasePath()
 
 	if err := store.Close(); err != nil {
 		return options.BackupPath, fmt.Errorf(
-			"error closing datastore before creating backup: %w",
+			"error closing datastore before creating backup: %v",
 			err,
 		)
 	}
@@ -126,7 +126,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 
 	if _, err := store.Open(); err != nil {
 		return options.BackupPath, fmt.Errorf(
-			"error opening datastore after creating backup: %w",
+			"error opening datastore after creating backup: %v",
 			err,
 		)
 	}
@@ -139,18 +139,20 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 // - default: restore latest from current edition
 // - restore a specific
 func (store *Store) restoreWithOptions(options *BackupOptions) error {
-	options = store.setDefaultBackupOptions(options)
+	options = store.setupOptions(options)
 
 	// Check if backup file exist before restoring
 	_, err := os.Stat(options.BackupPath)
 	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist")
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
+
 		return err
 	}
 
 	err = store.Close()
 	if err != nil {
 		log.Error().Err(err).Msg("error while closing store before restore")
+
 		return err
 	}
 
@@ -168,7 +170,7 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 func (store *Store) removeWithOptions(options *BackupOptions) error {
 	log.Info().Msg("removing DB backup")
 
-	options = store.setDefaultBackupOptions(options)
+	options = store.setupOptions(options)
 	_, err := os.Stat(options.BackupPath)
 
 	if os.IsNotExist(err) {