allow nested kube error that's thrown to bubble up

Co-authored-by: testa113 <testa113>

.eslintrc.yml

@@ -23,6 +23,8 @@ parserOptions:
     modules: true
 
 rules:
+  no-console: error
+  no-alert: error
   no-control-regex: 'off'
   no-empty: warn
   no-empty-function: warn
@@ -86,8 +88,8 @@ overrides:
       no-plusplus: off
       func-style: [error, 'declaration']
       import/prefer-default-export: off
-      no-use-before-define: "off"
-      '@typescript-eslint/no-use-before-define': ['error', { functions: false, "allowNamedExports": true }]
+      no-use-before-define: 'off'
+      '@typescript-eslint/no-use-before-define': ['error', { functions: false, 'allowNamedExports': true }]
       no-shadow: 'off'
       '@typescript-eslint/no-shadow': off
       jsx-a11y/no-autofocus: warn

.github/DISCUSSION_TEMPLATE/help.yaml

@@ -0,0 +1,11 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before asking a question, make sure it hasn't been already asked and answered. You can search our [discussions](https://github.com/orgs/portainer/discussions) and [bug reports](https://github.com/portainer/portainer/issues) in GitHub.  Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io/) first.
+
+  - type: textarea
+    attributes:
+      label: Ask a Question!
+    validations:
+      required: true

.github/DISCUSSION_TEMPLATE/ideas.yaml

@@ -0,0 +1,38 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        # Welcome!
+        
+        Thanks for suggesting an idea for Portainer!
+
+        Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion cagetory](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
+
+        Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io) as they may point you toward a solution.
+        
+        **DO NOT FILE DUPLICATE REQUESTS.**
+
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      description: Short list of what the feature request aims to address.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,54 +0,0 @@
----
-name: Bug report
-about: Create a bug report
-title: ''
-labels: bug/need-confirmation, kind/bug
-assignees: ''
----
-
-<!--
-
-Thanks for reporting a bug for Portainer !
-
-You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
-
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
-
-Before opening a new issue, make sure that we do not have any duplicates
-already open. You can ensure this by searching the issue list for this
-repository. If there is a duplicate, please close your issue and add a comment
-to the existing issue instead.
-
-Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
--->
-
-**Bug description**
-A clear and concise description of what the bug is.
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Portainer Logs**
-Provide the logs of your Portainer container or Service.
-You can see how [here](https://documentation.portainer.io/r/portainer-logs)
-
-**Steps to reproduce the issue:**
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Technical details:**
-
-- Portainer version:
-- Docker version (managed by Portainer):
-- Kubernetes version (managed by Portainer):
-- Platform (windows/linux):
-- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
-- Browser:
-- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
-- Have you reviewed our technical documentation and knowledge base? Yes/No
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,164 @@
+name: Bug Report
+description: Create a report to help us improve.
+labels: kind/bug,bug/need-confirmation
+body:
+
+  - type: markdown
+    attributes:
+      value: |
+        # Welcome!
+        
+        The issue tracker is for reporting bugs. If you have an [idea for a new feature](https://github.com/orgs/portainer/discussions/categories/ideas) or a [general question about Portainer](https://github.com/orgs/portainer/discussions/categories/help) please post in our [GitHub Discussions](https://github.com/orgs/portainer/discussions).
+        
+        You can also ask for help in our [community Slack channel](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA).
+        
+        **DO NOT FILE ISSUES FOR GENERAL SUPPORT QUESTIONS**.
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: Before you start please confirm the following.
+      options:
+        - label: Yes, I've searched similar issues on [GitHub](https://github.com/portainer/portainer/issues).
+          required: true
+        - label: Yes, I've checked whether this issue is covered in the Portainer [documentation](https://docs.portainer.io) or [knowledge base](https://portal.portainer.io/knowledge).
+          required: true
+
+  - type: markdown
+    attributes:
+      value: |
+        # About your issue
+
+        Tell us a bit about the issue you're having.
+
+        How to write a good bug report:
+
+        - Respect the issue template as much as possible.
+        - Summarize the issue so that we understand what is going wrong.
+        - Describe what you would have expected to have happened, and what actually happened instead.
+        - Provide easy to follow steps to reproduce the issue. 
+        - Remain clear and concise.
+        - Format your messages to help the reader focus on what matters and understand the structure of your message, use [Markdown syntax](https://help.github.com/articles/github-flavored-markdown).
+
+  - type: textarea
+    attributes:
+      label: Problem Description
+      description: A clear and concise description of what the bug is. 
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of what you expected to happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Actual Behavior
+      description: A clear and concise description of what actually happens.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Steps to Reproduce
+      description: Please be as detailed as possible when providing steps to reproduce.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. Scroll down to '....'
+        4. See error   
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Portainer logs or screenshots
+      description: Provide Portainer container logs or any screenshots related to the issue.
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        # About your environment
+
+        Tell us a bit about your Portainer environment.
+
+  - type: dropdown
+    attributes:
+      label: Portainer version
+      description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
+      multiple: false
+      options:
+        - '2.19.1'
+        - '2.19.0'
+        - '2.18.4'
+        - '2.18.3'
+        - '2.18.2'
+        - '2.18.1'
+        - '2.17.1'
+        - '2.17.0'
+        - '2.16.2'
+        - '2.16.1'
+        - '2.16.0'
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Portainer Edition
+      multiple: false
+      options:
+        - 'Business Edition (BE/EE) with 5NF / 3NF license'
+        - 'Business Edition (BE/EE) with Home & Student license'
+        - 'Business Edition (BE/EE) with Starter license'
+        - 'Business Edition (BE/EE) with Professional or Enterprise license'
+        - 'Community Edition (CE)'
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Platform and Version
+      description: |
+        Enter your container management platform (Docker | Swarm | Kubernetes) along with the version. 
+        Example: Docker 24.0.3 | Docker Swarm 24.0.3 | Kubernetes 1.26
+        You can find our supported platforms [in our documentation](https://docs.portainer.io/start/requirements-and-prerequisites).
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: OS and Architecture
+      description: |
+        Enter your Operating System, Version and Architecture. Example: Ubuntu 22.04, AMD64 | Raspbian OS, ARM64
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Browser
+      description: | 
+        Enter your browser and version. Example: Google Chrome 114.0
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: What command did you use to deploy Portainer?
+      description: |
+        Example: `docker run -d -p 8000:8000 -p 9443:9443 --name portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce:latest`
+        If you deployed Portainer using a compose file or manifest you can provide this here as well.
+      render: bash
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: Additional Information
+      description: Any additional information about your environment, the bug, or anything else you think might be helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,11 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Portainer Business Edition - Get 5 nodes free
-    url: https://portainer.io/pricing/take5 
-    about: Portainer Business Edition has more features, more support and you can now get 5 nodes free for as long as you want.
+  - name: Question
+    url: https://github.com/orgs/portainer/discussions/new?category=help
+    about: Ask us a question about Portainer usage or deployment.
+  - name: Idea or Feature Request
+    url: https://github.com/orgs/portainer/discussions/new?category=ideas
+    about: Suggest an idea or feature/enhancement that should be added in Portainer.
+  - name: Portainer Business Edition - Get 3 Nodes Free
+    url: https://www.portainer.io/take-3
+    about: Portainer Business Edition has more features, more support and you can now get 3 nodes free for as long as you want.

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
           cache: 'yarn'
       - uses: actions/setup-go@v4
         with:
-          go-version: 1.19.5
+          go-version: 1.21.0
       - run: yarn --frozen-lockfile
       - name: Run linters
         uses: wearerequired/lint-action@v1
@@ -41,6 +41,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: latest
+          version: v1.54.1
           working-directory: api
           args: --timeout=10m -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -61,7 +61,7 @@ jobs:
       - name: install Go 
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: '1.21.0'
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -72,7 +72,7 @@ jobs:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
       - name: upload scan result as develop artifact 
         uses: actions/upload-artifact@v3

.github/workflows/pr-security.yml

@@ -7,7 +7,7 @@ on:
       - edited
     paths:
       - 'package.json'
-      - 'api/go.mod'
+      - 'go.mod'
       - 'gruntfile.js'
       - 'build/linux/Dockerfile'
       - 'build/linux/alpine.Dockerfile'
@@ -84,7 +84,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: '1.21.0'
 
       - name: download Go modules
         run: cd ./api && go get -t -v -d ./...
@@ -95,7 +95,7 @@ jobs:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: |
           yarn global add snyk
-          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
+          snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
 
       - name: upload scan result as pull-request artifact
         uses: actions/upload-artifact@v3
@@ -143,10 +143,10 @@ jobs:
       - name: checkout code
         uses: actions/checkout@master
 
-      - name: install Go 1.19.5
+      - name: install Go 1.21.0
         uses: actions/setup-go@v3
         with:
-          go-version: '1.19.5'
+          go-version: '1.21.0'
 
       - name: install Node.js 18.x
         uses: actions/setup-node@v3

.github/workflows/stale.yml

@@ -1,7 +1,8 @@
 name: Close Stale Issues
 on:
   schedule:
-  - cron: '0 12 * * *'
+    - cron: '0 12 * * *'
+  workflow_dispatch:
 jobs:
   stale:
     runs-on: ubuntu-latest
@@ -9,7 +10,7 @@ jobs:
       issues: write
 
     steps:
-    - uses: actions/stale@v4.0.0
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/test.yaml

@@ -13,17 +13,13 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Run tests
-        run: yarn jest --maxWorkers=2
-  # test-server:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     GOPRIVATE: "github.com/portainer"
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - uses: actions/setup-go@v3
-  #       with:
-  #         go-version: '1.18'
-  #     - name: Run tests
-  #       run: |
-  #         cd api
-  #         go test ./...
+        run: make test-client ARGS="--maxWorkers=2"
+  test-server:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.0
+      - name: Run tests
+        run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -15,13 +15,13 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.21.0'
 
       - name: Download golang modules
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

.gitignore

@@ -11,7 +11,7 @@ storybook-static
 *.DS_Store
 
 .eslintcache
-__debug_bin
+__debug_bin*
 
 api/docs
 .idea

.prettierrc

@@ -2,18 +2,24 @@
   "printWidth": 180,
   "singleQuote": true,
   "htmlWhitespaceSensitivity": "strict",
+  "trailingComma": "es5",
   "overrides": [
     {
-      "files": ["*.html"],
+      "files": [
+        "*.html"
+      ],
       "options": {
         "parser": "angular"
       }
     },
     {
-      "files": ["*.{j,t}sx", "*.ts"],
+      "files": [
+        "*.{j,t}sx",
+        "*.ts"
+      ],
       "options": {
         "printWidth": 80
       }
     }
   ]
-}
+}

.storybook/main.js

@@ -1,58 +0,0 @@
-const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
-
-module.exports = {
-  stories: ['../app/**/*.stories.mdx', '../app/**/*.stories.@(ts|tsx)'],
-  addons: [
-    '@storybook/addon-links',
-    '@storybook/addon-essentials',
-    {
-      name: '@storybook/addon-postcss',
-      options: {
-        cssLoaderOptions: {
-          importLoaders: 1,
-          modules: {
-            localIdentName: '[path][name]__[local]',
-            auto: true,
-            exportLocalsConvention: 'camelCaseOnly',
-          },
-        },
-        postcssLoaderOptions: {
-          implementation: require('postcss'),
-        },
-      },
-    },
-  ],
-  webpackFinal: (config) => {
-    config.resolve.plugins = [
-      ...(config.resolve.plugins || []),
-      new TsconfigPathsPlugin({
-        extensions: config.resolve.extensions,
-      }),
-    ];
-
-    const svgRule = config.module.rules.find((rule) => rule.test && typeof rule.test.test === 'function' && rule.test.test('.svg'));
-    svgRule.test = new RegExp(svgRule.test.source.replace('svg|', ''));
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      type: 'asset',
-      resourceQuery: { not: [/c/] }, // exclude react component if *.svg?url
-    });
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      issuer: /\.(js|ts)(x)?$/,
-      resourceQuery: /c/, // *.svg?c
-      use: [{ loader: '@svgr/webpack', options: { icon: true } }],
-    });
-
-    return config;
-  },
-  core: {
-    builder: 'webpack5',
-  },
-  staticDirs: ['./public'],
-  typescript: {
-    reactDocgen: 'react-docgen-typescript-plugin',
-  },
-};

.storybook/main.ts

@@ -0,0 +1,95 @@
+import { StorybookConfig } from '@storybook/react-webpack5';
+
+import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
+import { Configuration } from 'webpack';
+import postcss from 'postcss';
+const config: StorybookConfig = {
+  stories: ['../app/**/*.stories.@(ts|tsx)'],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    {
+      name: '@storybook/addon-styling',
+      options: {
+        cssLoaderOptions: {
+          importLoaders: 1,
+          modules: {
+            localIdentName: '[path][name]__[local]',
+            auto: true,
+            exportLocalsConvention: 'camelCaseOnly',
+          },
+        },
+        postCss: {
+          implementation: postcss,
+        },
+      },
+    },
+  ],
+  webpackFinal: (config) => {
+    const rules = config?.module?.rules || [];
+
+    const imageRule = rules.find((rule) => {
+      const test = (rule as { test: RegExp }).test;
+
+      if (!test) {
+        return false;
+      }
+
+      return test.test('.svg');
+    }) as { [key: string]: any };
+
+    imageRule.exclude = /\.svg$/;
+
+    rules.unshift({
+      test: /\.svg$/i,
+      type: 'asset',
+      resourceQuery: {
+        not: [/c/],
+      }, // exclude react component if *.svg?url
+    });
+
+    rules.unshift({
+      test: /\.svg$/i,
+      issuer: /\.(js|ts)(x)?$/,
+      resourceQuery: /c/,
+      // *.svg?c
+      use: [
+        {
+          loader: '@svgr/webpack',
+          options: {
+            icon: true,
+          },
+        },
+      ],
+    });
+    return {
+      ...config,
+      resolve: {
+        ...config.resolve,
+        plugins: [
+          ...(config.resolve?.plugins || []),
+          new TsconfigPathsPlugin({
+            extensions: config.resolve?.extensions,
+          }),
+        ],
+      },
+      module: {
+        ...config.module,
+        rules,
+      },
+    } satisfies Configuration;
+  },
+  staticDirs: ['./public'],
+  typescript: {
+    reactDocgen: 'react-docgen-typescript',
+  },
+  framework: {
+    name: '@storybook/react-webpack5',
+    options: {},
+  },
+  docs: {
+    autodocs: true,
+  },
+};
+
+export default config;

Makefile

@@ -65,7 +65,7 @@ clean: ## Remove all build and download artifacts
 test: test-server test-client ## Run all tests
 
 test-client: ## Run client tests
-	yarn test
+	yarn test $(ARGS)
 
 test-server:	## Run server tests
 	cd api && $(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover  ./...

README.md

@@ -9,7 +9,7 @@ Portainer consists of a single container that can run on any cluster. It can be
 **Portainer Business Edition** builds on the open-source base and includes a range of advanced features and functions (like RBAC and Support) that are specific to the needs of business users.
 
 - [Compare Portainer CE and Compare Portainer BE](https://portainer.io/products)
-- [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
+- [Take3 – get 3 free nodes of Portainer Business for as long as you want them](https://www.portainer.io/take-3)
 - [Portainer BE install guide](https://install.portainer.io)
 
 ## Latest Version
@@ -21,8 +21,8 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
 ## Getting started
 
 - [Deploy Portainer](https://docs.portainer.io/start/install)
-- [Documentation](https://documentation.portainer.io)
-- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
+- [Documentation](https://docs.portainer.io)
+- [Contribute to the project](https://docs.portainer.io/contribute/contribute)
 
 ## Features & Functions
 
@@ -30,23 +30,22 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 - [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
 - [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
-- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
 
 ## Getting help
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
+Learn more about Portainer's community support channels [here.](https://www.portainer.io/get-support-for-portainer)
 
 - Issues: https://github.com/portainer/portainer/issues
 - Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
 
-You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
+You can join the Portainer Community by visiting [https://www.portainer.io/join-our-community](https://www.portainer.io/join-our-community). This will give you advance notice of events, content and other related Portainer content.
 
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
-- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://documentation.portainer.io/contributing/instructions/) to build it locally and make a pull request.
+- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
 
 ## Security
 
@@ -60,7 +59,7 @@ If you are a developer, and our code in this repo makes sense to you, we would l
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
 
-When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/documentation/in-app-analytics-and-privacy-policy/). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
+When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
 
 ## Limitations
 

api/.golangci.yaml

@@ -10,15 +10,21 @@ linters:
     - exportloopref
 linters-settings:
   depguard:
-    list-type: denylist
-    include-go-root: true
-    packages:
-      - github.com/sirupsen/logrus
-      - golang.org/x/exp
-    packages-with-error-message:
-      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
-    ignore-file-rules:
-      - '**/*_test.go'
+    rules:
+      main:
+        deny:
+          - pkg: 'github.com/sirupsen/logrus'
+            desc: 'logging is allowed only by github.com/rs/zerolog'
+          - pkg: 'golang.org/x/exp'
+            desc: 'exp is not allowed'
+          - pkg: 'github.com/portainer/libcrypto'
+            desc: 'use github.com/portainer/portainer/pkg/libcrypto'
+          - pkg: 'github.com/portainer/libhttp'
+            desc: 'use github.com/portainer/portainer/pkg/libhttp'
+        files:
+          - '!**/*_test.go'
+          - '!**/base.go'
+          - '!**/base_tx.go'
 
 # errorlint is causing a typecheck error for some reason. The go compiler will report these
 # anyway, so ignore them from the linter

api/adminmonitor/admin_monitor.go

@@ -7,9 +7,9 @@ import (
 	"sync"
 	"time"
 
-	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 
 	"github.com/rs/zerolog/log"
 )

api/apikey/apikey.go

@@ -1,9 +1,6 @@
 package apikey
 
 import (
-	"crypto/rand"
-	"io"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -18,13 +15,3 @@ type APIKeyService interface {
 	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
 	InvalidateUserKeyCache(userId portainer.UserID) bool
 }
-
-// generateRandomKey generates a random key of specified length
-// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
-func generateRandomKey(length int) []byte {
-	k := make([]byte, length)
-	if _, err := io.ReadFull(rand.Reader, k); err != nil {
-		return nil
-	}
-	return k
-}

api/apikey/apikey_test.go

@@ -3,6 +3,7 @@ package apikey
 import (
 	"testing"
 
+	"github.com/portainer/portainer/api/internal/securecookie"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,7 +34,7 @@ func Test_generateRandomKey(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := generateRandomKey(tt.wantLenth)
+			got := securecookie.GenerateRandomKey(tt.wantLenth)
 			is.Equal(tt.wantLenth, len(got))
 		})
 	}
@@ -41,7 +42,7 @@ func Test_generateRandomKey(t *testing.T) {
 	t.Run("Generated keys are unique", func(t *testing.T) {
 		keys := make(map[string]bool)
 		for i := 0; i < 100; i++ {
-			key := generateRandomKey(8)
+			key := securecookie.GenerateRandomKey(8)
 			_, ok := keys[string(key)]
 			is.False(ok)
 			keys[string(key)] = true

api/apikey/service.go

@@ -8,6 +8,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/securecookie"
 
 	"github.com/pkg/errors"
 )
@@ -39,7 +40,7 @@ func (a *apiKeyService) HashRaw(rawKey string) []byte {
 // GenerateApiKey generates a raw API key for a user (for one-time display).
 // The generated API key is stored in the cache and database.
 func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
-	randKey := generateRandomKey(32)
+	randKey := securecookie.GenerateRandomKey(32)
 	encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
 	prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
 
@@ -53,7 +54,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 		Digest:      hashDigest,
 	}
 
-	err := a.apiKeyRepository.CreateAPIKey(apiKey)
+	err := a.apiKeyRepository.Create(apiKey)
 	if err != nil {
 		return "", nil, errors.Wrap(err, "Unable to create API key")
 	}
@@ -66,7 +67,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 
 // GetAPIKey returns an API key by its ID.
 func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	return a.apiKeyRepository.GetAPIKey(apiKeyID)
+	return a.apiKeyRepository.Read(apiKeyID)
 }
 
 // GetAPIKeys returns all the API keys associated to a user.
@@ -88,7 +89,7 @@ func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, port
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve API key")
 	}
 
-	user, err := a.userRepository.User(apiKey.UserID)
+	user, err := a.userRepository.Read(apiKey.UserID)
 	if err != nil {
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve digest user")
 	}
@@ -106,20 +107,20 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
 		return errors.Wrap(err, "Unable to retrieve API key")
 	}
 	a.cache.Set(apiKey.Digest, user, *apiKey)
-	return a.apiKeyRepository.UpdateAPIKey(apiKey)
+	return a.apiKeyRepository.Update(apiKey.ID, apiKey)
 }
 
 // DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
 func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
 	// get api-key digest to remove from cache
-	apiKey, err := a.apiKeyRepository.GetAPIKey(apiKeyID)
+	apiKey, err := a.apiKeyRepository.Read(apiKeyID)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
 	}
 
 	// delete the user/api-key from cache
 	a.cache.Delete(apiKey.Digest)
-	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
+	return a.apiKeyRepository.Delete(apiKeyID)
 }
 
 func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {

api/backup/backup.go

@@ -30,6 +30,7 @@ var filesToBackup = []string{
 	"portainer.key",
 	"portainer.pub",
 	"tls",
+	"chisel",
 }
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
@@ -85,9 +86,9 @@ func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
 	if err != nil {
 		return err
 	}
-	// if err = datastore.BackupTo(backupWriter); err != nil {
-	// 	return err
-	// }
+	if err = datastore.BackupTo(backupWriter); err != nil {
+		return err
+	}
 	return backupWriter.Close()
 }
 

api/backup/restore.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
@@ -97,17 +98,16 @@ func restoreFiles(srcDir string, destinationDir string) error {
 	// TODO:  This is very boltdb module specific once again due to the filename.  Move to bolt module? Refactor for another day
 
 	// Prevent the possibility of having both databases.  Remove any default new instance
-	// os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
-	// os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
+	os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
+	os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
 
-	// // Now copy the database.  It'll be either portainer.db or portainer.edb
+	// Now copy the database.  It'll be either portainer.db or portainer.edb
 
-	// // Note: CopyPath does not return an error if the source file doesn't exist
-	// err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
-	// if err != nil {
-	// 	return err
-	// }
+	// Note: CopyPath does not return an error if the source file doesn't exist
+	err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
+	if err != nil {
+		return err
+	}
 
-	// return filesystem.CopyPath(filepath.Join(srcDir, boltdb.DatabaseFileName), destinationDir)
-	return nil
+	return filesystem.CopyPath(filepath.Join(srcDir, boltdb.DatabaseFileName), destinationDir)
 }

api/chisel/crypto/crypto.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+
+	chshare "github.com/jpillora/chisel/share"
+)
+
+var one = new(big.Int).SetInt64(1)
+
+// GenerateGo119CompatibleKey This function is basically copied from chshare.GenerateKey.
+func GenerateGo119CompatibleKey(seed string) ([]byte, error) {
+	r := chshare.NewDetermRand([]byte(seed))
+	priv, err := ecdsaGenerateKey(elliptic.P256(), r)
+	if err != nil {
+		return nil, err
+	}
+	b, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to marshal ECDSA private key: %w", err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil
+}
+
+// This function is copied from Go1.19
+func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
+	params := c.Params()
+	// Note that for P-521 this will actually be 63 bits more than the order, as
+	// division rounds down, but the extra bit is inconsequential.
+	b := make([]byte, params.N.BitLen()/8+8)
+	_, err = io.ReadFull(rand, b)
+	if err != nil {
+		return
+	}
+
+	k = new(big.Int).SetBytes(b)
+	n := new(big.Int).Sub(params.N, one)
+	k.Mod(k, n)
+	k.Add(k, one)
+	return
+}
+
+// This function is copied from Go1.19
+func ecdsaGenerateKey(c elliptic.Curve, rand io.Reader) (*ecdsa.PrivateKey, error) {
+	k, err := randFieldElement(c, rand)
+	if err != nil {
+		return nil, err
+	}
+
+	priv := new(ecdsa.PrivateKey)
+	priv.PublicKey.Curve = c
+	priv.D = k
+	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
+	return priv, nil
+}

api/chisel/crypto/crypto_test.go

@@ -0,0 +1,37 @@
+package crypto
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGenerateGo119CompatibleKey(t *testing.T) {
+	type args struct {
+		seed string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name:    "Generate Go 1.19 compatible private key with a given seed",
+			args:    args{seed: "94qh17MCIk8BOkiI"},
+			want:    []byte("-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIHeohwk0Gy3RHVVViaHz7pz/HOiqA7fkv1FTM3mGgfT3oAoGCCqGSM49\nAwEHoUQDQgAEN7riX06xDsLNPuUmOvYFluNEakcFwZZRVvOcIYk/9VYnanDzW0Km\n8/BUUiKyJDuuGdS4fj9SlQ4iL8yBK01uKg==\n-----END EC PRIVATE KEY-----\n"),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GenerateGo119CompatibleKey(tt.args.seed)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GenerateGo119CompatibleKey() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GenerateGo119CompatibleKey()\ngot: Z %v\nwant: %v", got, tt.want)
+			}
+		})
+	}
+}

api/chisel/schedules.go

@@ -18,6 +18,7 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	for idx, existingJob := range tunnel.Jobs {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
+
 			break
 		}
 	}

api/chisel/service.go

@@ -12,8 +12,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 
-	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
+	"github.com/jpillora/chisel/share/ccrypto"
 	"github.com/rs/zerolog/log"
 )
 
@@ -36,14 +36,16 @@ type Service struct {
 	shutdownCtx       context.Context
 	ProxyManager      *proxy.Manager
 	mu                sync.Mutex
+	fileService       portainer.FileService
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
+func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
 	return &Service{
 		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
 		dataStore:        dataStore,
 		shutdownCtx:      shutdownCtx,
+		fileService:      fileService,
 	}
 }
 
@@ -73,10 +75,11 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
 			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("start")
+			Msg("KeepTunnelAlive: start")
 
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
+
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
 		defer pingTicker.Stop()
 
@@ -89,13 +92,13 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 					log.Debug().
 						Int("endpoint_id", int(endpointID)).
 						Err(err).
-						Msg("ping agent")
+						Msg("KeepTunnelAlive: ping agent")
 				}
 			case <-maxAliveTicker.C:
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("tunnel keep alive timeout")
+					Msg("KeepTunnelAlive: tunnel keep alive timeout")
 
 				return
 			case <-ctx.Done():
@@ -103,7 +106,7 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("tunnel stop")
+					Msg("KeepTunnelAlive: tunnel stop")
 
 				return
 			}
@@ -117,14 +120,15 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // It starts the tunnel status verification process in the background.
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
-	keySeed, err := service.retrievePrivateKeySeed()
+	privateKeyFile, err := service.retrievePrivateKeyFile()
+
 	if err != nil {
 		return err
 	}
 
 	config := &chserver.Config{
 		Reverse: true,
-		KeySeed: keySeed,
+		KeyFile: privateKeyFile,
 	}
 
 	chiselServer, err := chserver.NewServer(config)
@@ -160,26 +164,41 @@ func (service *Service) StopTunnelServer() error {
 	return service.chiselServer.Close()
 }
 
-func (service *Service) retrievePrivateKeySeed() (string, error) {
-	var serverInfo *portainer.TunnelServerInfo
+func (service *Service) retrievePrivateKeyFile() (string, error) {
+	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
 
-	serverInfo, err := service.dataStore.TunnelServer().Info()
-	if service.dataStore.IsErrObjectNotFound(err) {
-		keySeed := uniuri.NewLen(16)
+	exist, _ := service.fileService.FileExists(privateKeyFile)
+	if !exist {
+		log.Debug().
+			Str("private-key", privateKeyFile).
+			Msg("Chisel private key file does not exist")
 
-		serverInfo = &portainer.TunnelServerInfo{
-			PrivateKeySeed: keySeed,
-		}
-
-		err := service.dataStore.TunnelServer().UpdateInfo(serverInfo)
+		privateKey, err := ccrypto.GenerateKey("")
 		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to generate chisel private key")
 			return "", err
 		}
-	} else if err != nil {
-		return "", err
+
+		err = service.fileService.StoreChiselPrivateKey(privateKey)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to save Chisel private key to disk")
+			return "", err
+		} else {
+			log.Info().
+				Str("private-key", privateKeyFile).
+				Msg("Generated a new Chisel private key file")
+		}
+	} else {
+		log.Info().
+			Str("private-key", privateKeyFile).
+			Msg("Found Chisel private key file on disk")
 	}
 
-	return serverInfo.PrivateKeySeed, nil
+	return privateKeyFile, nil
 }
 
 func (service *Service) startTunnelVerificationLoop() {
@@ -271,14 +290,7 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 		return err
 	}
 
-	endpointURL := endpoint.URL
-
 	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
-	err = service.snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		return err
-	}
 
-	endpoint.URL = endpointURL
-	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
+	return service.snapshotService.SnapshotEndpoint(endpoint)
 }

api/chisel/tunnel.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/portainer/libcrypto"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge/cache"
+	"github.com/portainer/portainer/pkg/libcrypto"
 
 	"github.com/dchest/uniuri"
 )
@@ -126,7 +126,10 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 	credentials := tunnel.Credentials
 	if credentials != "" {
 		tunnel.Credentials = ""
-		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
+
+		if service.chiselServer != nil {
+			service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
+		}
 	}
 
 	service.ProxyManager.DeleteEndpointProxy(endpointID)
@@ -161,9 +164,12 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
 
 		username, password := generateRandomCredentials()
 		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-		err = service.chiselServer.AddUser(username, password, authorizedRemote)
-		if err != nil {
-			return err
+
+		if service.chiselServer != nil {
+			err = service.chiselServer.AddUser(username, password, authorizedRemote)
+			if err != nil {
+				return err
+			}
 		}
 
 		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)

api/cli/pairlist.go

@@ -9,7 +9,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-type pairList portainer.MultiPair
+type pairList []portainer.Pair
 
 // Set implementation for a list of portainer.Pair
 func (l *pairList) Set(value string) error {
@@ -19,7 +19,7 @@ func (l *pairList) Set(value string) error {
 	}
 	p := new(portainer.Pair)
 	p.Name = parts[0]
-	p.Value1 = parts[1]
+	p.Value = parts[1]
 	*l = append(*l, *p)
 	return nil
 }
@@ -34,8 +34,8 @@ func (l *pairList) IsCumulative() bool {
 	return true
 }
 
-func pairs(s kingpin.Settings) (target *portainer.MultiPair) {
-	target = new(portainer.MultiPair)
+func pairs(s kingpin.Settings) (target *[]portainer.Pair) {
+	target = new([]portainer.Pair)
 	s.SetValue((*pairList)(target))
 	return
 }

api/cli/pairlistbool.go

@@ -8,7 +8,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-type pairListBool portainer.MultiPair
+type pairListBool []portainer.Pair
 
 // Set implementation for a list of portainer.Pair
 func (l *pairListBool) Set(value string) error {
@@ -18,10 +18,10 @@ func (l *pairListBool) Set(value string) error {
 	parts := strings.SplitN(value, "=", 2)
 	if len(parts) != 2 {
 		p.Name = parts[0]
-		p.Value1 = "true"
+		p.Value = "true"
 	} else {
 		p.Name = parts[0]
-		p.Value1 = parts[1]
+		p.Value = parts[1]
 	}
 
 	*l = append(*l, *p)
@@ -38,8 +38,8 @@ func (l *pairListBool) IsCumulative() bool {
 	return true
 }
 
-func BoolPairs(s kingpin.Settings) (target *portainer.MultiPair) {
-	target = new(portainer.MultiPair)
+func BoolPairs(s kingpin.Settings) (target *[]portainer.Pair) {
+	target = new([]portainer.Pair)
 	s.SetValue((*pairListBool)(target))
 	return
 }

api/cmd/portainer/log.go

@@ -39,9 +39,9 @@ func setLoggingMode(mode string) {
 	case "PRETTY":
 		log.Logger = log.Output(zerolog.ConsoleWriter{
 			Out:           os.Stderr,
-			NoColor:       true,
 			TimeFormat:    "2006/01/02 03:04PM",
-			FormatMessage: formatMessage})
+			FormatMessage: formatMessage,
+		})
 	case "JSON":
 		log.Logger = log.Output(os.Stderr)
 	}
@@ -51,5 +51,6 @@ func formatMessage(i interface{}) string {
 	if i == nil {
 		return ""
 	}
+
 	return fmt.Sprintf("%s |", i)
 }

api/cmd/portainer/main.go

@@ -9,8 +9,6 @@ import (
 	"strings"
 	"time"
 
-	libstack "github.com/portainer/docker-compose-wrapper"
-	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/build"
@@ -18,9 +16,11 @@ import (
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/database"
+	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -29,7 +29,6 @@ import (
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/hostmanagement/openamt"
 	"github.com/portainer/portainer/api/http"
-	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/authorization"
@@ -44,10 +43,13 @@ import (
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/pendingactions"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/pkg/featureflags"
 	"github.com/portainer/portainer/pkg/libhelm"
+	"github.com/portainer/portainer/pkg/libstack"
+	"github.com/portainer/portainer/pkg/libstack/compose"
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
@@ -78,11 +80,19 @@ func initFileService(dataStorePath string) portainer.FileService {
 }
 
 func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
-	connection, err := database.NewDatabase("sqlite", *flags.Data, secretKey)
+	connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating database connection")
 	}
 
+	if bconn, ok := connection.(*boltdb.DbConnection); ok {
+		bconn.MaxBatchSize = *flags.MaxBatchSize
+		bconn.MaxBatchDelay = *flags.MaxBatchDelay
+		bconn.InitialMmapSize = *flags.InitialMmapSize
+	} else {
+		log.Fatal().Msg("failed creating database connection: expecting a boltdb database type but a different one was received")
+	}
+
 	store := datastore.NewStore(*flags.Data, fileService, connection)
 	isNew, err := store.Open()
 	if err != nil {
@@ -111,30 +121,26 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 			log.Fatal().Err(err).Msg("failed generating instance id")
 		}
 
+		migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{})
+		migratorCount := migratorInstance.GetMigratorCountOfCurrentAPIVersion()
+
 		// from MigrateData
-		v := map[string]interface{}{
-			models.SchemaVersionKey: portainer.APIVersion,
-			models.EditionKey:       int(portainer.PortainerCE),
-			models.InstanceKey:      instanceId.String(),
-		}
-		err = store.VersionService.UpdateAll(v)
-		if err != nil {
-			log.Fatal().Err(err).Msg("failed inserting version bucket")
+		v := models.Version{
+			SchemaVersion: portainer.APIVersion,
+			Edition:       int(portainer.PortainerCE),
+			InstanceID:    instanceId.String(),
+			MigratorCount: migratorCount,
 		}
+		store.VersionService.UpdateVersion(&v)
 
 		err = updateSettingsFromFlags(store, flags)
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed updating settings from flags")
 		}
 	} else {
-		// err = store.MigrateData()
-		// if err != nil {
-		// 	log.Fatal().Err(err).Msg("failed migration")
-		// }
-
-		err := store.PostInit()
+		err = store.MigrateData()
 		if err != nil {
-			log.Fatal().Err(err).Msg("postinit failed")
+			log.Fatal().Err(err).Msg("failed migration")
 		}
 	}
 
@@ -152,7 +158,17 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	return store
 }
 
-func initComposeStackManager(composeDeployer libstack.Deployer, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+// checkDBSchemaServerVersionMatch checks if the server version matches the db scehma version
+func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersion string, serverEdition int) bool {
+	v, err := dbStore.Version().Version()
+	if err != nil {
+		return false
+	}
+
+	return v.SchemaVersion == serverVersion && v.Edition == serverEdition
+}
+
+func initComposeStackManager(composeDeployer libstack.Deployer, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating compose manager")
@@ -248,11 +264,12 @@ func initSnapshotService(
 	dockerClientFactory *dockerclient.ClientFactory,
 	kubernetesClientFactory *kubecli.ClientFactory,
 	shutdownCtx context.Context,
+	pendingActionsService *pendingactions.PendingActionsService,
 ) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
+	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx, pendingActionsService)
 	if err != nil {
 		return nil, err
 	}
@@ -347,143 +364,6 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	tlsConfiguration := portainer.TLSConfiguration{
-		TLS:           *flags.TLS,
-		TLSSkipVerify: *flags.TLSSkipVerify,
-	}
-
-	if *flags.TLS {
-		tlsConfiguration.TLSCACertPath = *flags.TLSCacert
-		tlsConfiguration.TLSCertPath = *flags.TLSCert
-		tlsConfiguration.TLSKeyPath = *flags.TLSKey
-	} else if !*flags.TLS && *flags.TLSSkipVerify {
-		tlsConfiguration.TLS = true
-	}
-
-	endpoint := &portainer.Endpoint{
-		Name:               "primary",
-		URL:                *flags.EndpointURL,
-		GroupID:            portainer.EndpointGroupID(1),
-		Type:               portainer.DockerEnvironment,
-		TLSConfig:          tlsConfiguration,
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
-	}
-
-	if strings.HasPrefix(endpoint.URL, "tcp://") {
-		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
-		if err != nil {
-			return err
-		}
-
-		agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig)
-		if err != nil {
-			return err
-		}
-
-		if agentOnDockerEnvironment {
-			endpoint.Type = portainer.AgentOnDockerEnvironment
-		}
-	}
-
-	err := snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		log.Error().
-			Str("endpoint", endpoint.Name).
-			Str("URL", endpoint.URL).
-			Err(err).
-			Msg("environment snapshot error")
-	}
-
-	return dataStore.Endpoint().Create(endpoint)
-}
-
-func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	if strings.HasPrefix(endpointURL, "tcp://") {
-		_, err := client.ExecutePingOperation(endpointURL, nil)
-		if err != nil {
-			return err
-		}
-	}
-
-	endpoint := &portainer.Endpoint{
-		Name:               "primary",
-		URL:                endpointURL,
-		GroupID:            portainer.EndpointGroupID(1),
-		Type:               portainer.DockerEnvironment,
-		TLSConfig:          portainer.TLSConfiguration{},
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
-	}
-
-	err := snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		log.Error().
-			Str("endpoint", endpoint.Name).
-			Str("URL", endpoint.URL).Err(err).
-			Msg("environment snapshot error")
-	}
-
-	return dataStore.Endpoint().Create(endpoint)
-}
-
-func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	if *flags.EndpointURL == "" {
-		return nil
-	}
-
-	endpoints, err := dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	if len(endpoints) > 0 {
-		log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
-
-		return nil
-	}
-
-	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, dataStore, snapshotService)
-	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
-}
-
 func loadEncryptionSecretKey(keyfilename string) []byte {
 	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
 	if err != nil {
@@ -520,6 +400,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("")
 	}
 
+	// check if the db schema version matches with server version
+	if !checkDBSchemaServerVersionMatch(dataStore, portainer.APIVersion, int(portainer.Edition)) {
+		log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
+	}
+
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed getting instance id")
@@ -566,20 +451,22 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
+	authorizationService := authorization.NewService(dataStore)
+	authorizationService.K8sClientFactory = kubernetesClientFactory
+
+	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
+
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed initializing snapshot service")
 	}
 	snapshotService.Start()
 
-	authorizationService := authorization.NewService(dataStore)
-	authorizationService.K8sClientFactory = kubernetesClientFactory
-
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
 	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
@@ -595,7 +482,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing compose deployer")
 	}
 
-	composeStackManager := initComposeStackManager(composeDeployer, reverseTunnelService, proxyManager)
+	composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
@@ -739,6 +626,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		DemoService:                 demoService,
 		UpgradeService:              upgradeService,
 		AdminCreationDone:           adminCreationDone,
+		PendingActionsService:       pendingActionsService,
 	}
 }
 

api/connection.go

@@ -1,6 +1,8 @@
 package portainer
 
-import "gorm.io/gorm"
+import (
+	"io"
+)
 
 type ReadTransaction interface {
 	GetObject(bucketName string, key []byte, object interface{}) error
@@ -23,15 +25,19 @@ type Transaction interface {
 }
 
 type Connection interface {
+	Transaction
+
 	Open() error
 	Close() error
-	Init() error
-	GetDB() *gorm.DB
 
-	GetByID(ID int, obj interface{}) error
-	DeleteByID(ID int, obj interface{}) error
+	UpdateTx(fn func(Transaction) error) error
+	ViewTx(fn func(Transaction) error) error
+
+	// write the db contents to filename as json (the schema needs defining)
+	ExportRaw(filename string) error
 
 	// TODO: this one is very database specific atm
+	BackupTo(w io.Writer) error
 	GetDatabaseFileName() string
 	GetDatabaseFilePath() string
 	GetStorePath() string
@@ -39,4 +45,10 @@ type Connection interface {
 	IsEncryptedStore() bool
 	NeedsEncryptionMigration() (bool, error)
 	SetEncrypted(encrypted bool)
+
+	BackupMetadata() (map[string]interface{}, error)
+	RestoreMetadata(s map[string]interface{}) error
+
+	UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
+	ConvertToKey(v int) []byte
 }

api/crypto/ecdsa.go

@@ -8,7 +8,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 
-	"github.com/portainer/libcrypto"
+	"github.com/portainer/portainer/pkg/libcrypto"
 )
 
 const (

api/database/boltdb/export.go

@@ -0,0 +1,112 @@
+package boltdb
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	bolt "go.etcd.io/bbolt"
+)
+
+func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
+	buckets := map[string]interface{}{}
+
+	err := connection.View(func(tx *bolt.Tx) error {
+		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+			bucketName := string(name)
+			seqId := bucket.Sequence()
+			buckets[bucketName] = int(seqId)
+			return nil
+		})
+
+		return err
+	})
+
+	return buckets, err
+}
+
+// ExportJSON creates a JSON representation from a DbConnection. You can include
+// the database's metadata or ignore it. Ensure the database is closed before
+// using this function.
+// inspired by github.com/konoui/boltdb-exporter (which has no license)
+// but very much simplified, based on how we use boltdb
+func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, error) {
+	log.Debug().Str("databasePath", databasePath).Msg("exportJson")
+
+	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
+	if err != nil {
+		return []byte("{}"), err
+	}
+	defer connection.Close()
+
+	backup := make(map[string]interface{})
+	if metadata {
+		meta, err := backupMetadata(connection)
+		if err != nil {
+			log.Error().Err(err).Msg("failed exporting metadata")
+		}
+
+		backup["__metadata"] = meta
+	}
+
+	err = connection.View(func(tx *bolt.Tx) error {
+		err = tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+			bucketName := string(name)
+			var list []interface{}
+			version := make(map[string]string)
+			cursor := bucket.Cursor()
+			for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+				if v == nil {
+					continue
+				}
+
+				var obj interface{}
+				err := c.UnmarshalObject(v, &obj)
+				if err != nil {
+					log.Error().
+						Str("bucket", bucketName).
+						Str("object", string(v)).
+						Err(err).
+						Msg("failed to unmarshal")
+
+					obj = v
+				}
+
+				if bucketName == "version" {
+					version[string(k)] = string(v)
+				} else {
+					list = append(list, obj)
+				}
+			}
+
+			if bucketName == "version" {
+				backup[bucketName] = version
+				return nil
+			}
+
+			if len(list) > 0 {
+				if bucketName == "ssl" ||
+					bucketName == "settings" ||
+					bucketName == "tunnel_server" {
+					backup[bucketName] = nil
+					if len(list) > 0 {
+						backup[bucketName] = list[0]
+					}
+					return nil
+				}
+				backup[bucketName] = list
+				return nil
+			}
+
+			return nil
+		})
+
+		return err
+	})
+
+	if err != nil {
+		return []byte("{}"), err
+	}
+
+	return json.MarshalIndent(backup, "", "  ")
+}

api/database/boltdb/json.go

@@ -0,0 +1,133 @@
+package boltdb
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+)
+
+var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
+
+// MarshalObject encodes an object to binary format
+func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
+	// Special case for the VERSION bucket. Here we're not using json
+	if v, ok := object.(string); ok {
+		data = []byte(v)
+	} else {
+		data, err = json.Marshal(object)
+		if err != nil {
+			return data, err
+		}
+	}
+	if connection.getEncryptionKey() == nil {
+		return data, nil
+	}
+	return encrypt(data, connection.getEncryptionKey())
+}
+
+// UnmarshalObject decodes an object from binary data
+func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
+	var err error
+	if connection.getEncryptionKey() != nil {
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return errors.Wrap(err, "Failed decrypting object")
+		}
+	}
+	e := json.Unmarshal(data, object)
+	if e != nil {
+		// Special case for the VERSION bucket. Here we're not using json
+		// So we need to return it as a string
+		s, ok := object.(*string)
+		if !ok {
+			return errors.Wrap(err, e.Error())
+		}
+
+		*s = string(data)
+	}
+	return err
+}
+
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// decoding at the moment.
+func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	if connection.getEncryptionKey() != nil {
+		var err error
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return err
+		}
+	}
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	err := jsoni.Unmarshal(data, &object)
+	if err != nil {
+		if s, ok := object.(*string); ok {
+			*s = string(data)
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// mmm, don't have a KMS .... aes GCM seems the most likely from
+// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
+
+func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
+	block, _ := aes.NewCipher(passphrase)
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return encrypted, err
+	}
+	ciphertextByte := gcm.Seal(
+		nonce,
+		nonce,
+		plaintext,
+		nil)
+	return ciphertextByte, nil
+}
+
+func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
+	if string(encrypted) == "false" {
+		return []byte("false"), nil
+	}
+	block, err := aes.NewCipher(passphrase)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating cypher block")
+	}
+
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating GCM")
+	}
+
+	nonceSize := gcm.NonceSize()
+	if len(encrypted) < nonceSize {
+		return encrypted, errEncryptedStringTooShort
+	}
+
+	nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
+	plaintextByte, err = gcm.Open(
+		nil,
+		nonce,
+		ciphertextByteClean,
+		nil)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error decrypting text")
+	}
+
+	return plaintextByte, err
+}

api/database/boltdb/json_test.go

@@ -0,0 +1,177 @@
+package boltdb
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"testing"
+
+	"github.com/gofrs/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","MPSUser":"","MPSPassword":"","MPSToken":"","CertFileContent":"","CertFileName":"","CertFilePassword":"","DomainName":""},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
+	passphrase = "my secret key"
+)
+
+func secretToEncryptionKey(passphrase string) []byte {
+	hash := sha256.Sum256([]byte(passphrase))
+	return hash[:]
+}
+
+func Test_MarshalObjectUnencrypted(t *testing.T) {
+	is := assert.New(t)
+
+	uuid := uuid.Must(uuid.NewV4())
+
+	tests := []struct {
+		object   interface{}
+		expected string
+	}{
+		{
+			object:   nil,
+			expected: `null`,
+		},
+		{
+			object:   true,
+			expected: `true`,
+		},
+		{
+			object:   false,
+			expected: `false`,
+		},
+		{
+			object:   123,
+			expected: `123`,
+		},
+		{
+			object:   "456",
+			expected: "456",
+		},
+		{
+			object:   uuid,
+			expected: "\"" + uuid.String() + "\"",
+		},
+		{
+			object:   uuid.String(),
+			expected: uuid.String(),
+		},
+		{
+			object:   map[string]interface{}{"key": "value"},
+			expected: `{"key":"value"}`,
+		},
+		{
+			object:   []bool{true, false},
+			expected: `[true,false]`,
+		},
+		{
+			object:   []int{1, 2, 3},
+			expected: `[1,2,3]`,
+		},
+		{
+			object:   []string{"1", "2", "3"},
+			expected: `["1","2","3"]`,
+		},
+		{
+			object:   []map[string]interface{}{{"key1": "value1"}, {"key2": "value2"}},
+			expected: `[{"key1":"value1"},{"key2":"value2"}]`,
+		},
+		{
+			object:   []interface{}{1, "2", false, map[string]interface{}{"key1": "value1"}},
+			expected: `[1,"2",false,{"key1":"value1"}]`,
+		},
+	}
+
+	conn := DbConnection{}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
+			data, err := conn.MarshalObject(test.object)
+			is.NoError(err)
+			is.Equal(test.expected, string(data))
+		})
+	}
+}
+
+func Test_UnMarshalObjectUnencrypted(t *testing.T) {
+	is := assert.New(t)
+
+	// Based on actual data entering and what we expect out of the function
+
+	tests := []struct {
+		object   []byte
+		expected string
+	}{
+		{
+			object:   []byte(""),
+			expected: "",
+		},
+		{
+			object:   []byte("35"),
+			expected: "35",
+		},
+		{
+			// An unmarshalled byte string should return the same without error
+			object:   []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
+			expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
+		},
+		{
+			// An un-marshalled json object string should return the same as a string without error also
+			object:   []byte(jsonobject),
+			expected: jsonobject,
+		},
+	}
+
+	conn := DbConnection{}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
+			var object string
+			err := conn.UnmarshalObject(test.object, &object)
+			is.NoError(err)
+			is.Equal(test.expected, object)
+		})
+	}
+}
+
+func Test_ObjectMarshallingEncrypted(t *testing.T) {
+	is := assert.New(t)
+
+	// Based on actual data entering and what we expect out of the function
+
+	tests := []struct {
+		object   []byte
+		expected string
+	}{
+		{
+			object: []byte(""),
+		},
+		{
+			object: []byte("35"),
+		},
+		{
+			// An unmarshalled byte string should return the same without error
+			object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
+		},
+		{
+			// An un-marshalled json object string should return the same as a string without error also
+			object: []byte(jsonobject),
+		},
+	}
+
+	key := secretToEncryptionKey(passphrase)
+	conn := DbConnection{EncryptionKey: key}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
+
+			data, err := conn.MarshalObject(test.object)
+			is.NoError(err)
+
+			var object []byte
+			err = conn.UnmarshalObject(data, &object)
+
+			is.NoError(err)
+			is.Equal(test.object, object)
+		})
+	}
+}

api/database/boltdb/tx.go

@@ -28,10 +28,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	data := make([]byte, len(value))
-	copy(data, value)
-
-	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
+	return tx.conn.UnmarshalObjectWithJsoniter(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -49,7 +46,9 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
 	return bucket.Delete(key)
 }
 
-func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
+	var ids []int
+
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	cursor := bucket.Cursor()
@@ -59,11 +58,14 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 			return err
 		}
 
-		if id, ok := matching(obj); ok {
-			err := bucket.Delete(tx.conn.ConvertToKey(id))
-			if err != nil {
-				return err
-			}
+		if id, ok := matchingFn(obj); ok {
+			ids = append(ids, id)
+		}
+	}
+
+	for _, id := range ids {
+		if err := bucket.Delete(tx.conn.ConvertToKey(id)); err != nil {
+			return err
 		}
 	}
 
@@ -115,45 +117,33 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
 	return bucket.Put(id, data)
 }
 
-func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObject(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
 
 	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
@@ -162,7 +152,7 @@ func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte
 			return err
 		}
 
-		obj, err = append(obj)
+		obj, err = appendFn(obj)
 		if err != nil {
 			return err
 		}

api/database/database.go

@@ -4,13 +4,13 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database/sqlite"
+	"github.com/portainer/portainer/api/database/boltdb"
 )
 
 // NewDatabase should use config options to return a connection to the requested database
 func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
-	if storeType == "sqlite" {
-		return &sqlite.DbConnection{
+	if storeType == "boltdb" {
+		return &boltdb.DbConnection{
 			Path:          storePath,
 			EncryptionKey: encryptionKey,
 		}, nil

api/database/models/version.go

@@ -1,23 +1,8 @@
 package models
 
-import (
-	"net/http"
-)
-
-const (
-	VersionKey       string = "DB_VERSION"
-	InstanceKey      string = "INSTANCE_ID"
-	EditionKey       string = "EDITION"
-	UpdatingKey      string = "DB_UPDATING"
-	MigratorCountKey string = "MIGRATOR_COUNT"
-	SchemaVersionKey string = "SCHEMA_VERSION"
-)
-
 type Version struct {
-	Key   string `json:"Key" gorm:"unique,primaryKey"`
-	Value string `json:"Value"`
-}
-
-func (r *Version) Validate(request *http.Request) error {
-	return nil
+	SchemaVersion string
+	MigratorCount int
+	Edition       int
+	InstanceID    string
 }

api/database/sqlite/db.go

@@ -1,291 +0,0 @@
-package sqlite
-
-import (
-	"errors"
-	"os"
-	"path"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database/models"
-	"github.com/rs/zerolog/log"
-)
-
-const (
-	DatabaseFileName          = "portainer.db"
-	EncryptedDatabaseFileName = "portainer.edb"
-)
-
-var (
-	ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start.  Only one database should exist")
-	ErrHaveEncryptedWithNoKey      = errors.New("The portainer database is encrypted, but no secret was loaded")
-)
-
-type DbConnection struct {
-	Path          string
-	EncryptionKey []byte
-	isEncrypted   bool
-
-	*gorm.DB
-}
-
-func (connection *DbConnection) GetDB() *gorm.DB {
-	if connection.DB == nil {
-		err := connection.Open()
-		if err != nil {
-			panic(err)
-		}
-	}
-	return connection.DB
-}
-
-// GetDatabaseFileName get the database filename
-func (connection *DbConnection) GetDatabaseFileName() string {
-	if connection.IsEncryptedStore() {
-		return EncryptedDatabaseFileName
-	}
-
-	return DatabaseFileName
-}
-
-// GetDataseFilePath get the path + filename for the database file
-func (connection *DbConnection) GetDatabaseFilePath() string {
-	if connection.IsEncryptedStore() {
-		return path.Join(connection.Path, EncryptedDatabaseFileName)
-	}
-
-	return path.Join(connection.Path, DatabaseFileName)
-}
-
-// GetStorePath get the filename and path for the database file
-func (connection *DbConnection) GetStorePath() string {
-	return connection.Path
-}
-
-// Return true if the database is encrypted
-func (connection *DbConnection) IsEncryptedStore() bool {
-	return connection.getEncryptionKey() != nil
-}
-
-func (connection *DbConnection) SetEncrypted(encrypted bool) {
-	connection.isEncrypted = encrypted
-}
-
-// NeedsEncryptionMigration returns true if database encryption is enabled and
-// we have an un-encrypted DB that requires migration to an encrypted DB
-func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
-
-	// Cases:  Note, we need to check both portainer.db and portainer.edb
-	// to determine if it's a new store.   We only need to differentiate between cases 2,3 and 5
-
-	// 1) portainer.edb + key     => False
-	// 2) portainer.edb + no key  => ERROR Fatal!
-	// 3) portainer.db  + key     => True  (needs migration)
-	// 4) portainer.db  + no key  => False
-	// 5) NoDB (new)    + key     => False
-	// 6) NoDB (new)    + no key  => False
-	// 7) portainer.db & portainer.edb => ERROR Fatal!
-
-	// If we have a loaded encryption key, always set encrypted
-	if connection.EncryptionKey != nil {
-		connection.SetEncrypted(true)
-	}
-
-	// Check for portainer.db
-	dbFile := path.Join(connection.Path, DatabaseFileName)
-	_, err := os.Stat(dbFile)
-	haveDbFile := err == nil
-
-	// Check for portainer.edb
-	edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
-	_, err = os.Stat(edbFile)
-	haveEdbFile := err == nil
-
-	if haveDbFile && haveEdbFile {
-		// 7 - encrypted and unencrypted db?
-		return false, ErrHaveEncryptedAndUnencrypted
-	}
-
-	if haveDbFile && connection.EncryptionKey != nil {
-		// 3 - needs migration
-		return true, nil
-	}
-
-	if haveEdbFile && connection.EncryptionKey == nil {
-		// 2 - encrypted db, but no key?
-		return false, ErrHaveEncryptedWithNoKey
-	}
-
-	// 1, 4, 5, 6
-	return false, nil
-}
-
-// Open opens and initializes the BoltDB database.
-func (connection *DbConnection) Open() error {
-
-	log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
-
-	// Now we open the db
-	databasePath := connection.GetDatabaseFilePath()
-
-	db, err := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
-	if err != nil {
-		return err
-	}
-	sqlDB, err := db.DB()
-	if err != nil {
-		return err
-	}
-	sqlDB.SetMaxOpenConns(5)
-	sqlDB.SetMaxOpenConns(10)
-	connection.DB = db
-
-	return nil
-}
-
-func (connection *DbConnection) Close() error {
-	sqlDB, err := connection.DB.DB()
-	if err != nil {
-		return err
-	}
-	connection.DB = nil
-	return sqlDB.Close()
-}
-
-func (connection *DbConnection) getEncryptionKey() []byte {
-	if !connection.isEncrypted {
-		return nil
-	}
-
-	return connection.EncryptionKey
-}
-
-func (connection *DbConnection) GetByID(ID int, obj interface{}) error {
-	tx := connection.DB.First(obj, `id = ?`, ID)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-func (connection *DbConnection) DeleteByID(ID int, obj interface{}) error {
-	tx := connection.DB.Model(obj).Delete(`id = ?`, ID)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-func (connection *DbConnection) Init() error {
-	err := connection.DB.AutoMigrate(&models.Version{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Version")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Settings{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Settings")
-	}
-	err = connection.DB.AutoMigrate(&portainer.APIKey{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate APIKey")
-	}
-	err = connection.DB.AutoMigrate(&portainer.User{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate User")
-	}
-	err = connection.DB.AutoMigrate(&portainer.CustomTemplate{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate CustomTemplate")
-	}
-	err = connection.DB.AutoMigrate(&portainer.DockerHub{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate DockerHub")
-	}
-	err = connection.DB.AutoMigrate(&portainer.EdgeGroup{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate EdgeGroup")
-	}
-	err = connection.DB.AutoMigrate(&portainer.EdgeJob{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate EdgeJob")
-	}
-	err = connection.DB.AutoMigrate(&portainer.EdgeStack{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate EdgeStack")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Endpoint{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Endpoint")
-	}
-	err = connection.DB.AutoMigrate(&portainer.EndpointGroup{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate EndpointGroup")
-	}
-	err = connection.DB.AutoMigrate(&portainer.EndpointRelation{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate EndpointRelation")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Extension{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Extension")
-	}
-	err = connection.DB.AutoMigrate(&portainer.FDOProfile{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate FDOProfile")
-	}
-	err = connection.DB.AutoMigrate(&portainer.HelmUserRepository{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate HelmUserRepository")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Registry{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Registry")
-	}
-	err = connection.DB.AutoMigrate(&portainer.ResourceControl{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate ResourceControl")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Role{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Role")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Schedule{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Schedule")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Snapshot{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Snapshot")
-	}
-	err = connection.DB.AutoMigrate(&portainer.SSLSettings{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate SSLSettings")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Stack{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Stack")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Tag{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Tag")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Team{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Team")
-	}
-	err = connection.DB.AutoMigrate(&portainer.TeamMembership{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate TeamMembership")
-	}
-	err = connection.DB.AutoMigrate(&portainer.TunnelServerInfo{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate TunnelServerInfo")
-	}
-	err = connection.DB.AutoMigrate(&portainer.Webhook{})
-	if err != nil {
-		log.Err(err).Msgf("failed to auto migrate Webhook")
-	}
-	return nil
-}

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -1,28 +1,37 @@
 package apikeyrepository
 
 import (
+	"bytes"
+	"errors"
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "api_key"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "api_key"
 
 // Service represents a service for managing api-key data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
@@ -30,91 +39,67 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error) {
 	var result = make([]portainer.APIKey, 0)
 
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.APIKey{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		record, ok := obj.(*portainer.APIKey)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-	// 			return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
-	// 		}
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.APIKey{},
+		func(obj interface{}) (interface{}, error) {
+			record, ok := obj.(*portainer.APIKey)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
+			}
 
-	// 		if record.UserID == userID {
-	// 			result = append(result, *record)
-	// 		}
+			if record.UserID == userID {
+				result = append(result, *record)
+			}
 
-	// 		return &portainer.APIKey{}, nil
-	// 	})
+			return &portainer.APIKey{}, nil
+		})
 
-	return result, nil
+	return result, err
 }
 
 // GetAPIKeyByDigest returns the API key for the associated digest.
 // Note: there is a 1-to-1 mapping of api-key and digest
 func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error) {
-	// var k *portainer.APIKey
-	// stop := fmt.Errorf("ok")
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.APIKey{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		key, ok := obj.(*portainer.APIKey)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-	// 			return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
-	// 		}
-	// 		if bytes.Equal(key.Digest, digest) {
-	// 			k = key
-	// 			return nil, stop
-	// 		}
+	var k *portainer.APIKey
+	stop := fmt.Errorf("ok")
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.APIKey{},
+		func(obj interface{}) (interface{}, error) {
+			key, ok := obj.(*portainer.APIKey)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
+			}
+			if bytes.Equal(key.Digest, digest) {
+				k = key
+				return nil, stop
+			}
 
-	// 		return &portainer.APIKey{}, nil
-	// 	})
+			return &portainer.APIKey{}, nil
+		})
 
-	// if errors.Is(err, stop) {
-	// 	return k, nil
-	// }
+	if errors.Is(err, stop) {
+		return k, nil
+	}
 
-	// if err == nil {
-	// 	return nil, dserrors.ErrObjectNotFound
-	// }
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
 
-	return nil, nil
+	return nil, err
 }
 
-// CreateAPIKey creates a new APIKey object.
-func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		record.ID = portainer.APIKeyID(id)
+// Create creates a new APIKey object.
+func (service *Service) Create(record *portainer.APIKey) error {
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			record.ID = portainer.APIKeyID(id)
 
-	// 		return int(record.ID), record
-	// 	},
-	// )
-	return nil
-}
-
-// GetAPIKey retrieves an existing APIKey object by api key ID.
-func (service *Service) GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	var key portainer.APIKey
-	// identifier := service.connection.ConvertToKey(int(keyID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &key)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &key, nil
-}
-
-func (service *Service) UpdateAPIKey(key *portainer.APIKey) error {
-	// identifier := service.connection.ConvertToKey(int(key.ID))
-	// return service.connection.UpdateObject(BucketName, identifier, key)
-	return nil
-}
-
-func (service *Service) DeleteAPIKey(ID portainer.APIKeyID) error {
-	return nil
+			return int(record.ID), record
+		},
+	)
 }

api/dataservices/base.go

@@ -0,0 +1,66 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseCRUD[T any, I constraints.Integer] interface {
+	Create(element *T) error
+	Read(ID I) (*T, error)
+	ReadAll() ([]T, error)
+	Update(ID I, element *T) error
+	Delete(ID I) error
+}
+
+type BaseDataService[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+}
+
+func (s *BaseDataService[T, I]) BucketName() string {
+	return s.Bucket
+}
+
+func (service *BaseDataService[T, I]) Tx(tx portainer.Transaction) BaseDataServiceTx[T, I] {
+	return BaseDataServiceTx[T, I]{
+		Bucket:     service.Bucket,
+		Connection: service.Connection,
+		Tx:         tx,
+	}
+}
+
+func (service BaseDataService[T, I]) Read(ID I) (*T, error) {
+	var element *T
+
+	return element, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		element, err = service.Tx(tx).Read(ID)
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		collection, err = service.Tx(tx).ReadAll()
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) Update(ID I, element *T) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Update(ID, element)
+	})
+}
+
+func (service BaseDataService[T, I]) Delete(ID I) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Delete(ID)
+	})
+}

api/dataservices/base_tx.go

@@ -0,0 +1,49 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseDataServiceTx[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+	Tx         portainer.Transaction
+}
+
+func (service BaseDataServiceTx[T, I]) BucketName() string {
+	return service.Bucket
+}
+
+func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
+	var element T
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	err := service.Tx.GetObject(service.Bucket, identifier, &element)
+	if err != nil {
+		return nil, err
+	}
+
+	return &element, nil
+}
+
+func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Tx.GetAllWithJsoniter(
+		service.Bucket,
+		new(T),
+		AppendFn(&collection),
+	)
+}
+
+func (service BaseDataServiceTx[T, I]) Update(ID I, element *T) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.UpdateObject(service.Bucket, identifier, element)
+}
+
+func (service BaseDataServiceTx[T, I]) Delete(ID I) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.DeleteObject(service.Bucket, identifier)
+}

api/dataservices/customtemplate/customtemplate.go

@@ -2,81 +2,39 @@ package customtemplate
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "customtemplates"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "customtemplates"
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// CustomTemplates return an array containing all the custom templates.
-func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
-	var customTemplates = make([]portainer.CustomTemplate, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.CustomTemplate{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		//var tag portainer.Tag
-	// 		customTemplate, ok := obj.(*portainer.CustomTemplate)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
-	// 			return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
-	// 		}
-	// 		customTemplates = append(customTemplates, *customTemplate)
-
-	// 		return &portainer.CustomTemplate{}, nil
-	// 	})
-
-	return customTemplates, nil
-}
-
-// CustomTemplate returns an custom template by ID.
-func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
-	var customTemplate portainer.CustomTemplate
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &customTemplate)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &customTemplate, nil
-}
-
-// UpdateCustomTemplate updates an custom template.
-func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
-	return nil
-}
-
-// DeleteCustomTemplate deletes an custom template.
-func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
-	return nil
-}
-
 // CreateCustomTemplate uses the existing id and saves it.
 // TODO: where does the ID come from, and is it safe?
 func (service *Service) Create(customTemplate *portainer.CustomTemplate) error {
-	return nil
+	return service.Connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/dockerhub/dockerhub.go

@@ -21,6 +21,11 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
 		connection: connection,
 	}, nil
@@ -30,10 +35,15 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
+	err := service.connection.GetObject(BucketName, []byte(dockerHubKey), &dockerhub)
+	if err != nil {
+		return nil, err
+	}
+
 	return &dockerhub, nil
 }
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return nil
+	return service.connection.UpdateObject(BucketName, []byte(dockerHubKey), dockerhub)
 }

api/dataservices/edgegroup/edgegroup.go

@@ -2,6 +2,7 @@ package edgegroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,7 +10,7 @@ const BucketName = "edgegroups"
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 func (service *Service) BucketName() string {
@@ -18,60 +19,42 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeGroups return a slice containing all the Edge groups.
-func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups []portainer.EdgeGroup
-	var err error
-
-	// err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-	// 	groups, err = service.Tx(tx).EdgeGroups()
-	// 	return err
-	// })
-
-	return groups, err
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group *portainer.EdgeGroup
-	var err error
-
-	// err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-	// 	group, err = service.Tx(tx).EdgeGroup(ID)
-	// 	return err
-	// })
-
-	return group, err
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	return nil
-}
-
 // Deprecated: UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
-	return nil
-}
+	id := service.Connection.ConvertToKey(int(ID))
+	edgeGroup := &portainer.EdgeGroup{}
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	return nil
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
+		updateFunc(edgeGroup)
+	})
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) Create(group *portainer.EdgeGroup) error {
-	return nil
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Create(group)
+	})
 }

api/dataservices/edgegroup/tx.go

@@ -4,48 +4,11 @@ import (
 	"errors"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeGroups return a slice containing all the Edge groups.
-func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups = make([]portainer.EdgeGroup, 0)
-
-	// err := service.tx.GetAllWithJsoniter(
-	// 	BucketName,
-	// 	&portainer.EdgeGroup{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		group, ok := obj.(*portainer.EdgeGroup)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
-	// 			return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
-	// 		}
-	// 		groups = append(groups, *group)
-
-	// 		return &portainer.EdgeGroup{}, nil
-	// 	})
-
-	return groups, nil
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service ServiceTx) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group portainer.EdgeGroup
-
-	return &group, nil
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service ServiceTx) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	return nil
+	dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 // UpdateEdgeGroupFunc is a no-op inside a transaction.
@@ -53,11 +16,12 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service ServiceTx) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	return nil
-}
-
 func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
-	return nil
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			group.ID = portainer.EdgeGroupID(id)
+			return int(group.ID), group
+		},
+	)
 }

api/dataservices/edgejob/edgejob.go

@@ -2,6 +2,7 @@ package edgejob
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,56 +10,61 @@ const BucketName = "edgejobs"
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeJobs returns a list of Edge jobs
-func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-	return edgeJobs, nil
-}
-
-// EdgeJob returns an Edge job by ID
-func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-
-	return &edgeJob, nil
-}
-
 // Create creates a new EdgeJob
-func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	return nil
+func (service *Service) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
 }
 
-// Deprecated: use UpdateEdgeJobFunc instead
-func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	return nil
+// CreateWithID creates a new EdgeJob
+func (service *Service) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.Connection.CreateObjectWithId(
+		BucketName,
+		int(edgeJob.ID),
+		edgeJob,
+	)
 }
 
 // UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
-	return nil
+	id := service.Connection.ConvertToKey(int(ID))
+	edgeJob := &portainer.EdgeJob{}
+
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+		updateFunc(edgeJob)
+	})
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	return nil
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgejob/tx.go

@@ -4,41 +4,23 @@ import (
 	"errors"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeJobs returns a list of Edge jobs
-func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-
-	return edgeJobs, nil
-}
-
-// EdgeJob returns an Edge job by ID
-func (service ServiceTx) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-
-	return &edgeJob, nil
+	dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // Create creates a new EdgeJob
-func (service ServiceTx) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	edgeJob.ID = ID
-
-	return service.tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+func (service ServiceTx) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
 }
 
-// UpdateEdgeJob updates an edge job
-func (service ServiceTx) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	return nil
+// CreateWithID creates a new EdgeJob
+func (service ServiceTx) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.Tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
 }
 
 // UpdateEdgeJobFunc is a no-op inside a transaction.
@@ -46,7 +28,7 @@ func (service ServiceTx) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc fu
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service ServiceTx) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	return nil
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
 }

api/dataservices/edgestack/edgestack.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -23,6 +24,11 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection, cacheInvalidationFn func(portainer.EdgeStackID)) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	s := &Service{
 		connection:          connection,
 		idxVersion:          make(map[portainer.EdgeStackID]int),
@@ -55,12 +61,24 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 // EdgeStacks returns an array containing all edge stacks
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
-	return stacks, nil
+
+	return stacks, service.connection.GetAll(
+		BucketName,
+		&portainer.EdgeStack{},
+		dataservices.AppendFn(&stacks),
+	)
 }
 
 // EdgeStack returns an Edge stack by ID.
 func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error) {
 	var stack portainer.EdgeStack
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
+
 	return &stack, nil
 }
 
@@ -77,6 +95,15 @@ func (service *Service) EdgeStackVersion(ID portainer.EdgeStackID) (int, bool) {
 func (service *Service) Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	edgeStack.ID = id
 
+	err := service.connection.CreateObjectWithId(
+		BucketName,
+		int(edgeStack.ID),
+		edgeStack,
+	)
+	if err != nil {
+		return err
+	}
+
 	service.mu.Lock()
 	service.idxVersion[id] = edgeStack.Version
 	service.cacheInvalidationFn(id)
@@ -90,6 +117,13 @@ func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *por
 	service.mu.Lock()
 	defer service.mu.Unlock()
 
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.UpdateObject(BucketName, identifier, edgeStack)
+	if err != nil {
+		return err
+	}
+
 	service.idxVersion[ID] = edgeStack.Version
 	service.cacheInvalidationFn(ID)
 
@@ -98,7 +132,18 @@ func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *por
 
 // UpdateEdgeStackFunc updates an Edge stack inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
-	return nil
+	id := service.connection.ConvertToKey(int(ID))
+	edgeStack := &portainer.EdgeStack{}
+
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeStack, func() {
+		updateFunc(edgeStack)
+
+		service.idxVersion[ID] = edgeStack.Version
+		service.cacheInvalidationFn(ID)
+	})
 }
 
 // UpdateEdgeStackFuncTx is a helper function used to call UpdateEdgeStackFunc inside a transaction.
@@ -111,7 +156,21 @@ func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	service.mu.Lock()
 	defer service.mu.Unlock()
 
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.DeleteObject(BucketName, identifier)
+	if err != nil {
+		return err
+	}
+
+	delete(service.idxVersion, ID)
+
 	service.cacheInvalidationFn(ID)
 
 	return nil
 }
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	return service.connection.GetNextIdentifier(BucketName)
+}

api/dataservices/edgestack/tx.go

@@ -1,7 +1,11 @@
 package edgestack
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
@@ -17,12 +21,33 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	return stacks, nil
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EdgeStack{},
+		func(obj interface{}) (interface{}, error) {
+			stack, ok := obj.(*portainer.EdgeStack)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
+				return nil, fmt.Errorf("failed to convert to EdgeStack object: %s", obj)
+			}
+
+			stacks = append(stacks, *stack)
+
+			return &portainer.EdgeStack{}, nil
+		})
+
+	return stacks, err
 }
 
 // EdgeStack returns an Edge stack by ID.
 func (service ServiceTx) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error) {
 	var stack portainer.EdgeStack
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
 
 	return &stack, nil
 }
@@ -40,6 +65,15 @@ func (service ServiceTx) EdgeStackVersion(ID portainer.EdgeStackID) (int, bool)
 func (service ServiceTx) Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	edgeStack.ID = id
 
+	err := service.tx.CreateObjectWithId(
+		BucketName,
+		int(edgeStack.ID),
+		edgeStack,
+	)
+	if err != nil {
+		return err
+	}
+
 	service.service.mu.Lock()
 	service.service.idxVersion[id] = edgeStack.Version
 	service.service.cacheInvalidationFn(id)
@@ -53,6 +87,13 @@ func (service ServiceTx) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *po
 	service.service.mu.Lock()
 	defer service.service.mu.Unlock()
 
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.UpdateObject(BucketName, identifier, edgeStack)
+	if err != nil {
+		return err
+	}
+
 	service.service.idxVersion[ID] = edgeStack.Version
 	service.service.cacheInvalidationFn(ID)
 
@@ -75,5 +116,22 @@ func (service ServiceTx) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFun
 func (service ServiceTx) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	service.service.mu.Lock()
 	defer service.service.mu.Unlock()
+
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.DeleteObject(BucketName, identifier)
+	if err != nil {
+		return err
+	}
+
+	delete(service.service.idxVersion, ID)
+
+	service.service.cacheInvalidationFn(ID)
+
 	return nil
 }
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.tx.GetNextIdentifier(BucketName)
+}

api/dataservices/endpoint/endpoint.go

@@ -7,6 +7,9 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "endpoints"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
 	connection portainer.Connection
@@ -15,31 +18,36 @@ type Service struct {
 	heartbeats sync.Map
 }
 
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	s := &Service{
 		connection: connection,
 		idxEdgeID:  make(map[string]portainer.EndpointID),
 	}
 
-	return s, nil
-}
-
-func (service *Service) Init() error {
-	es, err := service.endpoints()
+	es, err := s.endpoints()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	for _, e := range es {
 		if len(e.EdgeID) > 0 {
-			service.idxEdgeID[e.EdgeID] = e.ID
+			s.idxEdgeID[e.EdgeID] = e.ID
 		}
 
-		service.heartbeats.Store(e.ID, e.LastCheckInDate)
+		s.heartbeats.Store(e.ID, e.LastCheckInDate)
 	}
 
-	return nil
+	return s, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
@@ -51,48 +59,44 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 
 // Endpoint returns an environment(endpoint) by ID.
 func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
-	var obj portainer.Endpoint
+	var endpoint *portainer.Endpoint
+	var err error
 
-	err := service.connection.GetByID(int(ID), &obj)
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		endpoint, err = service.Tx(tx).Endpoint(ID)
+		return err
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	return &obj, nil
+	endpoint.LastCheckInDate, _ = service.Heartbeat(ID)
+
+	return endpoint, nil
 }
 
 // UpdateEndpoint updates an environment(endpoint).
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
-	db := service.connection.GetDB()
-	endpoint.ID = ID
-	tx := db.Save(&endpoint)
-	if tx.Error != nil {
-		return tx.Error
-	}
-
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).UpdateEndpoint(ID, endpoint)
+	})
 }
 
 // DeleteEndpoint deletes an environment(endpoint).
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&portainer.Endpoint{}).Delete("id = ?", ID)
-	if tx.Error != nil {
-		return tx.Error
-	}
-
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).DeleteEndpoint(ID)
+	})
 }
 
 func (service *Service) endpoints() ([]portainer.Endpoint, error) {
 	var endpoints []portainer.Endpoint
 	var err error
 
-	db := service.connection.GetDB()
-	tx := db.Find(&endpoints)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		endpoints, err = service.Tx(tx).Endpoints()
+		return err
+	})
 
 	return endpoints, err
 }
@@ -135,10 +139,20 @@ func (service *Service) UpdateHeartbeat(endpointID portainer.EndpointID) {
 
 // CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
 func (service *Service) Create(endpoint *portainer.Endpoint) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&portainer.Endpoint{}).Create(&endpoint)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Create(endpoint)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	var identifier int
+
+	service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		identifier = service.Tx(tx).GetNextIdentifier()
+
+		return nil
+	})
+
+	return identifier
 }

api/dataservices/endpoint/tx.go

@@ -2,6 +2,7 @@ package endpoint
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -12,27 +13,33 @@ type ServiceTx struct {
 	tx      portainer.Transaction
 }
 
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
 // Endpoint returns an environment(endpoint) by ID.
 func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
 	var endpoint portainer.Endpoint
-	// identifier := service.service.connection.ConvertToKey(int(ID))
+	identifier := service.service.connection.ConvertToKey(int(ID))
 
-	// err := service.tx.GetObject(BucketName, identifier, &endpoint)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := service.tx.GetObject(BucketName, identifier, &endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	endpoint.LastCheckInDate, _ = service.service.Heartbeat(ID)
 
 	return &endpoint, nil
 }
 
 // UpdateEndpoint updates an environment(endpoint).
 func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
+	identifier := service.service.connection.ConvertToKey(int(ID))
 
-	// err := service.tx.UpdateObject(BucketName, identifier, endpoint)
-	// if err != nil {
-	// 	return err
-	// }
+	err := service.tx.UpdateObject(BucketName, identifier, endpoint)
+	if err != nil {
+		return err
+	}
 
 	service.service.mu.Lock()
 	if len(endpoint.EdgeID) > 0 {
@@ -48,17 +55,18 @@ func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *porta
 
 // DeleteEndpoint deletes an environment(endpoint).
 func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
+	identifier := service.service.connection.ConvertToKey(int(ID))
 
-	// err := service.tx.DeleteObject(BucketName, identifier)
-	// if err != nil {
-	// 	return err
-	// }
+	err := service.tx.DeleteObject(BucketName, identifier)
+	if err != nil {
+		return err
+	}
 
 	service.service.mu.Lock()
 	for edgeID, endpointID := range service.service.idxEdgeID {
 		if endpointID == ID {
 			delete(service.service.idxEdgeID, edgeID)
+
 			break
 		}
 	}
@@ -74,22 +82,11 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	// err := service.tx.GetAllWithJsoniter(
-	// 	BucketName,
-	// 	&portainer.Endpoint{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		endpoint, ok := obj.(*portainer.Endpoint)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
-	// 			return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
-	// 		}
-
-	// 		endpoints = append(endpoints, *endpoint)
-
-	// 		return &portainer.Endpoint{}, nil
-	// 	})
-
-	return endpoints, nil
+	return endpoints, service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.AppendFn(&endpoints),
+	)
 }
 
 func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {
@@ -110,17 +107,22 @@ func (service ServiceTx) UpdateHeartbeat(endpointID portainer.EndpointID) {
 
 // CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
 func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
-	// err := service.tx.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
-	// if err != nil {
-	// 	return err
-	// }
+	err := service.tx.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
+	if err != nil {
+		return err
+	}
 
-	// service.service.mu.Lock()
-	// if len(endpoint.EdgeID) > 0 {
-	// 	service.service.idxEdgeID[endpoint.EdgeID] = endpoint.ID
-	// }
-	// service.service.heartbeats.Store(endpoint.ID, endpoint.LastCheckInDate)
-	// service.service.mu.Unlock()
+	service.service.mu.Lock()
+	if len(endpoint.EdgeID) > 0 {
+		service.service.idxEdgeID[endpoint.EdgeID] = endpoint.ID
+	}
+	service.service.heartbeats.Store(endpoint.ID, endpoint.LastCheckInDate)
+	service.service.mu.Unlock()
 
 	return nil
 }
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.tx.GetNextIdentifier(BucketName)
+}

api/dataservices/endpointgroup/endpointgroup.go

@@ -2,62 +2,51 @@ package endpointgroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_groups"
 )
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &endpointGroup)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, endpointGroup)
-	return nil
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	return endpointGroups, nil
-}
-
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service *Service) Create(endpointGroup *portainer.EndpointGroup) error {
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			endpointGroup.ID = portainer.EndpointGroupID(id)
+			return int(endpointGroup.ID), endpointGroup
+		},
+	)
 }

api/dataservices/endpointgroup/tx.go

@@ -2,48 +2,20 @@ package endpointgroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service ServiceTx) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &endpointGroup)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service ServiceTx) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, endpointGroup)
-	return nil
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	return endpointGroups, nil
+	dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
-	return nil
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			endpointGroup.ID = portainer.EndpointGroupID(id)
+			return int(endpointGroup.ID), endpointGroup
+		},
+	)
 }

api/dataservices/endpointrelation/endpointrelation.go

@@ -2,11 +2,15 @@ package endpointrelation
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "endpoint_relations"
+
 // Service represents a service for managing environment(endpoint) relation data.
 type Service struct {
 	connection      portainer.Connection
@@ -14,6 +18,10 @@ type Service struct {
 	updateStackFnTx func(tx portainer.Transaction, ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 }
 
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
 func (service *Service) RegisterUpdateStackFunction(
 	updateFunc func(portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
 	updateFuncTx func(portainer.Transaction, portainer.EdgeStackID, func(*portainer.EdgeStack)) error,
@@ -24,6 +32,10 @@ func (service *Service) RegisterUpdateStackFunction(
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
 		connection: connection,
@@ -41,32 +53,44 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	return all, nil
+	return all, service.connection.GetAll(
+		BucketName,
+		&portainer.EndpointRelation{},
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID
 func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
 	var endpointRelation portainer.EndpointRelation
+	identifier := service.connection.ConvertToKey(int(endpointID))
+
+	err := service.connection.GetObject(BucketName, identifier, &endpointRelation)
+	if err != nil {
+		return nil, err
+	}
 
 	return &endpointRelation, nil
 }
 
 // CreateEndpointRelation saves endpointRelation
 func (service *Service) Create(endpointRelation *portainer.EndpointRelation) error {
+	err := service.connection.CreateObjectWithId(BucketName, int(endpointRelation.EndpointID), endpointRelation)
+	cache.Del(endpointRelation.EndpointID)
 
-	return nil
+	return err
 }
 
 // UpdateEndpointRelation updates an Environment(Endpoint) relation object
 func (service *Service) UpdateEndpointRelation(endpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	previousRelationState, _ := service.EndpointRelation(endpointID)
 
-	// identifier := service.connection.ConvertToKey(int(endpointID))
-	// err := service.connection.UpdateObject(BucketName, identifier, endpointRelation)
-	// cache.Del(endpointID)
-	// if err != nil {
-	// 	return err
-	// }
+	identifier := service.connection.ConvertToKey(int(endpointID))
+	err := service.connection.UpdateObject(BucketName, identifier, endpointRelation)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
 	updatedRelationState, _ := service.EndpointRelation(endpointID)
 
@@ -79,12 +103,12 @@ func (service *Service) UpdateEndpointRelation(endpointID portainer.EndpointID,
 func (service *Service) DeleteEndpointRelation(endpointID portainer.EndpointID) error {
 	deletedRelation, _ := service.EndpointRelation(endpointID)
 
-	// identifier := service.connection.ConvertToKey(int(endpointID))
-	// err := service.connection.DeleteObject(BucketName, identifier)
-	// cache.Del(endpointID)
-	// if err != nil {
-	// 	return err
-	// }
+	identifier := service.connection.ConvertToKey(int(endpointID))
+	err := service.connection.DeleteObject(BucketName, identifier)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
 	service.updateEdgeStacksAfterRelationChange(deletedRelation, nil)
 

api/dataservices/endpointrelation/tx.go

@@ -2,6 +2,7 @@ package endpointrelation
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -12,59 +13,52 @@ type ServiceTx struct {
 	tx      portainer.Transaction
 }
 
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
 // EndpointRelations returns an array of all EndpointRelations
 func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	// err := service.tx.GetAll(
-	// 	BucketName,
-	// 	&portainer.EndpointRelation{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		r, ok := obj.(*portainer.EndpointRelation)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-	// 			return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
-	// 		}
-
-	// 		all = append(all, *r)
-
-	// 		return &portainer.EndpointRelation{}, nil
-	// 	})
-
-	return all, nil
+	return all, service.tx.GetAll(
+		BucketName,
+		&portainer.EndpointRelation{},
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns an Environment(Endpoint) relation object by EndpointID
 func (service ServiceTx) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
 	var endpointRelation portainer.EndpointRelation
-	// identifier := service.service.connection.ConvertToKey(int(endpointID))
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
 
-	// err := service.tx.GetObject(BucketName, identifier, &endpointRelation)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := service.tx.GetObject(BucketName, identifier, &endpointRelation)
+	if err != nil {
+		return nil, err
+	}
 
 	return &endpointRelation, nil
 }
 
 // CreateEndpointRelation saves endpointRelation
 func (service ServiceTx) Create(endpointRelation *portainer.EndpointRelation) error {
-	// err := service.tx.CreateObjectWithId(BucketName, int(endpointRelation.EndpointID), endpointRelation)
-	// cache.Del(endpointRelation.EndpointID)
+	err := service.tx.CreateObjectWithId(BucketName, int(endpointRelation.EndpointID), endpointRelation)
+	cache.Del(endpointRelation.EndpointID)
 
-	return nil
+	return err
 }
 
 // UpdateEndpointRelation updates an Environment(Endpoint) relation object
 func (service ServiceTx) UpdateEndpointRelation(endpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	previousRelationState, _ := service.EndpointRelation(endpointID)
 
-	// identifier := service.service.connection.ConvertToKey(int(endpointID))
-	// err := service.tx.UpdateObject(BucketName, identifier, endpointRelation)
-	// cache.Del(endpointID)
-	// if err != nil {
-	// 	return err
-	// }
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	err := service.tx.UpdateObject(BucketName, identifier, endpointRelation)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
 	updatedRelationState, _ := service.EndpointRelation(endpointID)
 
@@ -77,12 +71,12 @@ func (service ServiceTx) UpdateEndpointRelation(endpointID portainer.EndpointID,
 func (service ServiceTx) DeleteEndpointRelation(endpointID portainer.EndpointID) error {
 	deletedRelation, _ := service.EndpointRelation(endpointID)
 
-	// identifier := service.service.connection.ConvertToKey(int(endpointID))
-	// err := service.tx.DeleteObject(BucketName, identifier)
-	// cache.Del(endpointID)
-	// if err != nil {
-	// 	return err
-	// }
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	err := service.tx.DeleteObject(BucketName, identifier)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
 	service.updateEdgeStacksAfterRelationChange(deletedRelation, nil)
 

api/dataservices/errors/errors.go

@@ -5,7 +5,6 @@ import (
 )
 
 var (
-	ErrNoSuchTable        = errors.New("no such table")
 	ErrObjectNotFound     = errors.New("object not found inside the database")
 	ErrWrongDBEdition     = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 	ErrDBImportFailed     = errors.New("importing backup failed")

api/dataservices/extension/extension.go

@@ -2,12 +2,11 @@ package extension
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "extension"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "extension"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -20,10 +19,10 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
 		connection: connection,
@@ -33,12 +32,12 @@ func NewService(connection portainer.Connection) (*Service, error) {
 // Extension returns a extension by ID
 func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extension, error) {
 	var extension portainer.Extension
-	// identifier := service.connection.ConvertToKey(int(ID))
+	identifier := service.connection.ConvertToKey(int(ID))
 
-	// err := service.connection.GetObject(BucketName, identifier, &extension)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := service.connection.GetObject(BucketName, identifier, &extension)
+	if err != nil {
+		return nil, err
+	}
 
 	return &extension, nil
 }
@@ -47,33 +46,21 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Extension{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		extension, ok := obj.(*portainer.Extension)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
-	// 			return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
-	// 		}
+	return extensions, service.connection.GetAll(
+		BucketName,
+		&portainer.Extension{},
+		dataservices.AppendFn(&extensions),
+	)
 
-	// 		extensions = append(extensions, *extension)
-
-	// 		return &portainer.Extension{}, nil
-	// 	})
-
-	return extensions, nil
 }
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	// return service.connection.CreateObjectWithId(BucketName, int(extension.ID), extension)
-	return nil
+	return service.connection.CreateObjectWithId(BucketName, int(extension.ID), extension)
 }
 
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
 }

api/dataservices/fdoprofile/fdoprofile.go

@@ -2,88 +2,42 @@ package fdoprofile
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "fdo_profiles"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "fdo_profiles"
 
 // Service represents a service for managingFDO Profiles data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// FDOProfiles return an array containing all the FDO Profiles.
-func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
-	var fdoProfiles = make([]portainer.FDOProfile, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.FDOProfile{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		fdoProfile, ok := obj.(*portainer.FDOProfile)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
-
-	// 			return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
-	// 		}
-	// 		fdoProfiles = append(fdoProfiles, *fdoProfile)
-	// 		return &portainer.FDOProfile{}, nil
-	// 	})
-
-	return fdoProfiles, nil
-}
-
-// FDOProfile returns an FDO Profile by ID.
-func (service *Service) FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error) {
-	var FDOProfile portainer.FDOProfile
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &FDOProfile)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &FDOProfile, nil
-}
-
 // Create assign an ID to a new FDO Profile and saves it.
 func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
-	// return service.connection.CreateObjectWithId(
-	// 	BucketName,
-	// 	int(FDOProfile.ID),
-	// 	FDOProfile,
-	// )
-	return nil
+	return service.Connection.CreateObjectWithId(
+		BucketName,
+		int(FDOProfile.ID),
+		FDOProfile,
+	)
 }
 
-// Update updates an FDO Profile.
-func (service *Service) Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, FDOProfile)
-	return nil
-}
-
-// Delete deletes an FDO Profile.
-func (service *Service) Delete(ID portainer.FDOProfileID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+// GetNextIdentifier returns the next identifier for a FDO Profile.
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -2,102 +2,52 @@ package helmuserrepository
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "helm_user_repository"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "helm_user_repository"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// HelmUserRepository returns an array of all HelmUserRepository
-func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
-	var repos = make([]portainer.HelmUserRepository, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.HelmUserRepository{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		r, ok := obj.(*portainer.HelmUserRepository)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-	// 			return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-	// 		}
-
-	// 		repos = append(repos, *r)
-
-	// 		return &portainer.HelmUserRepository{}, nil
-	// 	})
-
-	return repos, nil
-}
-
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
 func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.HelmUserRepository{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		record, ok := obj.(*portainer.HelmUserRepository)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-	// 			return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-	// 		}
-
-	// 		if record.UserID == userID {
-	// 			result = append(result, *record)
-	// 		}
-
-	// 		return &portainer.HelmUserRepository{}, nil
-	// 	})
-
-	return result, nil
+	return result, service.Connection.GetAll(
+		BucketName,
+		&portainer.HelmUserRepository{},
+		dataservices.FilterFn(&result, func(e portainer.HelmUserRepository) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // CreateHelmUserRepository creates a new HelmUserRepository object.
 func (service *Service) Create(record *portainer.HelmUserRepository) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		record.ID = portainer.HelmUserRepositoryID(id)
-	// 		return int(record.ID), record
-	// 	},
-	// )
-	return nil
-}
-
-// UpdateHelmUserRepostory updates an registry.
-func (service *Service) UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, registry *portainer.HelmUserRepository) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, registry)
-	return nil
-}
-
-// DeleteHelmUserRepository deletes an registry.
-func (service *Service) DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			record.ID = portainer.HelmUserRepositoryID(id)
+			return int(record.ID), record
+		},
+	)
 }

api/dataservices/helpers.go

@@ -0,0 +1,68 @@
+package dataservices
+
+import (
+	"errors"
+	"fmt"
+
+	perrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+// ErrStop signals the stop of computation when filtering results
+var ErrStop = errors.New("stop")
+
+func IsErrObjectNotFound(e error) bool {
+	return errors.Is(e, perrors.ErrObjectNotFound)
+}
+
+// AppendFn appends elements to the given collection slice
+func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		*collection = append(*collection, *element)
+
+		return new(T), nil
+	}
+}
+
+// FilterFn appends elements to the given collection when the predicate is true
+func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*element) {
+			*collection = append(*collection, *element)
+		}
+
+		return new(T), nil
+	}
+}
+
+// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
+// success
+func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		e, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*e) {
+			*element = *e
+			return new(T), ErrStop
+		}
+
+		return new(T), nil
+	}
+}

api/dataservices/interface.go

@@ -1,13 +1,11 @@
 package dataservices
 
 import (
-	"errors"
+	"io"
 	"time"
 
-	"github.com/mattn/go-sqlite3"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
-	"gorm.io/gorm"
 )
 
 type (
@@ -37,15 +35,19 @@ type (
 		User() UserService
 		Version() VersionService
 		Webhook() WebhookService
+		PendingActions() PendingActionsService
 	}
 
-	// DataStore defines the interface to manage the data
 	DataStore interface {
 		Open() (newStore bool, err error)
 		Init() error
 		Close() error
+		UpdateTx(func(DataStoreTx) error) error
+		ViewTx(func(DataStoreTx) error) error
+		MigrateData() error
 		Rollback(force bool) error
 		CheckCurrentEdition() error
+		BackupTo(w io.Writer) error
 		Export(filename string) (err error)
 
 		DataStoreTx
@@ -53,31 +55,27 @@ type (
 
 	// CustomTemplateService represents a service to manage custom templates
 	CustomTemplateService interface {
-		CustomTemplates() ([]portainer.CustomTemplate, error)
-		CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error)
-		Create(customTemplate *portainer.CustomTemplate) error
-		UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error
-		DeleteCustomTemplate(ID portainer.CustomTemplateID) error
+		BaseCRUD[portainer.CustomTemplate, portainer.CustomTemplateID]
+		GetNextIdentifier() int
 	}
 
 	// EdgeGroupService represents a service to manage Edge groups
 	EdgeGroupService interface {
-		EdgeGroups() ([]portainer.EdgeGroup, error)
-		EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error)
-		Create(group *portainer.EdgeGroup) error
-		UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error
+		BaseCRUD[portainer.EdgeGroup, portainer.EdgeGroupID]
 		UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(group *portainer.EdgeGroup)) error
-		DeleteEdgeGroup(ID portainer.EdgeGroupID) error
 	}
 
 	// EdgeJobService represents a service to manage Edge jobs
 	EdgeJobService interface {
-		EdgeJobs() ([]portainer.EdgeJob, error)
-		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
-		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
-		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		BaseCRUD[portainer.EdgeJob, portainer.EdgeJobID]
+		CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
-		DeleteEdgeJob(ID portainer.EdgeJobID) error
+		GetNextIdentifier() int
+	}
+
+	PendingActionsService interface {
+		BaseCRUD[portainer.PendingActions, portainer.PendingActionsID]
+		GetNextIdentifier() int
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks
@@ -86,10 +84,11 @@ type (
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
 		EdgeStackVersion(ID portainer.EdgeStackID) (int, bool)
 		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
-		// Deprecated: Use UpdateEdgeStackFunc instead.
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
+		GetNextIdentifier() int
+		BucketName() string
 	}
 
 	// EndpointService represents a service for managing environment(endpoint) data
@@ -102,15 +101,13 @@ type (
 		Create(endpoint *portainer.Endpoint) error
 		UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error
 		DeleteEndpoint(ID portainer.EndpointID) error
+		GetNextIdentifier() int
+		BucketName() string
 	}
 
 	// EndpointGroupService represents a service for managing environment(endpoint) group data
 	EndpointGroupService interface {
-		EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error)
-		EndpointGroups() ([]portainer.EndpointGroup, error)
-		Create(group *portainer.EndpointGroup) error
-		UpdateEndpointGroup(ID portainer.EndpointGroupID, group *portainer.EndpointGroup) error
-		DeleteEndpointGroup(ID portainer.EndpointGroupID) error
+		BaseCRUD[portainer.EndpointGroup, portainer.EndpointGroupID]
 	}
 
 	// EndpointRelationService represents a service for managing environment(endpoint) relations data
@@ -120,24 +117,19 @@ type (
 		Create(endpointRelation *portainer.EndpointRelation) error
 		UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error
 		DeleteEndpointRelation(EndpointID portainer.EndpointID) error
+		BucketName() string
 	}
 
 	// FDOProfileService represents a service to manage FDO Profiles
 	FDOProfileService interface {
-		FDOProfiles() ([]portainer.FDOProfile, error)
-		FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error)
-		Create(FDOProfile *portainer.FDOProfile) error
-		Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error
-		Delete(ID portainer.FDOProfileID) error
+		BaseCRUD[portainer.FDOProfile, portainer.FDOProfileID]
+		GetNextIdentifier() int
 	}
 
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
+		BaseCRUD[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
-		Create(record *portainer.HelmUserRepository) error
-		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
-		DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error
 	}
 
 	// JWTService represents a service for managing JWT tokens
@@ -151,37 +143,23 @@ type (
 
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
-		Registry(ID portainer.RegistryID) (*portainer.Registry, error)
-		Registries() ([]portainer.Registry, error)
-		Create(registry *portainer.Registry) error
-		UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error
-		DeleteRegistry(ID portainer.RegistryID) error
+		BaseCRUD[portainer.Registry, portainer.RegistryID]
 	}
 
 	// ResourceControlService represents a service for managing resource control data
 	ResourceControlService interface {
-		ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error)
+		BaseCRUD[portainer.ResourceControl, portainer.ResourceControlID]
 		ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error)
-		ResourceControls() ([]portainer.ResourceControl, error)
-		Create(rc *portainer.ResourceControl) error
-		UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error
-		DeleteResourceControl(ID portainer.ResourceControlID) error
 	}
 
 	// RoleService represents a service for managing user roles
 	RoleService interface {
-		Role(ID portainer.RoleID) (*portainer.Role, error)
-		Roles() ([]portainer.Role, error)
-		Create(role *portainer.Role) error
-		UpdateRole(ID portainer.RoleID, role *portainer.Role) error
+		BaseCRUD[portainer.Role, portainer.RoleID]
 	}
 
 	// APIKeyRepositoryService
 	APIKeyRepository interface {
-		CreateAPIKey(key *portainer.APIKey) error
-		GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error)
-		UpdateAPIKey(key *portainer.APIKey) error
-		DeleteAPIKey(ID portainer.APIKeyID) error
+		BaseCRUD[portainer.APIKey, portainer.APIKeyID]
 		GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error)
 		GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error)
 	}
@@ -190,64 +168,47 @@ type (
 	SettingsService interface {
 		Settings() (*portainer.Settings, error)
 		UpdateSettings(settings *portainer.Settings) error
+		BucketName() string
 	}
 
 	SnapshotService interface {
-		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
-		Snapshots() ([]portainer.Snapshot, error)
-		UpdateSnapshot(snapshot *portainer.Snapshot) error
-		DeleteSnapshot(endpointID portainer.EndpointID) error
-		Create(snapshot *portainer.Snapshot) error
+		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
 	}
 
 	// SSLSettingsService represents a service for managing application settings
 	SSLSettingsService interface {
 		Settings() (*portainer.SSLSettings, error)
 		UpdateSettings(settings *portainer.SSLSettings) error
+		BucketName() string
 	}
 
 	// StackService represents a service for managing stack data
 	StackService interface {
-		Stack(ID portainer.StackID) (*portainer.Stack, error)
+		BaseCRUD[portainer.Stack, portainer.StackID]
 		StackByName(name string) (*portainer.Stack, error)
 		StacksByName(name string) ([]portainer.Stack, error)
-		Stacks() ([]portainer.Stack, error)
-		Create(stack *portainer.Stack) error
-		UpdateStack(ID portainer.StackID, stack *portainer.Stack) error
-		DeleteStack(ID portainer.StackID) error
+		GetNextIdentifier() int
 		StackByWebhookID(ID string) (*portainer.Stack, error)
 		RefreshableStacks() ([]portainer.Stack, error)
 	}
 
 	// TagService represents a service for managing tag data
 	TagService interface {
-		Tags() ([]portainer.Tag, error)
-		Tag(ID portainer.TagID) (*portainer.Tag, error)
-		Create(tag *portainer.Tag) error
-		UpdateTag(ID portainer.TagID, tag *portainer.Tag) error
+		BaseCRUD[portainer.Tag, portainer.TagID]
 		UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error
-		DeleteTag(ID portainer.TagID) error
 	}
 
 	// TeamService represents a service for managing user data
 	TeamService interface {
-		Team(ID portainer.TeamID) (*portainer.Team, error)
+		BaseCRUD[portainer.Team, portainer.TeamID]
 		TeamByName(name string) (*portainer.Team, error)
-		Teams() ([]portainer.Team, error)
-		Create(team *portainer.Team) error
-		UpdateTeam(ID portainer.TeamID, team *portainer.Team) error
-		DeleteTeam(ID portainer.TeamID) error
 	}
 
 	// TeamMembershipService represents a service for managing team membership data
 	TeamMembershipService interface {
-		TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error)
-		TeamMemberships() ([]portainer.TeamMembership, error)
+		BaseCRUD[portainer.TeamMembership, portainer.TeamMembershipID]
 		TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error)
 		TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error)
-		Create(membership *portainer.TeamMembership) error
-		UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error
-		DeleteTeamMembership(ID portainer.TeamMembershipID) error
 		DeleteTeamMembershipByUserID(userID portainer.UserID) error
 		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
 		DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error
@@ -257,48 +218,29 @@ type (
 	TunnelServerService interface {
 		Info() (*portainer.TunnelServerInfo, error)
 		UpdateInfo(info *portainer.TunnelServerInfo) error
+		BucketName() string
 	}
 
 	// UserService represents a service for managing user data
 	UserService interface {
-		User(ID portainer.UserID) (*portainer.User, error)
+		BaseCRUD[portainer.User, portainer.UserID]
 		UserByUsername(username string) (*portainer.User, error)
-		Users() ([]portainer.User, error)
 		UsersByRole(role portainer.UserRole) ([]portainer.User, error)
-		Create(user *portainer.User) error
-		UpdateUser(ID portainer.UserID, user *portainer.User) error
-		DeleteUser(ID portainer.UserID) error
 	}
 
 	// VersionService represents a service for managing version data
 	VersionService interface {
-		Edition() (portainer.SoftwareEdition, error)
 		InstanceID() (string, error)
-		// UpdateInstanceID(ID string) error
+		UpdateInstanceID(ID string) error
+		Edition() (portainer.SoftwareEdition, error)
 		Version() (*models.Version, error)
 		UpdateVersion(*models.Version) error
-		GetAll() (map[string]interface{}, error)
-		UpdateAll(map[string]interface{}) error
 	}
 
 	// WebhookService represents a service for managing webhook data.
 	WebhookService interface {
-		Webhooks() ([]portainer.Webhook, error)
-		Webhook(ID portainer.WebhookID) (*portainer.Webhook, error)
-		Create(portainer *portainer.Webhook) error
-		UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error
+		BaseCRUD[portainer.Webhook, portainer.WebhookID]
 		WebhookByResourceID(resourceID string) (*portainer.Webhook, error)
 		WebhookByToken(token string) (*portainer.Webhook, error)
-		DeleteWebhook(ID portainer.WebhookID) error
 	}
 )
-
-func IsErrObjectNotFound(e error) bool {
-	var sqliteErr sqlite3.Error
-	errNotFound := false
-	if errors.As(e, &sqliteErr) {
-		errNotFound = sqliteErr.Code == sqlite3.ErrError
-	}
-
-	return errNotFound || errors.Is(e, gorm.ErrRecordNotFound)
-}

api/dataservices/pendingactions/pendingactions.go

@@ -0,0 +1,74 @@
+package pendingactions
+
+import (
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+const (
+	BucketName = "pending_actions"
+)
+
+type Service struct {
+	dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]
+}
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]
+}
+
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		BaseDataService: dataservices.BaseDataService[portainer.PendingActions, portainer.PendingActionsID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
+	}, nil
+}
+
+func (s Service) Create(config *portainer.PendingActions) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Create(config)
+	})
+}
+
+func (s Service) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
+	return s.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Update(ID, config)
+	})
+}
+
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.PendingActions, portainer.PendingActionsID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
+	}
+}
+
+func (s ServiceTx) Create(config *portainer.PendingActions) error {
+	return s.Tx.CreateObject(BucketName, func(id uint64) (int, interface{}) {
+		config.ID = portainer.PendingActionsID(id)
+		config.CreatedAt = time.Now().Unix()
+
+		return int(config.ID), config
+	})
+}
+
+func (s ServiceTx) Update(ID portainer.PendingActionsID, config *portainer.PendingActions) error {
+	return s.BaseDataServiceTx.Update(ID, config)
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
+}

api/dataservices/registry/registry.go

@@ -2,6 +2,7 @@ package registry
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,89 +10,41 @@ const BucketName = "registries"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Registry returns a registry by ID.
-func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &registry)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service *Service) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Registry{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		registry, ok := obj.(*portainer.Registry)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-	// 			return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-	// 		}
-
-	// 		registries = append(registries, *registry)
-
-	// 		return &portainer.Registry{}, nil
-	// 	})
-
-	return registries, nil
-}
-
 // Create creates a new registry.
 func (service *Service) Create(registry *portainer.Registry) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		registry.ID = portainer.RegistryID(id)
-	// 		return int(registry.ID), registry
-	// 	},
-	// )
-	return nil
-}
-
-// UpdateRegistry updates a registry.
-func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, registry)
-	return nil
-}
-
-// DeleteRegistry deletes a registry.
-func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			registry.ID = portainer.RegistryID(id)
+			return int(registry.ID), registry
+		},
+	)
 }

api/dataservices/registry/tx.go

@@ -1,59 +1,17 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Registry returns a registry by ID.
-func (service ServiceTx) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &registry)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service ServiceTx) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
+	dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]
 }
 
 // Create creates a new registry.
 func (service ServiceTx) Create(registry *portainer.Registry) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -61,17 +19,3 @@ func (service ServiceTx) Create(registry *portainer.Registry) error {
 		},
 	)
 }
-
-// UpdateRegistry updates a registry.
-func (service ServiceTx) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, registry)
-	return nil
-}
-
-// DeleteRegistry deletes a registry.
-func (service ServiceTx) DeleteRegistry(ID portainer.RegistryID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
-}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -1,7 +1,13 @@
 package resourcecontrol
 
 import (
+	"errors"
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,126 +15,78 @@ const BucketName = "resource_control"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// ResourceControl returns a ResourceControl object by ID
-func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &resourceControl)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &resourceControl, nil
-}
-
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
 // to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
 // if no ResourceControl was found.
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
-	// var resourceControl *portainer.ResourceControl
-	// stop := fmt.Errorf("ok")
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.ResourceControl{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		rc, ok := obj.(*portainer.ResourceControl)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-	// 			return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-	// 		}
+	var resourceControl *portainer.ResourceControl
+	stop := fmt.Errorf("ok")
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.ResourceControl{},
+		func(obj interface{}) (interface{}, error) {
+			rc, ok := obj.(*portainer.ResourceControl)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
+			}
 
-	// 		if rc.ResourceID == resourceID && rc.Type == resourceType {
-	// 			resourceControl = rc
-	// 			return nil, stop
-	// 		}
+			if rc.ResourceID == resourceID && rc.Type == resourceType {
+				resourceControl = rc
+				return nil, stop
+			}
 
-	// 		for _, subResourceID := range rc.SubResourceIDs {
-	// 			if subResourceID == resourceID {
-	// 				resourceControl = rc
-	// 				return nil, stop
-	// 			}
-	// 		}
+			for _, subResourceID := range rc.SubResourceIDs {
+				if subResourceID == resourceID {
+					resourceControl = rc
+					return nil, stop
+				}
+			}
 
-	// 		return &portainer.ResourceControl{}, nil
-	// 	})
-	// if errors.Is(err, stop) {
-	// 	return resourceControl, nil
-	// }
+			return &portainer.ResourceControl{}, nil
+		})
+	if errors.Is(err, stop) {
+		return resourceControl, nil
+	}
 
-	return nil, nil
-}
-
-// ResourceControls returns all the ResourceControl objects
-func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.ResourceControl{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		rc, ok := obj.(*portainer.ResourceControl)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-	// 			return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-	// 		}
-
-	// 		rcs = append(rcs, *rc)
-
-	// 		return &portainer.ResourceControl{}, nil
-	// 	})
-
-	return rcs, nil
+	return nil, err
 }
 
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		resourceControl.ID = portainer.ResourceControlID(id)
-	// 		return int(resourceControl.ID), resourceControl
-	// 	},
-	// )
-	return nil
-}
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, resourceControl)
-	return nil
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			resourceControl.ID = portainer.ResourceControlID(id)
+			return int(resourceControl.ID), resourceControl
+		},
+	)
 }

api/dataservices/resourcecontrol/tx.go

@@ -5,30 +5,13 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// ResourceControl returns a ResourceControl object by ID
-func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &resourceControl)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &resourceControl, nil
+	dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
@@ -37,14 +20,14 @@ func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*porta
 func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.tx.GetAll(
+	err := service.Tx.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -68,31 +51,9 @@ func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, r
 	return nil, err
 }
 
-// ResourceControls returns all the ResourceControl objects
-func (service ServiceTx) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
-}
-
 // CreateResourceControl creates a new ResourceControl object
 func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -100,17 +61,3 @@ func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) erro
 		},
 	)
 }
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service ServiceTx) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, resourceControl)
-	return nil
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service ServiceTx) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
-}

api/dataservices/role/role.go

@@ -2,70 +2,49 @@ package role
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "roles"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.Role, portainer.RoleID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Role returns a Role by ID
-func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	db := service.connection.GetDB()
-
-	tx := db.First(&set, `id = ?`, ID)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service *Service) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	db := service.connection.GetDB()
-	tx := db.Find(&sets)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return sets, nil
-}
-
 // CreateRole creates a new Role.
 func (service *Service) Create(role *portainer.Role) error {
-	db := service.connection.GetDB()
-	tx := db.Create(&role)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-// UpdateRole updates a role.
-func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	db := service.connection.GetDB()
-	role.ID = ID
-	tx := db.Save(&role)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			role.ID = portainer.RoleID(id)
+			return int(role.ID), role
+		},
+	)
 }

api/dataservices/role/tx.go

@@ -2,64 +2,20 @@ package role
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-// Service represents a service for managing environment(endpoint) data.
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-// Role returns a Role by ID
-func (service ServiceTx) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &set)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service ServiceTx) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	// err := service.tx.GetAll(
-	// 	BucketName,
-	// 	&portainer.Role{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		set, ok := obj.(*portainer.Role)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-	// 			return nil, fmt.Errorf("failed to convert to Role object: %s", obj)
-	// 		}
-
-	// 		sets = append(sets, *set)
-
-	// 		return &portainer.Role{}, nil
-	// 	})
-
-	return sets, nil
+	dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]
 }
 
 // CreateRole creates a new Role.
 func (service ServiceTx) Create(role *portainer.Role) error {
-	// return service.tx.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		role.ID = portainer.RoleID(id)
-	// 		return int(role.ID), role
-	// 	},
-	// )
-	return nil
-}
-
-// UpdateRole updates a role.
-func (service ServiceTx) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, role)
-	return nil
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			role.ID = portainer.RoleID(id)
+			return int(role.ID), role
+		},
+	)
 }

api/dataservices/schedule/schedule.go

@@ -2,12 +2,11 @@ package schedule
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "schedules"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "schedules"
 
 // Service represents a service for managing schedule data.
 type Service struct {
@@ -20,10 +19,10 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
 		connection: connection,
@@ -33,50 +32,37 @@ func NewService(connection portainer.Connection) (*Service, error) {
 // Schedule returns a schedule by ID.
 func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule, error) {
 	var schedule portainer.Schedule
-	// identifier := service.connection.ConvertToKey(int(ID))
+	identifier := service.connection.ConvertToKey(int(ID))
 
-	// err := service.connection.GetObject(BucketName, identifier, &schedule)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := service.connection.GetObject(BucketName, identifier, &schedule)
+	if err != nil {
+		return nil, err
+	}
 
 	return &schedule, nil
 }
 
 // UpdateSchedule updates a schedule.
 func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, schedule)
-	return nil
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
 func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Schedule{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		schedule, ok := obj.(*portainer.Schedule)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-	// 			return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-	// 		}
-
-	// 		schedules = append(schedules, *schedule)
-
-	// 		return &portainer.Schedule{}, nil
-	// 	})
-
-	return schedules, nil
+	return schedules, service.connection.GetAll(
+		BucketName,
+		&portainer.Schedule{},
+		dataservices.AppendFn(&schedules),
+	)
 }
 
 // SchedulesByJobType return a array containing all the schedules
@@ -84,28 +70,21 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Schedule{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		schedule, ok := obj.(*portainer.Schedule)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-	// 			return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-	// 		}
-
-	// 		if schedule.JobType == jobType {
-	// 			schedules = append(schedules, *schedule)
-	// 		}
-
-	// 		return &portainer.Schedule{}, nil
-	// 	})
-
-	return schedules, nil
+	return schedules, service.connection.GetAll(
+		BucketName,
+		&portainer.Schedule{},
+		dataservices.FilterFn(&schedules, func(e portainer.Schedule) bool {
+			return e.JobType == jobType
+		}),
+	)
 }
 
 // Create assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	// return service.connection.CreateObjectWithId(BucketName, int(schedule.ID), schedule)
-	return nil
+	return service.connection.CreateObjectWithId(BucketName, int(schedule.ID), schedule)
+}
+
+// GetNextIdentifier returns the next identifier for a schedule.
+func (service *Service) GetNextIdentifier() int {
+	return service.connection.GetNextIdentifier(BucketName)
 }

api/dataservices/settings/settings.go

@@ -15,8 +15,17 @@ type Service struct {
 	connection portainer.Connection
 }
 
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
 		connection: connection,
 	}, nil
@@ -31,9 +40,9 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 
 // Settings retrieve the settings object.
 func (service *Service) Settings() (*portainer.Settings, error) {
-	settings := portainer.Settings{ID: 1}
+	var settings portainer.Settings
 
-	err := service.connection.GetByID(1, &settings)
+	err := service.connection.GetObject(BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -43,10 +52,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&portainer.Settings{}).Where(portainer.Settings{ID: 1}).Save(settings)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
 }

api/dataservices/snapshot/snapshot.go

@@ -2,6 +2,7 @@ package snapshot
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -9,51 +10,33 @@ const (
 )
 
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]
 }
 
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-
-	return &snapshot, nil
-}
-
-func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	return snapshots, nil
-}
-
-func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	// identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
-	// return service.connection.UpdateObject(BucketName, identifier, snapshot)
-	return nil
-}
-
-func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	// identifier := service.connection.ConvertToKey(int(endpointID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
-}
-
 func (service *Service) Create(snapshot *portainer.Snapshot) error {
-	// return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
-	return nil
+	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/snapshot/tx.go

@@ -2,61 +2,13 @@ package snapshot
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-	// identifier := service.service.connection.ConvertToKey(int(endpointID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &snapshot)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &snapshot, nil
-}
-
-func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	// err := service.tx.GetAllWithJsoniter(
-	// 	BucketName,
-	// 	&portainer.Snapshot{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		snapshot, ok := obj.(*portainer.Snapshot)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-	// 			return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-	// 		}
-	// 		snapshots = append(snapshots, *snapshot)
-	// 		return &portainer.Snapshot{}, nil
-	// 	})
-
-	return snapshots, nil
-}
-
-func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	// identifier := service.service.connection.ConvertToKey(int(snapshot.EndpointID))
-	// return service.tx.UpdateObject(BucketName, identifier, snapshot)
-	return nil
-}
-
-func (service ServiceTx) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	// identifier := service.service.connection.ConvertToKey(int(endpointID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
+	dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]
 }
 
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
-	// return service.tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
-	return nil
+	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/ssl/ssl.go

@@ -4,13 +4,28 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "ssl"
+	key        = "SSL"
+)
+
 // Service represents a service for managing ssl data.
 type Service struct {
 	connection portainer.Connection
 }
 
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
 		connection: connection,
 	}, nil
@@ -18,22 +33,17 @@ func NewService(connection portainer.Connection) (*Service, error) {
 
 // Settings retrieve the ssl settings object.
 func (service *Service) Settings() (*portainer.SSLSettings, error) {
-	var obj portainer.SSLSettings
-	err := service.connection.GetByID(1, &obj)
+	var settings portainer.SSLSettings
+
+	err := service.connection.GetObject(BucketName, []byte(key), &settings)
 	if err != nil {
 		return nil, err
 	}
 
-	return &obj, nil
+	return &settings, nil
 }
 
 // UpdateSettings persists a SSLSettings object.
 func (service *Service) UpdateSettings(settings *portainer.SSLSettings) error {
-	db := service.connection.GetDB()
-	settings.ID = 1
-	tx := db.Save(settings)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return service.connection.UpdateObject(BucketName, []byte(key), settings)
 }

api/dataservices/stack/stack.go

@@ -1,136 +1,123 @@
 package stack
 
 import (
+	"errors"
+	"strings"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "stacks"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.Stack, portainer.StackID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// Stack returns a stack object by ID.
-func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
-	var obj portainer.Stack
-
-	err := service.connection.GetByID(int(ID), &obj)
+	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
 	}
 
-	return &obj, nil
+	return &Service{
+		BaseDataService: dataservices.BaseDataService[portainer.Stack, portainer.StackID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
+	}, nil
+}
+
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: service.BaseDataService.Tx(tx),
+	}
 }
 
 // StackByName returns a stack object by name.
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 	var s portainer.Stack
 
-	db := service.connection.GetDB()
-	tx := db.First(&s, `name = ?`, name)
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 
-	if tx.Error != nil {
-		return nil, tx.Error
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
 
-	return &s, nil
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
 }
 
 // Stacks returns an array containing all the stacks with same name
 func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	db := service.connection.GetDB()
-	tx := db.Find(&stacks, `name = ?`, name)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return stacks, nil
+	return stacks, service.Connection.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 }
 
-// Stacks returns an array containing all the stacks.
-func (service *Service) Stacks() ([]portainer.Stack, error) {
-	var stacks = make([]portainer.Stack, 0)
-
-	db := service.connection.GetDB()
-	tx := db.Find(&stacks)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return stacks, nil
+// GetNextIdentifier returns the next identifier for a stack.
+func (service *Service) GetNextIdentifier() int {
+	return service.Connection.GetNextIdentifier(BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) Create(stack *portainer.Stack) error {
-	db := service.connection.GetDB()
-	tx := db.Create(&stack)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-// UpdateStack updates a stack.
-func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
-	db := service.connection.GetDB()
-	stack.ID = ID
-	tx := db.Save(&stack)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-// DeleteStack deletes a stack.
-func (service *Service) DeleteStack(ID portainer.StackID) error {
-	return service.connection.DeleteByID(int(ID), &portainer.Stack{})
+	return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
 }
 
 // StackByWebhookID returns a pointer to a stack object by webhook ID.
 // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
 func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	db := service.connection.GetDB()
+	var s portainer.Stack
 
-	var stacks = make([]portainer.Stack, 0)
-	tx := db.Find(&stacks)
-	if tx.Error != nil {
-		return nil, tx.Error
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
 
-	for _, stack := range stacks {
-		if stack.AutoUpdate != nil && stack.AutoUpdate.Webhook != "" && stack.AutoUpdate.Webhook == id {
-			return &stack, nil
-		}
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
 	}
 
-	return nil, nil
+	return nil, err
+
 }
 
 // RefreshableStacks returns stacks that are configured for a periodic update
 func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
 
-	db := service.connection.GetDB()
-
-	var stacks = make([]portainer.Stack, 0)
-	var stacksRes = make([]portainer.Stack, 0)
-	tx := db.Find(&stacks)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	for _, stack := range stacks {
-		if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
-			stacksRes = append(stacksRes, stack)
-		}
-	}
-
-	return stacksRes, nil
+	return stacks, service.Connection.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
 }

api/dataservices/stack/tests/stack_test.go

@@ -58,7 +58,7 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 		Type:         portainer.DockerComposeStack,
 		EndpointID:   2,
 		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Env:          portainer.MultiPair{{Name: "Name1", Value1: "Value1"}},
+		Env:          []portainer.Pair{{Name: "Name1", Value: "Value1"}},
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 		ProjectPath:  "/tmp/project",

api/dataservices/stack/tx.go

@@ -0,0 +1,98 @@
+package stack
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.Stack, portainer.StackID]
+}
+
+// StackByName returns a stack object by name.
+func (service ServiceTx) StackByName(name string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// Stacks returns an array containing all the stacks with same name
+func (service ServiceTx) StacksByName(name string) ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+}
+
+// GetNextIdentifier returns the next identifier for a stack.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
+// CreateStack creates a new stack.
+func (service ServiceTx) Create(stack *portainer.Stack) error {
+	return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack)
+}
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
+}

api/dataservices/tag/tag.go

@@ -2,6 +2,7 @@ package tag
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -11,96 +12,51 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Tag, portainer.TagID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Tags return an array containing all the tags.
-func (service *Service) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Tag{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		tag, ok := obj.(*portainer.Tag)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-	// 			return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
-	// 		}
-
-	// 		tags = append(tags, *tag)
-
-	// 		return &portainer.Tag{}, nil
-	// 	})
-
-	return tags, nil
-}
-
-// Tag returns a tag by ID.
-func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &tag)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &tag, nil
-}
-
 // CreateTag creates a new tag.
 func (service *Service) Create(tag *portainer.Tag) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		tag.ID = portainer.TagID(id)
-	// 		return int(tag.ID), tag
-	// 	},
-	// )
-	return nil
-}
-
-// Deprecated: Use UpdateTagFunc instead.
-func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, tag)
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			tag.ID = portainer.TagID(id)
+			return int(tag.ID), tag
+		},
+	)
 }
 
 // UpdateTagFunc updates a tag inside a transaction avoiding data races.
 func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
-	// id := service.connection.ConvertToKey(int(ID))
-	// tag := &portainer.Tag{}
+	id := service.Connection.ConvertToKey(int(ID))
+	tag := &portainer.Tag{}
 
-	// return service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
-	// 	updateFunc(tag)
-	// })
-	return nil
-}
-
-// DeleteTag deletes a tag.
-func (service *Service) DeleteTag(ID portainer.TagID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	return service.Connection.UpdateObjectFunc(BucketName, id, tag, func() {
+		updateFunc(tag)
+	})
 }

api/dataservices/tag/tx.go

@@ -2,60 +2,18 @@ package tag
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Tags return an array containing all the tags.
-func (service ServiceTx) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
-}
-
-// Tag returns a tag by ID.
-func (service ServiceTx) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &tag)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &tag, nil
+	dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]
 }
 
 // CreateTag creates a new tag.
 func (service ServiceTx) Create(tag *portainer.Tag) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -64,21 +22,7 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTag updates a tag
-func (service ServiceTx) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, tag)
-	return nil
-}
-
 // UpdateTagFunc is a no-op inside a transaction
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }
-
-// DeleteTag deletes a tag.
-func (service ServiceTx) DeleteTag(ID portainer.TagID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
-}

api/dataservices/team/team.go

@@ -1,91 +1,67 @@
 package team
 
 import (
-	"fmt"
+	"errors"
+	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "teams"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.Team, portainer.TeamID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// Team returns a Team by ID
-func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
-	var obj portainer.Team
-
-	err := service.connection.GetByID(int(ID), &obj)
+	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
 	}
 
-	return &obj, nil
+	return &Service{
+		BaseDataService: dataservices.BaseDataService[portainer.Team, portainer.TeamID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
+	}, nil
 }
 
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
-	var team portainer.Team
+	var t portainer.Team
 
-	db := service.connection.GetDB()
-	tx := db.First(&team, `name = ?`, name)
-	if tx.Error != nil {
-		return nil, tx.Error
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.Team{},
+		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+			return strings.EqualFold(e.Name, name)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &t, nil
 	}
 
-	return &team, nil
-}
-
-// Teams return an array containing all the teams.
-func (service *Service) Teams() ([]portainer.Team, error) {
-	var teams = make([]portainer.Team, 0)
-
-	db := service.connection.GetDB()
-	tx := db.Find(&teams)
-	if tx.Error != nil {
-		return nil, tx.Error
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
 	}
 
-	return teams, nil
-}
-
-// UpdateTeam saves a Team.
-func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
-	db := service.connection.GetDB()
-	team.ID = ID
-	tx := db.Save(&team)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return nil, err
 }
 
 // CreateTeam creates a new Team.
 func (service *Service) Create(team *portainer.Team) error {
-	db := service.connection.GetDB()
-
-	tx := db.Create(&team)
-	if tx.Error != nil {
-		fmt.Println(tx.Error)
-		return tx.Error
-	}
-
-	return nil
-}
-
-// DeleteTeam deletes a Team.
-func (service *Service) DeleteTeam(ID portainer.TeamID) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&portainer.Team{}).Delete("id = ?", ID)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			team.ID = portainer.TeamID(id)
+			return int(team.ID), team
+		},
+	)
 }

api/dataservices/teammembership/teammembership.go

@@ -1,120 +1,142 @@
 package teammembership
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "team_membership"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// TeamMembership returns a TeamMembership object by ID
-func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var obj portainer.TeamMembership
-
-	err := service.connection.GetByID(int(ID), &obj)
-	if err != nil {
-		return nil, err
-	}
-
-	return &obj, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	db := service.connection.GetDB()
-	tx := db.Find(&memberships)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return memberships, nil
-}
-
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	db := service.connection.GetDB()
-	tx := db.Find(&memberships, `user_id = ?`, userID)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return memberships, nil
+	return memberships, service.Connection.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	db := service.connection.GetDB()
-	tx := db.Find(&memberships, `team_id = ?`, teamID)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return memberships, nil
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	db := service.connection.GetDB()
-
-	membership.ID = ID
-	tx := db.Save(&membership)
-	if tx.Error != nil {
-		return tx.Error
-	}
-
-	return nil
+	return memberships, service.Connection.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) Create(membership *portainer.TeamMembership) error {
-	db := service.connection.GetDB()
-	tx := db.Create(&membership)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	return service.connection.DeleteByID(int(ID), &portainer.TeamMembership{})
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			membership.ID = portainer.TeamMembershipID(id)
+			return int(membership.ID), membership
+		},
+	)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	db := service.connection.GetDB()
-	return db.Model(&portainer.TeamMembership{}).Delete("user_id = ?", userID).Error
+	return service.Connection.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.UserID == userID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
 }
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	db := service.connection.GetDB()
-	return db.Model(&portainer.TeamMembership{}).Delete("team_id = ?", teamID).Error
+	return service.Connection.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.TeamID == teamID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
 }
 
 func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	db := service.connection.GetDB()
-	return db.Model(&portainer.TeamMembership{}).Delete("team_id = ? AND user_id = ?", teamID, userID).Error
+	return service.Connection.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
+
+			if membership.TeamID == teamID && membership.UserID == userID {
+				return int(membership.ID), true
+			}
+
+			return -1, false
+		})
 }

api/dataservices/teammembership/tx.go

@@ -1,184 +1,113 @@
 package teammembership
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-// TeamMembership returns a TeamMembership object by ID
-func (service ServiceTx) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var membership portainer.TeamMembership
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-
-	// err := service.tx.GetObject(BucketName, identifier, &membership)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service ServiceTx) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	// err := service.tx.GetAll(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		membership, ok := obj.(*portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 		}
-
-	// 		memberships = append(memberships, *membership)
-
-	// 		return &portainer.TeamMembership{}, nil
-	// 	})
-
-	return memberships, nil
+	dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service ServiceTx) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	// err := service.tx.GetAll(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		membership, ok := obj.(*portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 		}
-
-	// 		if membership.UserID == userID {
-	// 			memberships = append(memberships, *membership)
-	// 		}
-
-	// 		return &portainer.TeamMembership{}, nil
-	// 	})
-
-	return memberships, nil
+	return memberships, service.Tx.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	// err := service.tx.GetAll(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		membership, ok := obj.(*portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 		}
-
-	// 		if membership.TeamID == teamID {
-	// 			memberships = append(memberships, *membership)
-	// 		}
-
-	// 		return &portainer.TeamMembership{}, nil
-	// 	})
-
-	return memberships, nil
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service ServiceTx) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.UpdateObject(BucketName, identifier, membership)
-	return nil
+	return memberships, service.Tx.GetAll(
+		BucketName,
+		&portainer.TeamMembership{},
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
-	// return service.tx.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		membership.ID = portainer.TeamMembershipID(id)
-	// 		return int(membership.ID), membership
-	// 	},
-	// )
-	return nil
-}
-
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service ServiceTx) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	// identifier := service.service.connection.ConvertToKey(int(ID))
-	// return service.tx.DeleteObject(BucketName, identifier)
-	return nil
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			membership.ID = portainer.TeamMembershipID(id)
+			return int(membership.ID), membership
+		},
+	)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	// return service.tx.DeleteAllObjects(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (id int, ok bool) {
-	// 		membership, ok := obj.(portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 			return -1, false
-	// 		}
+	return service.Tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
 
-	// 		if membership.UserID == userID {
-	// 			return int(membership.ID), true
-	// 		}
+			if membership.UserID == userID {
+				return int(membership.ID), true
+			}
 
-	// 		return -1, false
-	// 	})
-	return nil
+			return -1, false
+		})
 }
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	// return service.tx.DeleteAllObjects(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (id int, ok bool) {
-	// 		membership, ok := obj.(portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 			return -1, false
-	// 		}
+	return service.Tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
 
-	// 		if membership.TeamID == teamID {
-	// 			return int(membership.ID), true
-	// 		}
+			if membership.TeamID == teamID {
+				return int(membership.ID), true
+			}
 
-	// 		return -1, false
-	// 	})
-	return nil
+			return -1, false
+		})
 }
 
 func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	// return service.tx.DeleteAllObjects(
-	// 	BucketName,
-	// 	&portainer.TeamMembership{},
-	// 	func(obj interface{}) (id int, ok bool) {
-	// 		membership, ok := obj.(portainer.TeamMembership)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-	// 			//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-	// 			return -1, false
-	// 		}
+	return service.Tx.DeleteAllObjects(
+		BucketName,
+		&portainer.TeamMembership{},
+		func(obj interface{}) (id int, ok bool) {
+			membership, ok := obj.(portainer.TeamMembership)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
+				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
+				return -1, false
+			}
 
-	// 		if membership.TeamID == teamID && membership.UserID == userID {
-	// 			return int(membership.ID), true
-	// 		}
+			if membership.TeamID == teamID && membership.UserID == userID {
+				return int(membership.ID), true
+			}
 
-	// 		return -1, false
-	// 	})
-	return nil
+			return -1, false
+		})
 }

api/dataservices/tunnelserver/tunnelserver.go

@@ -21,6 +21,11 @@ func (service *Service) BucketName() string {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
 		connection: connection,
 	}, nil
@@ -30,16 +35,15 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 	var info portainer.TunnelServerInfo
 
-	// err := service.connection.GetObject(BucketName, []byte(infoKey), &info)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := service.connection.GetObject(BucketName, []byte(infoKey), &info)
+	if err != nil {
+		return nil, err
+	}
 
 	return &info, nil
 }
 
 // UpdateInfo persists a TunnelServerInfo object.
 func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
-	// return service.connection.UpdateObject(BucketName, []byte(infoKey), settings)
-	return nil
+	return service.connection.UpdateObject(BucketName, []byte(infoKey), settings)
 }

api/dataservices/user/tx.go

@@ -0,0 +1,63 @@
+package user
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]
+}
+
+// UserByUsername returns a user by username.
+func (service ServiceTx) UserByUsername(username string) (*portainer.User, error) {
+	var u portainer.User
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// UsersByRole return an array containing all the users with the specified role.
+func (service ServiceTx) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+
+	return users, service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
+}
+
+// CreateUser creates a new user.
+func (service ServiceTx) Create(user *portainer.User) error {
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			user.ID = portainer.UserID(id)
+			user.Username = strings.ToLower(user.Username)
+
+			return int(user.ID), user
+		},
+	)
+}

api/dataservices/user/user.go

@@ -1,96 +1,92 @@
 package user
 
 import (
+	"errors"
+	"strings"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "users"
+
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.User, portainer.UserID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// User returns a user by ID
-func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
-	var obj portainer.User
-
-	err := service.connection.GetByID(int(ID), &obj)
+	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
 	}
 
-	return &obj, nil
+	return &Service{
+		BaseDataService: dataservices.BaseDataService[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
+	}, nil
+}
+
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
+	}
 }
 
 // UserByUsername returns a user by username.
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
 	var u portainer.User
 
-	db := service.connection.GetDB()
-	tx := db.First(&u, `username = ?`, username)
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
 
-	if tx.Error != nil {
-		return nil, tx.Error
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
 	}
 
-	return &u, nil
-}
-
-// Users return an array containing all the users.
-func (service *Service) Users() ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-
-	db := service.connection.GetDB()
-	tx := db.Find(&users)
-	if tx.Error != nil {
-		return nil, tx.Error
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
 	}
 
-	return users, nil
+	return nil, err
 }
 
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	db := service.connection.GetDB()
-	tx := db.Find(&users, `role = ?`, role)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-
-	return users, nil
-}
-
-// UpdateUser saves a user.
-func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
-	db := service.connection.GetDB()
-	user.ID = ID
-	tx := db.Save(&user)
-	if tx.Error != nil {
-		return tx.Error
-	}
-
-	return nil
+	return users, service.Connection.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
 }
 
 // CreateUser creates a new user.
 func (service *Service) Create(user *portainer.User) error {
-	db := service.connection.GetDB()
-	tx := db.Create(&user)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			user.ID = portainer.UserID(id)
+			user.Username = strings.ToLower(user.Username)
 
-// DeleteUser deletes a user.
-func (service *Service) DeleteUser(ID portainer.UserID) error {
-	return service.connection.DeleteByID(int(ID), &portainer.User{})
+			return int(user.ID), user
+		},
+	)
 }

api/dataservices/version/version.go

@@ -1,12 +1,19 @@
 package version
 
 import (
-	"fmt"
-	"strconv"
+	"errors"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
-	"gorm.io/gorm"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName  = "version"
+	versionKey  = "VERSION"
+	updatingKey = "DB_UPDATING"
 )
 
 // Service represents a service to manage stored versions.
@@ -16,136 +23,101 @@ type Service struct {
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Service{
 		connection: connection,
 	}, nil
 }
 
-// DBVersion retrieves the stored database version.
-func (service *Service) DBVersion() (int, error) {
-	db := service.connection.GetDB()
-	var version models.Version
-	tx := db.First(&version, `key = ?`, models.VersionKey)
-	if tx.Error != nil {
-		return 0, tx.Error
+func (service *Service) SchemaVersion() (string, error) {
+	v, err := service.Version()
+	if err != nil {
+		return "", err
 	}
-	return strconv.Atoi(version.Value)
+
+	return v.SchemaVersion, nil
 }
 
-// Edition retrieves the stored portainer edition.
-func (service *Service) Edition() (portainer.SoftwareEdition, error) {
-	db := service.connection.GetDB()
-	var version models.Version
-	tx := db.First(&version, `key = ?`, models.EditionKey)
-	if tx.Error != nil {
-		return 0, tx.Error
+func (service *Service) UpdateSchemaVersion(version string) error {
+	v, err := service.Version()
+	if err != nil {
+		return err
 	}
-	e, err := strconv.Atoi(version.Value)
+
+	v.SchemaVersion = version
+	return service.UpdateVersion(v)
+}
+
+func (service *Service) Edition() (portainer.SoftwareEdition, error) {
+	v, err := service.Version()
 	if err != nil {
 		return 0, err
 	}
-	fmt.Println(portainer.SoftwareEdition(e))
-	return portainer.SoftwareEdition(e), nil
-}
 
-// StoreDBVersion store the database version.
-func (service *Service) StoreDBVersion(v int) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&models.Version{}).Where("key = ?", models.VersionKey).Update("value", strconv.FormatInt(int64(v), 10)).Limit(1)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
+	return portainer.SoftwareEdition(v.Edition), nil
 }
 
 // IsUpdating retrieves the database updating status.
 func (service *Service) IsUpdating() (bool, error) {
-	db := service.connection.GetDB()
-	var version models.Version
-	tx := db.First(&version, `key = ?`, models.UpdatingKey)
-	if tx.Error != nil {
-		return false, tx.Error
+	var isUpdating bool
+	err := service.connection.GetObject(BucketName, []byte(updatingKey), &isUpdating)
+	if err != nil && errors.Is(err, dserrors.ErrObjectNotFound) {
+		return false, nil
 	}
-	return version.Value == "true", nil
+	return isUpdating, err
 }
 
 // StoreIsUpdating store the database updating status.
 func (service *Service) StoreIsUpdating(isUpdating bool) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&models.Version{}).Where("key = ?", models.UpdatingKey).Update("value", strconv.FormatBool(isUpdating)).Limit(1)
-	if tx.Error != nil {
-		return tx.Error
+	if isUpdating {
+		return service.connection.UpdateObject(BucketName, []byte(updatingKey), isUpdating)
 	}
-	return nil
+
+	return service.connection.DeleteObject(BucketName, []byte(updatingKey))
 }
 
 // InstanceID retrieves the stored instance ID.
 func (service *Service) InstanceID() (string, error) {
-	db := service.connection.GetDB()
-	var version models.Version
-	tx := db.First(&version, `key = ?`, models.InstanceKey)
-	if tx.Error != nil {
-		return "", tx.Error
+	v, err := service.Version()
+	if err != nil {
+		return "", err
 	}
-	return version.Value, nil
+
+	return v.InstanceID, nil
 }
 
 // StoreInstanceID store the instance ID.
-func (service *Service) StoreInstanceID(ID string) error {
-	db := service.connection.GetDB()
-	tx := db.FirstOrCreate(&models.Version{Key: models.InstanceKey, Value: ID})
-	if tx.Error != nil {
-		return tx.Error
+func (service *Service) UpdateInstanceID(id string) error {
+	v, err := service.Version()
+	if err != nil {
+		if !dataservices.IsErrObjectNotFound(err) {
+			return err
+		}
+
+		v = &models.Version{}
 	}
 
-	return nil
+	v.InstanceID = id
+	return service.UpdateVersion(v)
 }
 
 // Version retrieve the version object.
 func (service *Service) Version() (*models.Version, error) {
-	db := service.connection.GetDB()
-	var version models.Version
-	tx := db.First(&version, `key = ?`, models.VersionKey)
-	if tx.Error != nil {
-		return nil, tx.Error
+	var v models.Version
+
+	err := service.connection.GetObject(BucketName, []byte(versionKey), &v)
+	if err != nil {
+		return nil, err
 	}
-	return &version, nil
+
+	return &v, nil
 }
 
 // UpdateVersion persists a Version object.
 func (service *Service) UpdateVersion(version *models.Version) error {
-	db := service.connection.GetDB()
-	tx := db.Model(&models.Version{}).Where("key = ?", models.VersionKey).Update("value", version).Limit(1)
-	if tx.Error != nil {
-		return tx.Error
-	}
-	return nil
-}
-
-// Version retrieve the version object.
-func (service *Service) GetAll() (map[string]interface{}, error) {
-	db := service.connection.GetDB()
-	var all map[string]interface{}
-	tx := db.Find(&all)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-	return all, nil
-}
-
-// Version retrieve the version object.
-func (service *Service) UpdateAll(all map[string]interface{}) error {
-	db := service.connection.GetDB()
-	db.Transaction(func(tx *gorm.DB) error {
-		for k, v := range all {
-			tx := db.Model(&models.Version{}).Where(models.Version{Key: k}).FirstOrCreate(&models.Version{Key: k, Value: fmt.Sprintf("%v", v)})
-			if tx.Error != nil {
-				tx.Rollback()
-				return tx.Error
-			}
-		}
-		tx.Commit()
-		return nil
-	})
-	return nil
+	return service.connection.UpdateObject(BucketName, []byte(versionKey), version)
 }

api/dataservices/webhook/webhook.go

@@ -1,160 +1,89 @@
 package webhook
 
 import (
+	"errors"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "webhooks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "webhooks"
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]
 }
 
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
-	// err := connection.SetServiceName(BucketName)
-	// if err != nil {
-	// 	return nil, err
-	// }
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Webhooks returns an array of all webhooks
-func (service *Service) Webhooks() ([]portainer.Webhook, error) {
-	var webhooks = make([]portainer.Webhook, 0)
-
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Webhook{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		webhook, ok := obj.(*portainer.Webhook)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
-	// 			return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-	// 		}
-
-	// 		webhooks = append(webhooks, *webhook)
-
-	// 		return &portainer.Webhook{}, nil
-	// 	})
-
-	return webhooks, nil
-}
-
-// Webhook returns a webhook by ID.
-func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
-	var webhook portainer.Webhook
-	// identifier := service.connection.ConvertToKey(int(ID))
-
-	// err := service.connection.GetObject(BucketName, identifier, &webhook)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	return &webhook, nil
-}
-
 // WebhookByResourceID returns a webhook by the ResourceID it is associated with.
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
-	// var w *portainer.Webhook
-	// stop := fmt.Errorf("ok")
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Webhook{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		webhook, ok := obj.(*portainer.Webhook)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+	var w portainer.Webhook
 
-	// 			return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-	// 		}
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.Webhook{},
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.ResourceID == ID
+		}),
+	)
 
-	// 		if webhook.ResourceID == ID {
-	// 			w = webhook
-	// 			return nil, stop
-	// 		}
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
+	}
 
-	// 		return &portainer.Webhook{}, nil
-	// 	})
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
 
-	// if errors.Is(err, stop) {
-	// 	return w, nil
-	// }
-
-	// if err == nil {
-	// 	return nil, dserrors.ErrObjectNotFound
-	// }
-
-	return nil, nil
+	return nil, err
 }
 
 // WebhookByToken returns a webhook by the random token it is associated with.
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
-	// var w *portainer.Webhook
-	// stop := fmt.Errorf("ok")
-	// err := service.connection.GetAll(
-	// 	BucketName,
-	// 	&portainer.Webhook{},
-	// 	func(obj interface{}) (interface{}, error) {
-	// 		webhook, ok := obj.(*portainer.Webhook)
-	// 		if !ok {
-	// 			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+	var w portainer.Webhook
 
-	// 			return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-	// 		}
+	err := service.Connection.GetAll(
+		BucketName,
+		&portainer.Webhook{},
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.Token == token
+		}),
+	)
 
-	// 		if webhook.Token == token {
-	// 			w = webhook
-	// 			return nil, stop
-	// 		}
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
+	}
 
-	// 		return &portainer.Webhook{}, nil
-	// 	})
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
 
-	// if errors.Is(err, stop) {
-	// 	return w, nil
-	// }
-
-	// if err == nil {
-	// 	return nil, dserrors.ErrObjectNotFound
-	// }
-
-	return nil, nil
-}
-
-// DeleteWebhook deletes a webhook.
-func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.DeleteObject(BucketName, identifier)
-	return nil
+	return nil, err
 }
 
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) Create(webhook *portainer.Webhook) error {
-	// return service.connection.CreateObject(
-	// 	BucketName,
-	// 	func(id uint64) (int, interface{}) {
-	// 		webhook.ID = portainer.WebhookID(id)
-	// 		return int(webhook.ID), webhook
-	// 	},
-	// )
-	return nil
-}
-
-// UpdateWebhook update a webhook.
-func (service *Service) UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error {
-	// identifier := service.connection.ConvertToKey(int(ID))
-	// return service.connection.UpdateObject(BucketName, identifier, webhook)
-	return nil
+	return service.Connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			webhook.ID = portainer.WebhookID(id)
+			return int(webhook.ID), webhook
+		},
+	)
 }

api/datastore/backup.go

@@ -1,189 +1,189 @@
 package datastore
 
-// import (
-// 	"fmt"
-// 	"os"
-// 	"path"
-// 	"time"
+import (
+	"fmt"
+	"os"
+	"path"
+	"time"
 
-// 	"github.com/portainer/portainer/api/database/models"
-// 	"github.com/rs/zerolog/log"
-// )
+	"github.com/portainer/portainer/api/database/models"
+	"github.com/rs/zerolog/log"
+)
 
-// var backupDefaults = struct {
-// 	backupDir string
-// 	commonDir string
-// }{
-// 	"backups",
-// 	"common",
-// }
+var backupDefaults = struct {
+	backupDir string
+	commonDir string
+}{
+	"backups",
+	"common",
+}
 
-// //
-// // Backup Helpers
-// //
+//
+// Backup Helpers
+//
 
-// // createBackupFolders create initial folders for backups
-// func (store *Store) createBackupFolders() {
-// 	// create common dir
-// 	commonDir := store.commonBackupDir()
-// 	if exists, _ := store.fileService.FileExists(commonDir); !exists {
-// 		if err := os.MkdirAll(commonDir, 0700); err != nil {
-// 			log.Error().Err(err).Msg("error while creating common backup folder")
-// 		}
-// 	}
-// }
+// createBackupFolders create initial folders for backups
+func (store *Store) createBackupFolders() {
+	// create common dir
+	commonDir := store.commonBackupDir()
+	if exists, _ := store.fileService.FileExists(commonDir); !exists {
+		if err := os.MkdirAll(commonDir, 0700); err != nil {
+			log.Error().Err(err).Msg("error while creating common backup folder")
+		}
+	}
+}
 
-// func (store *Store) databasePath() string {
-// 	return store.connection.GetDatabaseFilePath()
-// }
+func (store *Store) databasePath() string {
+	return store.connection.GetDatabaseFilePath()
+}
 
-// func (store *Store) commonBackupDir() string {
-// 	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
-// }
+func (store *Store) commonBackupDir() string {
+	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
+}
 
-// func (store *Store) copyDBFile(from string, to string) error {
-// 	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
+func (store *Store) copyDBFile(from string, to string) error {
+	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
 
-// 	err := store.fileService.Copy(from, to, true)
-// 	if err != nil {
-// 		log.Error().Err(err).Msg("failed")
-// 	}
+	err := store.fileService.Copy(from, to, true)
+	if err != nil {
+		log.Error().Err(err).Msg("failed")
+	}
 
-// 	return err
-// }
+	return err
+}
 
-// // BackupOptions provide a helper to inject backup options
-// type BackupOptions struct {
-// 	Version        string
-// 	BackupDir      string
-// 	BackupFileName string
-// 	BackupPath     string
-// }
+// BackupOptions provide a helper to inject backup options
+type BackupOptions struct {
+	Version        string
+	BackupDir      string
+	BackupFileName string
+	BackupPath     string
+}
 
-// // getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// // - db backup prior to version upgrade
-// // - db rollback
-// func getBackupRestoreOptions(backupDir string) *BackupOptions {
-// 	return &BackupOptions{
-// 		BackupDir:      backupDir, //connection.commonBackupDir(),
-// 		BackupFileName: beforePortainerVersionUpgradeBackup,
-// 	}
-// }
+// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
+// - db backup prior to version upgrade
+// - db rollback
+func getBackupRestoreOptions(backupDir string) *BackupOptions {
+	return &BackupOptions{
+		BackupDir:      backupDir, //connection.commonBackupDir(),
+		BackupFileName: beforePortainerVersionUpgradeBackup,
+	}
+}
 
-// // Backup current database with default options
-// func (store *Store) Backup(version *models.Version) (string, error) {
-// 	if version == nil {
-// 		return store.backupWithOptions(nil)
-// 	}
+// Backup current database with default options
+func (store *Store) Backup(version *models.Version) (string, error) {
+	if version == nil {
+		return store.backupWithOptions(nil)
+	}
 
-// 	return store.backupWithOptions(&BackupOptions{
-// 		Version: version.SchemaVersion,
-// 	})
-// }
+	return store.backupWithOptions(&BackupOptions{
+		Version: version.SchemaVersion,
+	})
+}
 
-// func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-// 	if options == nil {
-// 		options = &BackupOptions{}
-// 	}
-// 	if options.Version == "" {
-// 		v, err := store.VersionService.Version()
-// 		if err != nil {
-// 			options.Version = ""
-// 		}
-// 		options.Version = v.SchemaVersion
-// 	}
-// 	if options.BackupDir == "" {
-// 		options.BackupDir = store.commonBackupDir()
-// 	}
-// 	if options.BackupFileName == "" {
-// 		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
-// 	}
-// 	if options.BackupPath == "" {
-// 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-// 	}
-// 	return options
-// }
+func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
+	if options == nil {
+		options = &BackupOptions{}
+	}
+	if options.Version == "" {
+		v, err := store.VersionService.Version()
+		if err != nil {
+			options.Version = ""
+		}
+		options.Version = v.SchemaVersion
+	}
+	if options.BackupDir == "" {
+		options.BackupDir = store.commonBackupDir()
+	}
+	if options.BackupFileName == "" {
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
+	}
+	if options.BackupPath == "" {
+		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
+	}
+	return options
+}
 
-// // BackupWithOptions backup current database with options
-// func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
-// 	log.Info().Msg("creating DB backup")
+// BackupWithOptions backup current database with options
+func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
+	log.Info().Msg("creating DB backup")
 
-// 	store.createBackupFolders()
+	store.createBackupFolders()
 
-// 	options = store.setupOptions(options)
-// 	dbPath := store.databasePath()
+	options = store.setupOptions(options)
+	dbPath := store.databasePath()
 
-// if err := store.Close(); err != nil {
-// 	return options.BackupPath, fmt.Errorf(
-// 		"error closing datastore before creating backup: %w",
-// 		err,
-// 	)
-// }
+	if err := store.Close(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error closing datastore before creating backup: %w",
+			err,
+		)
+	}
 
-// 	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
-// 		return options.BackupPath, err
-// 	}
+	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
+		return options.BackupPath, err
+	}
 
-// if _, err := store.Open(); err != nil {
-// 	return options.BackupPath, fmt.Errorf(
-// 		"error opening datastore after creating backup: %w",
-// 		err,
-// 	)
-// }
+	if _, err := store.Open(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error opening datastore after creating backup: %w",
+			err,
+		)
+	}
 
-// 	return options.BackupPath, nil
-// }
+	return options.BackupPath, nil
+}
 
-// // RestoreWithOptions previously saved backup for the current Edition  with options
-// // Restore strategies:
-// // - default: restore latest from current edition
-// // - restore a specific
-// func (store *Store) restoreWithOptions(options *BackupOptions) error {
-// 	options = store.setupOptions(options)
+// RestoreWithOptions previously saved backup for the current Edition  with options
+// Restore strategies:
+// - default: restore latest from current edition
+// - restore a specific
+func (store *Store) restoreWithOptions(options *BackupOptions) error {
+	options = store.setupOptions(options)
 
-// 	// Check if backup file exist before restoring
-// 	_, err := os.Stat(options.BackupPath)
-// 	if os.IsNotExist(err) {
-// 		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
+	// Check if backup file exist before restoring
+	_, err := os.Stat(options.BackupPath)
+	if os.IsNotExist(err) {
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
 
-// 		return err
-// 	}
+		return err
+	}
 
-// 	err = store.Close()
-// 	if err != nil {
-// 		log.Error().Err(err).Msg("error while closing store before restore")
+	err = store.Close()
+	if err != nil {
+		log.Error().Err(err).Msg("error while closing store before restore")
 
-// 		return err
-// 	}
+		return err
+	}
 
-// 	log.Info().Msg("restoring DB backup")
-// 	err = store.copyDBFile(options.BackupPath, store.databasePath())
-// 	if err != nil {
-// 		return err
-// 	}
+	log.Info().Msg("restoring DB backup")
+	err = store.copyDBFile(options.BackupPath, store.databasePath())
+	if err != nil {
+		return err
+	}
 
-// 	_, err = store.Open()
-// 	return err
-// }
+	_, err = store.Open()
+	return err
+}
 
-// // RemoveWithOptions removes backup database based on supplied options
-// func (store *Store) removeWithOptions(options *BackupOptions) error {
-// 	log.Info().Msg("removing DB backup")
+// RemoveWithOptions removes backup database based on supplied options
+func (store *Store) removeWithOptions(options *BackupOptions) error {
+	log.Info().Msg("removing DB backup")
 
-// 	options = store.setupOptions(options)
-// 	_, err := os.Stat(options.BackupPath)
+	options = store.setupOptions(options)
+	_, err := os.Stat(options.BackupPath)
 
-// 	if os.IsNotExist(err) {
-// 		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
-// 		return err
-// 	}
+	if os.IsNotExist(err) {
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
+		return err
+	}
 
-// 	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
-// 	err = os.Remove(options.BackupPath)
-// 	if err != nil {
-// 		log.Error().Err(err).Msg("failed")
-// 		return err
-// 	}
+	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
+	err = os.Remove(options.BackupPath)
+	if err != nil {
+		log.Error().Err(err).Msg("failed")
+		return err
+	}
 
-// 	return nil
-// }
+	return nil
+}

api/datastore/backup_test.go

@@ -1,107 +1,107 @@
 package datastore
 
-// import (
-// 	"fmt"
-// 	"os"
-// 	"path"
-// 	"testing"
+import (
+	"fmt"
+	"os"
+	"path"
+	"testing"
 
-// 	portainer "github.com/portainer/portainer/api"
-// 	"github.com/portainer/portainer/api/database/models"
-// )
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
+)
 
-// func TestCreateBackupFolders(t *testing.T) {
-// 	_, store := MustNewTestStore(t, true, true)
+func TestCreateBackupFolders(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
 
-// 	connection := store.GetConnection()
-// 	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
+	connection := store.GetConnection()
+	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
 
-// 	if isFileExist(backupPath) {
-// 		t.Error("Expect backups folder to not exist")
-// 	}
+	if isFileExist(backupPath) {
+		t.Error("Expect backups folder to not exist")
+	}
 
-// 	store.createBackupFolders()
-// 	if !isFileExist(backupPath) {
-// 		t.Error("Expect backups folder to exist")
-// 	}
-// }
+	store.createBackupFolders()
+	if !isFileExist(backupPath) {
+		t.Error("Expect backups folder to exist")
+	}
+}
 
-// func TestStoreCreation(t *testing.T) {
-// 	_, store := MustNewTestStore(t, true, true)
-// 	if store == nil {
-// 		t.Error("Expect to create a store")
-// 	}
+func TestStoreCreation(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
+	if store == nil {
+		t.Error("Expect to create a store")
+	}
 
-// 	if store.CheckCurrentEdition() != nil {
-// 		t.Error("Expect to get CE Edition")
-// 	}
-// }
+	if store.CheckCurrentEdition() != nil {
+		t.Error("Expect to get CE Edition")
+	}
+}
 
-// func TestBackup(t *testing.T) {
-// 	_, store := MustNewTestStore(t, true, true)
-// 	connection := store.GetConnection()
+func TestBackup(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
+	connection := store.GetConnection()
 
-// 	t.Run("Backup should create default db backup", func(t *testing.T) {
-// 		v := models.Version{
-// 			SchemaVersion: portainer.APIVersion,
-// 		}
-// 		store.VersionService.UpdateVersion(&v)
-// 		store.backupWithOptions(nil)
+	t.Run("Backup should create default db backup", func(t *testing.T) {
+		v := models.Version{
+			SchemaVersion: portainer.APIVersion,
+		}
+		store.VersionService.UpdateVersion(&v)
+		store.backupWithOptions(nil)
 
-// 		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
-// 		if !isFileExist(backupFileName) {
-// 			t.Errorf("Expect backup file to be created %s", backupFileName)
-// 		}
-// 	})
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+	})
 
-// 	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-// 		store.backupWithOptions(&BackupOptions{
-// 			BackupFileName: beforePortainerVersionUpgradeBackup,
-// 			BackupDir:      store.commonBackupDir(),
-// 		})
-// 		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
-// 		if !isFileExist(backupFileName) {
-// 			t.Errorf("Expect backup file to be created %s", backupFileName)
-// 		}
-// 	})
-// }
+	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
+		store.backupWithOptions(&BackupOptions{
+			BackupFileName: beforePortainerVersionUpgradeBackup,
+			BackupDir:      store.commonBackupDir(),
+		})
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+	})
+}
 
-// func TestRemoveWithOptions(t *testing.T) {
-// 	_, store := MustNewTestStore(t, true, true)
+func TestRemoveWithOptions(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
 
-// 	t.Run("successfully removes file if existent", func(t *testing.T) {
-// 		store.createBackupFolders()
-// 		options := &BackupOptions{
-// 			BackupDir:      store.commonBackupDir(),
-// 			BackupFileName: "test.txt",
-// 		}
+	t.Run("successfully removes file if existent", func(t *testing.T) {
+		store.createBackupFolders()
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
 
-// 		filePath := path.Join(options.BackupDir, options.BackupFileName)
-// 		f, err := os.Create(filePath)
-// 		if err != nil {
-// 			t.Fatalf("file should be created; err=%s", err)
-// 		}
-// 		f.Close()
+		filePath := path.Join(options.BackupDir, options.BackupFileName)
+		f, err := os.Create(filePath)
+		if err != nil {
+			t.Fatalf("file should be created; err=%s", err)
+		}
+		f.Close()
 
-// 		err = store.removeWithOptions(options)
-// 		if err != nil {
-// 			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
-// 		}
+		err = store.removeWithOptions(options)
+		if err != nil {
+			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
+		}
 
-// 		if isFileExist(f.Name()) {
-// 			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
-// 		}
-// 	})
+		if isFileExist(f.Name()) {
+			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
+		}
+	})
 
-// 	t.Run("fails to removes file if non-existent", func(t *testing.T) {
-// 		options := &BackupOptions{
-// 			BackupDir:      store.commonBackupDir(),
-// 			BackupFileName: "test.txt",
-// 		}
+	t.Run("fails to removes file if non-existent", func(t *testing.T) {
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
 
-// 		err := store.removeWithOptions(options)
-// 		if err == nil {
-// 			t.Error("RemoveWithOptions should fail for non-existent file")
-// 		}
-// 	})
-// }
+		err := store.removeWithOptions(options)
+		if err == nil {
+			t.Error("RemoveWithOptions should fail for non-existent file")
+		}
+	})
+}

api/datastore/datastore.go

@@ -3,14 +3,14 @@ package datastore
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"time"
 
-	"github.com/mattn/go-sqlite3"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
-	"gorm.io/gorm"
 
 	"github.com/rs/zerolog/log"
 )
@@ -25,7 +25,6 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
-
 	encryptionReq, err := store.connection.NeedsEncryptionMigration()
 	if err != nil {
 		return false, err
@@ -40,10 +39,6 @@ func (store *Store) Open() (newStore bool, err error) {
 
 	err = store.connection.Open()
 	if err != nil {
-		fmt.Printf("connection open returned error: %s\n", err.Error())
-		if store.IsErrObjectNotFound(err) || store.IsErrNoSuchTable(err) {
-			return true, nil
-		}
 		return false, err
 	}
 
@@ -55,7 +50,7 @@ func (store *Store) Open() (newStore bool, err error) {
 	// If no settings object exists then assume we have a new store
 	_, err = store.SettingsService.Settings()
 	if err != nil {
-		if store.IsErrObjectNotFound(err) || store.IsErrNoSuchTable(err) {
+		if store.IsErrObjectNotFound(err) {
 			return true, nil
 		}
 		return false, err
@@ -68,9 +63,32 @@ func (store *Store) Close() error {
 	return store.connection.Close()
 }
 
+func (store *Store) UpdateTx(fn func(dataservices.DataStoreTx) error) error {
+	return store.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return fn(&StoreTx{
+			store: store,
+			tx:    tx,
+		})
+	})
+}
+
+func (store *Store) ViewTx(fn func(dataservices.DataStoreTx) error) error {
+	return store.connection.ViewTx(func(tx portainer.Transaction) error {
+		return fn(&StoreTx{
+			store: store,
+			tx:    tx,
+		})
+	})
+}
+
+// BackupTo backs up db to a provided writer.
+// It does hot backup and doesn't block other database reads and writes
+func (store *Store) BackupTo(w io.Writer) error {
+	return store.connection.BackupTo(w)
+}
+
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
-	fmt.Println("Edition:", store.edition(), portainer.Edition)
 	if store.edition() != portainer.Edition {
 		return portainerErrors.ErrWrongDBEdition
 	}
@@ -85,19 +103,9 @@ func (store *Store) edition() portainer.SoftwareEdition {
 	return edition
 }
 
-func (store *Store) IsErrNoSuchTable(e error) bool {
-	return errors.Is(e, portainerErrors.ErrNoSuchTable)
-}
-
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	var sqliteErr sqlite3.Error
-	errNotFound := false
-	if errors.As(e, &sqliteErr) {
-		errNotFound = sqliteErr.Code == sqlite3.ErrError
-	}
-
-	return errNotFound || errors.Is(e, gorm.ErrRecordNotFound)
+	return errors.Is(e, portainerErrors.ErrObjectNotFound)
 }
 
 func (store *Store) Connection() portainer.Connection {
@@ -105,7 +113,7 @@ func (store *Store) Connection() portainer.Connection {
 }
 
 func (store *Store) Rollback(force bool) error {
-	return nil
+	return store.connectionRollback(force)
 }
 
 func (store *Store) encryptDB() error {
@@ -125,7 +133,7 @@ func (store *Store) encryptDB() error {
 	log.Info().Msg("encrypting database")
 
 	// export file path for backup
-	exportFilename := path.Join(store.connection.GetDatabaseFilePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
+	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
 
 	log.Info().Str("filename", exportFilename).Msg("exporting database backup")
 

api/datastore/datastore_test.go

@@ -1,416 +1,416 @@
 package datastore
 
-// import (
-// 	"fmt"
-// 	"runtime"
-// 	"strings"
-// 	"testing"
-
-// 	"github.com/dchest/uniuri"
-// 	"github.com/pkg/errors"
-// 	portainer "github.com/portainer/portainer/api"
-// 	"github.com/portainer/portainer/api/chisel"
-// 	"github.com/portainer/portainer/api/crypto"
-// 	"github.com/stretchr/testify/assert"
-// )
-
-// const (
-// 	adminUsername                       = "admin"
-// 	adminPassword                       = "password"
-// 	standardUsername                    = "standard"
-// 	standardPassword                    = "password"
-// 	agentOnDockerEnvironmentUrl         = "tcp://192.168.167.207:30775"
-// 	edgeAgentOnKubernetesEnvironmentUrl = "tcp://192.168.167.207"
-// 	kubernetesLocalEnvironmentUrl       = "https://kubernetes.default.svc"
-// )
-
-// // TestStoreFull an eventually comprehensive set of tests for the Store.
-// // The idea is what we write to the store, we should read back.
-// func TestStoreFull(t *testing.T) {
-// 	_, store := MustNewTestStore(t, true, true)
-
-// 	testCases := map[string]func(t *testing.T){
-// 		"User Accounts": func(t *testing.T) {
-// 			store.testUserAccounts(t)
-// 		},
-// 		"Environments": func(t *testing.T) {
-// 			store.testEnvironments(t)
-// 		},
-// 		"Settings": func(t *testing.T) {
-// 			store.testSettings(t)
-// 		},
-// 		"SSL Settings": func(t *testing.T) {
-// 			store.testSSLSettings(t)
-// 		},
-// 		"Tunnel Server": func(t *testing.T) {
-// 			store.testTunnelServer(t)
-// 		},
-// 		"Custom Templates": func(t *testing.T) {
-// 			store.testCustomTemplates(t)
-// 		},
-// 		"Registries": func(t *testing.T) {
-// 			store.testRegistries(t)
-// 		},
-// 		"Resource Control": func(t *testing.T) {
-// 			store.testResourceControl(t)
-// 		},
-// 		"Schedules": func(t *testing.T) {
-// 			store.testSchedules(t)
-// 		},
-// 		"Tags": func(t *testing.T) {
-// 			store.testTags(t)
-// 		},
-
-// 		// "Test Title": func(t *testing.T) {
-// 		// },
-// 	}
-
-// 	for name, test := range testCases {
-// 		t.Run(name, test)
-// 	}
-
-// }
-
-// func (store *Store) testEnvironments(t *testing.T) {
-// 	id := store.CreateEndpoint(t, "local", portainer.KubernetesLocalEnvironment, "", true)
-// 	store.CreateEndpointRelation(id)
-
-// 	id = store.CreateEndpoint(t, "agent", portainer.AgentOnDockerEnvironment, agentOnDockerEnvironmentUrl, true)
-// 	store.CreateEndpointRelation(id)
-
-// 	id = store.CreateEndpoint(t, "edge", portainer.EdgeAgentOnKubernetesEnvironment, edgeAgentOnKubernetesEnvironmentUrl, true)
-// 	store.CreateEndpointRelation(id)
-// }
-
-// func newEndpoint(endpointType portainer.EndpointType, id portainer.EndpointID, name, URL string, TLS bool) *portainer.Endpoint {
-// 	endpoint := &portainer.Endpoint{
-// 		ID:        id,
-// 		Name:      name,
-// 		URL:       URL,
-// 		Type:      endpointType,
-// 		GroupID:   portainer.EndpointGroupID(1),
-// 		PublicURL: "",
-// 		TLSConfig: portainer.TLSConfiguration{
-// 			TLS: false,
-// 		},
-// 		UserAccessPolicies: portainer.UserAccessPolicies{},
-// 		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-// 		TagIDs:             []portainer.TagID{},
-// 		Status:             portainer.EndpointStatusUp,
-// 		Snapshots:          []portainer.DockerSnapshot{},
-// 		Kubernetes:         portainer.KubernetesDefault(),
-// 	}
-
-// 	if TLS {
-// 		endpoint.TLSConfig = portainer.TLSConfiguration{
-// 			TLS:           true,
-// 			TLSSkipVerify: true,
-// 		}
-// 	}
-
-// 	return endpoint
-// }
-
-// func setEndpointAuthorizations(endpoint *portainer.Endpoint) {
-// 	endpoint.SecuritySettings = portainer.EndpointSecuritySettings{
-// 		AllowVolumeBrowserForRegularUsers: false,
-// 		EnableHostManagementFeatures:      false,
-
-// 		AllowSysctlSettingForRegularUsers:         true,
-// 		AllowBindMountsForRegularUsers:            true,
-// 		AllowPrivilegedModeForRegularUsers:        true,
-// 		AllowHostNamespaceForRegularUsers:         true,
-// 		AllowContainerCapabilitiesForRegularUsers: true,
-// 		AllowDeviceMappingForRegularUsers:         true,
-// 		AllowStackManagementForRegularUsers:       true,
-// 	}
-// }
-
-// func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType portainer.EndpointType, URL string, tls bool) portainer.EndpointID {
-// 	is := assert.New(t)
-
-// 	var expectedEndpoint *portainer.Endpoint
-// 	id := portainer.EndpointID(store.Endpoint().GetNextIdentifier())
-
-// 	switch endpointType {
-// 	case portainer.DockerEnvironment:
-// 		if URL == "" {
-// 			URL = "unix:///var/run/docker.sock"
-// 			if runtime.GOOS == "windows" {
-// 				URL = "npipe:////./pipe/docker_engine"
-// 			}
-// 		}
-// 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
-
-// 	case portainer.AgentOnDockerEnvironment:
-// 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
-
-// 	case portainer.AgentOnKubernetesEnvironment:
-// 		URL = strings.TrimPrefix(URL, "tcp://")
-// 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
-
-// 	case portainer.EdgeAgentOnKubernetesEnvironment:
-// 		cs := chisel.NewService(store, nil)
-// 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
-// 		edgeKey := cs.GenerateEdgeKey(URL, "", int(id))
-// 		expectedEndpoint.EdgeKey = edgeKey
-// 		store.testTunnelServer(t)
-
-// 	case portainer.KubernetesLocalEnvironment:
-// 		if URL == "" {
-// 			URL = kubernetesLocalEnvironmentUrl
-// 		}
-// 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
-// 	}
-
-// 	setEndpointAuthorizations(expectedEndpoint)
-// 	store.Endpoint().Create(expectedEndpoint)
-
-// 	endpoint, err := store.Endpoint().Endpoint(id)
-// 	is.NoError(err, "Endpoint() should not return an error")
-// 	is.Equal(expectedEndpoint, endpoint, "endpoint should be the same")
-
-// 	return endpoint.ID
-// }
-
-// func (store *Store) CreateEndpointRelation(id portainer.EndpointID) {
-// 	relation := &portainer.EndpointRelation{
-// 		EndpointID: id,
-// 		EdgeStacks: map[portainer.EdgeStackID]bool{},
-// 	}
-
-// 	store.EndpointRelation().Create(relation)
-// }
-
-// func (store *Store) testSSLSettings(t *testing.T) {
-// 	is := assert.New(t)
-// 	ssl := &portainer.SSLSettings{
-// 		CertPath:    "/data/certs/cert.pem",
-// 		HTTPEnabled: true,
-// 		KeyPath:     "/data/certs/key.pem",
-// 		SelfSigned:  true,
-// 	}
-
-// 	store.SSLSettings().UpdateSettings(ssl)
-
-// 	settings, err := store.SSLSettings().Settings()
-// 	is.NoError(err, "Get sslsettings should succeed")
-// 	is.Equal(ssl, settings, "Stored SSLSettings should be the same as what is read out")
-// }
-
-// func (store *Store) testTunnelServer(t *testing.T) {
-// 	is := assert.New(t)
-// 	expectPrivateKeySeed := uniuri.NewLen(16)
-
-// 	err := store.TunnelServer().UpdateInfo(&portainer.TunnelServerInfo{PrivateKeySeed: expectPrivateKeySeed})
-// 	is.NoError(err, "UpdateInfo should have succeeded")
-
-// 	serverInfo, err := store.TunnelServer().Info()
-// 	is.NoError(err, "Info should have succeeded")
-
-// 	is.Equal(expectPrivateKeySeed, serverInfo.PrivateKeySeed, "hashed passwords should not differ")
-// }
-
-// // add users, read them back and check the details are unchanged
-// func (store *Store) testUserAccounts(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	err := store.createAccount(adminUsername, adminPassword, portainer.AdministratorRole)
-// 	is.NoError(err, "CreateAccount should succeed")
-// 	store.checkAccount(adminUsername, adminPassword, portainer.AdministratorRole)
-// 	is.NoError(err, "Account failure")
-
-// 	err = store.createAccount(standardUsername, standardPassword, portainer.StandardUserRole)
-// 	is.NoError(err, "CreateAccount should succeed")
-// 	store.checkAccount(standardUsername, standardPassword, portainer.StandardUserRole)
-// 	is.NoError(err, "Account failure")
-// }
-
-// // create an account with the provided details
-// func (store *Store) createAccount(username, password string, role portainer.UserRole) error {
-// 	var err error
-// 	user := &portainer.User{Username: username, Role: role}
-
-// 	// encrypt the password
-// 	cs := &crypto.Service{}
-// 	user.Password, err = cs.Hash(password)
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	err = store.User().Create(user)
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	return nil
-// }
-
-// func (store *Store) checkAccount(username, expectPassword string, expectRole portainer.UserRole) error {
-// 	// Read the account for username.  Check password and role is what we expect
-
-// 	user, err := store.User().UserByUsername(username)
-// 	if err != nil {
-// 		return errors.Wrap(err, "failed to find user")
-// 	}
-
-// 	if user.Username != username || user.Role != expectRole {
-// 		return fmt.Errorf("%s user details do not match", user.Username)
-// 	}
-
-// 	// Check the password
-// 	cs := &crypto.Service{}
-// 	expectPasswordHash, err := cs.Hash(expectPassword)
-// 	if err != nil {
-// 		return errors.Wrap(err, "hash failed")
-// 	}
-
-// 	if user.Password != expectPasswordHash {
-// 		return fmt.Errorf("%s user password hash failure", user.Username)
-// 	}
-
-// 	return nil
-// }
-
-// func (store *Store) testSettings(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	// since many settings are default and basically nil, I'm going to update some and read them back
-// 	expectedSettings, err := store.Settings().Settings()
-// 	is.NoError(err, "Settings() should not return an error")
-// 	expectedSettings.TemplatesURL = "http://portainer.io/application-templates"
-// 	expectedSettings.HelmRepositoryURL = "http://portainer.io/helm-repository"
-// 	expectedSettings.EdgeAgentCheckinInterval = 60
-// 	expectedSettings.AuthenticationMethod = portainer.AuthenticationLDAP
-// 	expectedSettings.LDAPSettings = portainer.LDAPSettings{
-// 		AnonymousMode:   true,
-// 		StartTLS:        true,
-// 		AutoCreateUsers: true,
-// 		Password:        "random",
-// 	}
-// 	expectedSettings.SnapshotInterval = "10m"
-
-// 	err = store.Settings().UpdateSettings(expectedSettings)
-// 	is.NoError(err, "UpdateSettings() should succeed")
-
-// 	settings, err := store.Settings().Settings()
-// 	is.NoError(err, "Settings() should not return an error")
-// 	is.Equal(expectedSettings, settings, "stored settings should match")
-// }
-
-// func (store *Store) testCustomTemplates(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	customTemplate := store.CustomTemplate()
-// 	is.NotNil(customTemplate, "customTemplate Service shouldn't be nil")
-
-// 	expectedTemplate := &portainer.CustomTemplate{
-// 		ID:              portainer.CustomTemplateID(customTemplate.GetNextIdentifier()),
-// 		Title:           "Custom Title",
-// 		Description:     "Custom Template Description",
-// 		ProjectPath:     "/data/custom_template/1",
-// 		Note:            "A note about this custom template",
-// 		EntryPoint:      "docker-compose.yaml",
-// 		CreatedByUserID: 10,
-// 	}
-
-// 	customTemplate.Create(expectedTemplate)
-
-// 	actualTemplate, err := customTemplate.CustomTemplate(expectedTemplate.ID)
-// 	is.NoError(err, "CustomTemplate should not return an error")
-// 	is.Equal(expectedTemplate, actualTemplate, "expected and actual template do not match")
-// }
-
-// func (store *Store) testRegistries(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	regService := store.RegistryService
-// 	is.NotNil(regService, "RegistryService shouldn't be nil")
-
-// 	reg1 := &portainer.Registry{
-// 		ID:   1,
-// 		Type: portainer.DockerHubRegistry,
-// 		Name: "Dockerhub Registry Test",
-// 	}
-
-// 	reg2 := &portainer.Registry{
-// 		ID:   2,
-// 		Type: portainer.GitlabRegistry,
-// 		Name: "Gitlab Registry Test",
-// 		Gitlab: portainer.GitlabRegistryData{
-// 			ProjectID:   12345,
-// 			InstanceURL: "http://gitlab.com/12345",
-// 			ProjectPath: "mytestproject",
-// 		},
-// 	}
-
-// 	err := regService.Create(reg1)
-// 	is.NoError(err)
-
-// 	err = regService.Create(reg2)
-// 	is.NoError(err)
-
-// 	actualReg1, err := regService.Registry(reg1.ID)
-// 	is.NoError(err)
-// 	is.Equal(reg1, actualReg1, "registries differ")
-
-// 	actualReg2, err := regService.Registry(reg2.ID)
-// 	is.NoError(err)
-// 	is.Equal(reg2, actualReg2, "registries differ")
-// }
-
-// func (store *Store) testResourceControl(t *testing.T) {
-// 	// is := assert.New(t)
-// 	// resControl := store.ResourceControl()
-// 	// ctrl := &portainer.ResourceControl{
-// 	// }
-// 	// resControl().Create()
-// }
-
-// func (store *Store) testSchedules(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	schedule := store.ScheduleService
-// 	s := &portainer.Schedule{
-// 		ID:             portainer.ScheduleID(schedule.GetNextIdentifier()),
-// 		Name:           "My Custom Schedule 1",
-// 		CronExpression: "*/5 * * * * portainer /bin/sh -c echo 'hello world'",
-// 	}
-
-// 	err := schedule.CreateSchedule(s)
-// 	is.NoError(err, "CreateSchedule should succeed")
-
-// 	actual, err := schedule.Schedule(s.ID)
-// 	is.NoError(err, "schedule should be found")
-// 	is.Equal(s, actual, "schedules differ")
-// }
-
-// func (store *Store) testTags(t *testing.T) {
-// 	is := assert.New(t)
-
-// 	tags := store.TagService
-
-// 	tag1 := &portainer.Tag{
-// 		ID:   1,
-// 		Name: "Tag 1",
-// 	}
-
-// 	tag2 := &portainer.Tag{
-// 		ID:   2,
-// 		Name: "Tag 1",
-// 	}
-
-// 	err := tags.Create(tag1)
-// 	is.NoError(err, "Tags.Create should succeed")
-
-// 	err = tags.Create(tag2)
-// 	is.NoError(err, "Tags.Create should succeed")
-
-// 	actual, err := tags.Tag(tag1.ID)
-// 	is.NoError(err, "tag1 should be found")
-// 	is.Equal(tag1, actual, "tags differ")
-
-// 	actual, err = tags.Tag(tag2.ID)
-// 	is.NoError(err, "tag2 should be found")
-// 	is.Equal(tag2, actual, "tags differ")
-// }
+import (
+	"fmt"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/chisel"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	adminUsername                       = "admin"
+	adminPassword                       = "password"
+	standardUsername                    = "standard"
+	standardPassword                    = "password"
+	agentOnDockerEnvironmentUrl         = "tcp://192.168.167.207:30775"
+	edgeAgentOnKubernetesEnvironmentUrl = "tcp://192.168.167.207"
+	kubernetesLocalEnvironmentUrl       = "https://kubernetes.default.svc"
+)
+
+// TestStoreFull an eventually comprehensive set of tests for the Store.
+// The idea is what we write to the store, we should read back.
+func TestStoreFull(t *testing.T) {
+	_, store := MustNewTestStore(t, true, true)
+
+	testCases := map[string]func(t *testing.T){
+		"User Accounts": func(t *testing.T) {
+			store.testUserAccounts(t)
+		},
+		"Environments": func(t *testing.T) {
+			store.testEnvironments(t)
+		},
+		"Settings": func(t *testing.T) {
+			store.testSettings(t)
+		},
+		"SSL Settings": func(t *testing.T) {
+			store.testSSLSettings(t)
+		},
+		"Tunnel Server": func(t *testing.T) {
+			store.testTunnelServer(t)
+		},
+		"Custom Templates": func(t *testing.T) {
+			store.testCustomTemplates(t)
+		},
+		"Registries": func(t *testing.T) {
+			store.testRegistries(t)
+		},
+		"Resource Control": func(t *testing.T) {
+			store.testResourceControl(t)
+		},
+		"Schedules": func(t *testing.T) {
+			store.testSchedules(t)
+		},
+		"Tags": func(t *testing.T) {
+			store.testTags(t)
+		},
+
+		// "Test Title": func(t *testing.T) {
+		// },
+	}
+
+	for name, test := range testCases {
+		t.Run(name, test)
+	}
+
+}
+
+func (store *Store) testEnvironments(t *testing.T) {
+	id := store.CreateEndpoint(t, "local", portainer.KubernetesLocalEnvironment, "", true)
+	store.CreateEndpointRelation(id)
+
+	id = store.CreateEndpoint(t, "agent", portainer.AgentOnDockerEnvironment, agentOnDockerEnvironmentUrl, true)
+	store.CreateEndpointRelation(id)
+
+	id = store.CreateEndpoint(t, "edge", portainer.EdgeAgentOnKubernetesEnvironment, edgeAgentOnKubernetesEnvironmentUrl, true)
+	store.CreateEndpointRelation(id)
+}
+
+func newEndpoint(endpointType portainer.EndpointType, id portainer.EndpointID, name, URL string, TLS bool) *portainer.Endpoint {
+	endpoint := &portainer.Endpoint{
+		ID:        id,
+		Name:      name,
+		URL:       URL,
+		Type:      endpointType,
+		GroupID:   portainer.EndpointGroupID(1),
+		PublicURL: "",
+		TLSConfig: portainer.TLSConfiguration{
+			TLS: false,
+		},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             []portainer.TagID{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+	}
+
+	if TLS {
+		endpoint.TLSConfig = portainer.TLSConfiguration{
+			TLS:           true,
+			TLSSkipVerify: true,
+		}
+	}
+
+	return endpoint
+}
+
+func setEndpointAuthorizations(endpoint *portainer.Endpoint) {
+	endpoint.SecuritySettings = portainer.EndpointSecuritySettings{
+		AllowVolumeBrowserForRegularUsers: false,
+		EnableHostManagementFeatures:      false,
+
+		AllowSysctlSettingForRegularUsers:         true,
+		AllowBindMountsForRegularUsers:            true,
+		AllowPrivilegedModeForRegularUsers:        true,
+		AllowHostNamespaceForRegularUsers:         true,
+		AllowContainerCapabilitiesForRegularUsers: true,
+		AllowDeviceMappingForRegularUsers:         true,
+		AllowStackManagementForRegularUsers:       true,
+	}
+}
+
+func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType portainer.EndpointType, URL string, tls bool) portainer.EndpointID {
+	is := assert.New(t)
+
+	var expectedEndpoint *portainer.Endpoint
+	id := portainer.EndpointID(store.Endpoint().GetNextIdentifier())
+
+	switch endpointType {
+	case portainer.DockerEnvironment:
+		if URL == "" {
+			URL = "unix:///var/run/docker.sock"
+			if runtime.GOOS == "windows" {
+				URL = "npipe:////./pipe/docker_engine"
+			}
+		}
+		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
+
+	case portainer.AgentOnDockerEnvironment:
+		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
+
+	case portainer.AgentOnKubernetesEnvironment:
+		URL = strings.TrimPrefix(URL, "tcp://")
+		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
+
+	case portainer.EdgeAgentOnKubernetesEnvironment:
+		cs := chisel.NewService(store, nil, nil)
+		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
+		edgeKey := cs.GenerateEdgeKey(URL, "", int(id))
+		expectedEndpoint.EdgeKey = edgeKey
+		store.testTunnelServer(t)
+
+	case portainer.KubernetesLocalEnvironment:
+		if URL == "" {
+			URL = kubernetesLocalEnvironmentUrl
+		}
+		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
+	}
+
+	setEndpointAuthorizations(expectedEndpoint)
+	store.Endpoint().Create(expectedEndpoint)
+
+	endpoint, err := store.Endpoint().Endpoint(id)
+	is.NoError(err, "Endpoint() should not return an error")
+	is.Equal(expectedEndpoint, endpoint, "endpoint should be the same")
+
+	return endpoint.ID
+}
+
+func (store *Store) CreateEndpointRelation(id portainer.EndpointID) {
+	relation := &portainer.EndpointRelation{
+		EndpointID: id,
+		EdgeStacks: map[portainer.EdgeStackID]bool{},
+	}
+
+	store.EndpointRelation().Create(relation)
+}
+
+func (store *Store) testSSLSettings(t *testing.T) {
+	is := assert.New(t)
+	ssl := &portainer.SSLSettings{
+		CertPath:    "/data/certs/cert.pem",
+		HTTPEnabled: true,
+		KeyPath:     "/data/certs/key.pem",
+		SelfSigned:  true,
+	}
+
+	store.SSLSettings().UpdateSettings(ssl)
+
+	settings, err := store.SSLSettings().Settings()
+	is.NoError(err, "Get sslsettings should succeed")
+	is.Equal(ssl, settings, "Stored SSLSettings should be the same as what is read out")
+}
+
+func (store *Store) testTunnelServer(t *testing.T) {
+	is := assert.New(t)
+	expectPrivateKeySeed := uniuri.NewLen(16)
+
+	err := store.TunnelServer().UpdateInfo(&portainer.TunnelServerInfo{PrivateKeySeed: expectPrivateKeySeed})
+	is.NoError(err, "UpdateInfo should have succeeded")
+
+	serverInfo, err := store.TunnelServer().Info()
+	is.NoError(err, "Info should have succeeded")
+
+	is.Equal(expectPrivateKeySeed, serverInfo.PrivateKeySeed, "hashed passwords should not differ")
+}
+
+// add users, read them back and check the details are unchanged
+func (store *Store) testUserAccounts(t *testing.T) {
+	is := assert.New(t)
+
+	err := store.createAccount(adminUsername, adminPassword, portainer.AdministratorRole)
+	is.NoError(err, "CreateAccount should succeed")
+	store.checkAccount(adminUsername, adminPassword, portainer.AdministratorRole)
+	is.NoError(err, "Account failure")
+
+	err = store.createAccount(standardUsername, standardPassword, portainer.StandardUserRole)
+	is.NoError(err, "CreateAccount should succeed")
+	store.checkAccount(standardUsername, standardPassword, portainer.StandardUserRole)
+	is.NoError(err, "Account failure")
+}
+
+// create an account with the provided details
+func (store *Store) createAccount(username, password string, role portainer.UserRole) error {
+	var err error
+	user := &portainer.User{Username: username, Role: role}
+
+	// encrypt the password
+	cs := &crypto.Service{}
+	user.Password, err = cs.Hash(password)
+	if err != nil {
+		return err
+	}
+
+	err = store.User().Create(user)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *Store) checkAccount(username, expectPassword string, expectRole portainer.UserRole) error {
+	// Read the account for username.  Check password and role is what we expect
+
+	user, err := store.User().UserByUsername(username)
+	if err != nil {
+		return errors.Wrap(err, "failed to find user")
+	}
+
+	if user.Username != username || user.Role != expectRole {
+		return fmt.Errorf("%s user details do not match", user.Username)
+	}
+
+	// Check the password
+	cs := &crypto.Service{}
+	expectPasswordHash, err := cs.Hash(expectPassword)
+	if err != nil {
+		return errors.Wrap(err, "hash failed")
+	}
+
+	if user.Password != expectPasswordHash {
+		return fmt.Errorf("%s user password hash failure", user.Username)
+	}
+
+	return nil
+}
+
+func (store *Store) testSettings(t *testing.T) {
+	is := assert.New(t)
+
+	// since many settings are default and basically nil, I'm going to update some and read them back
+	expectedSettings, err := store.Settings().Settings()
+	is.NoError(err, "Settings() should not return an error")
+	expectedSettings.TemplatesURL = "http://portainer.io/application-templates"
+	expectedSettings.HelmRepositoryURL = "http://portainer.io/helm-repository"
+	expectedSettings.EdgeAgentCheckinInterval = 60
+	expectedSettings.AuthenticationMethod = portainer.AuthenticationLDAP
+	expectedSettings.LDAPSettings = portainer.LDAPSettings{
+		AnonymousMode:   true,
+		StartTLS:        true,
+		AutoCreateUsers: true,
+		Password:        "random",
+	}
+	expectedSettings.SnapshotInterval = "10m"
+
+	err = store.Settings().UpdateSettings(expectedSettings)
+	is.NoError(err, "UpdateSettings() should succeed")
+
+	settings, err := store.Settings().Settings()
+	is.NoError(err, "Settings() should not return an error")
+	is.Equal(expectedSettings, settings, "stored settings should match")
+}
+
+func (store *Store) testCustomTemplates(t *testing.T) {
+	is := assert.New(t)
+
+	customTemplate := store.CustomTemplate()
+	is.NotNil(customTemplate, "customTemplate Service shouldn't be nil")
+
+	expectedTemplate := &portainer.CustomTemplate{
+		ID:              portainer.CustomTemplateID(customTemplate.GetNextIdentifier()),
+		Title:           "Custom Title",
+		Description:     "Custom Template Description",
+		ProjectPath:     "/data/custom_template/1",
+		Note:            "A note about this custom template",
+		EntryPoint:      "docker-compose.yaml",
+		CreatedByUserID: 10,
+	}
+
+	customTemplate.Create(expectedTemplate)
+
+	actualTemplate, err := customTemplate.Read(expectedTemplate.ID)
+	is.NoError(err, "CustomTemplate should not return an error")
+	is.Equal(expectedTemplate, actualTemplate, "expected and actual template do not match")
+}
+
+func (store *Store) testRegistries(t *testing.T) {
+	is := assert.New(t)
+
+	regService := store.RegistryService
+	is.NotNil(regService, "RegistryService shouldn't be nil")
+
+	reg1 := &portainer.Registry{
+		ID:   1,
+		Type: portainer.DockerHubRegistry,
+		Name: "Dockerhub Registry Test",
+	}
+
+	reg2 := &portainer.Registry{
+		ID:   2,
+		Type: portainer.GitlabRegistry,
+		Name: "Gitlab Registry Test",
+		Gitlab: portainer.GitlabRegistryData{
+			ProjectID:   12345,
+			InstanceURL: "http://gitlab.com/12345",
+			ProjectPath: "mytestproject",
+		},
+	}
+
+	err := regService.Create(reg1)
+	is.NoError(err)
+
+	err = regService.Create(reg2)
+	is.NoError(err)
+
+	actualReg1, err := regService.Read(reg1.ID)
+	is.NoError(err)
+	is.Equal(reg1, actualReg1, "registries differ")
+
+	actualReg2, err := regService.Read(reg2.ID)
+	is.NoError(err)
+	is.Equal(reg2, actualReg2, "registries differ")
+}
+
+func (store *Store) testResourceControl(t *testing.T) {
+	// is := assert.New(t)
+	// resControl := store.ResourceControl()
+	// ctrl := &portainer.ResourceControl{
+	// }
+	// resControl().Create()
+}
+
+func (store *Store) testSchedules(t *testing.T) {
+	is := assert.New(t)
+
+	schedule := store.ScheduleService
+	s := &portainer.Schedule{
+		ID:             portainer.ScheduleID(schedule.GetNextIdentifier()),
+		Name:           "My Custom Schedule 1",
+		CronExpression: "*/5 * * * * portainer /bin/sh -c echo 'hello world'",
+	}
+
+	err := schedule.CreateSchedule(s)
+	is.NoError(err, "CreateSchedule should succeed")
+
+	actual, err := schedule.Schedule(s.ID)
+	is.NoError(err, "schedule should be found")
+	is.Equal(s, actual, "schedules differ")
+}
+
+func (store *Store) testTags(t *testing.T) {
+	is := assert.New(t)
+
+	tags := store.TagService
+
+	tag1 := &portainer.Tag{
+		ID:   1,
+		Name: "Tag 1",
+	}
+
+	tag2 := &portainer.Tag{
+		ID:   2,
+		Name: "Tag 1",
+	}
+
+	err := tags.Create(tag1)
+	is.NoError(err, "Tags.Create should succeed")
+
+	err = tags.Create(tag2)
+	is.NoError(err, "Tags.Create should succeed")
+
+	actual, err := tags.Read(tag1.ID)
+	is.NoError(err, "tag1 should be found")
+	is.Equal(tag1, actual, "tags differ")
+
+	actual, err = tags.Read(tag2.ID)
+	is.NoError(err, "tag2 should be found")
+	is.Equal(tag2, actual, "tags differ")
+}