close db before restore. fix log

* Revert "fix(rollback): reversed rollback code from 2.19.4 [EE-6435] (#10787)"
* close the db before backup for windows shares and better error handling (#10809)

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Portainer Business Edition - Get 5 nodes free
-    url: https://portainer.io/pricing/take5 
-    about: Portainer Business Edition has more features, more support and you can now get 5 nodes free for as long as you want.
+  - name: Portainer Business Edition - Get 3 nodes free
+    url: https://www.portainer.io/take-3 
+    about: Portainer Business Edition has more features, more support and you can now get 3 nodes free for as long as you want.

.github/workflows/lint.yml

@@ -41,6 +41,6 @@ jobs:
       - name: GolangCI-Lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: latest
+          version: v1.54.1
           working-directory: api
           args: --timeout=10m -c .golangci.yaml

.github/workflows/test.yaml

@@ -14,16 +14,12 @@ jobs:
 
       - name: Run tests
         run: yarn jest --maxWorkers=2
-  # test-server:
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     GOPRIVATE: "github.com/portainer"
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - uses: actions/setup-go@v3
-  #       with:
-  #         go-version: '1.18'
-  #     - name: Run tests
-  #       run: |
-  #         cd api
-  #         go test ./...
+  test-server:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.5
+      - name: Run tests
+        run: make test-server

.github/workflows/validate-openapi-spec.yaml

@@ -21,7 +21,7 @@ jobs:
         run: cd ./api && go get -t -v -d ./...
       - uses: actions/setup-node@v3
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
       - run: yarn --frozen-lockfile
 

.gitignore

@@ -11,7 +11,7 @@ storybook-static
 *.DS_Store
 
 .eslintcache
-__debug_bin
+__debug_bin*
 
 api/docs
 .idea

.storybook/main.js

@@ -1,58 +0,0 @@
-const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
-
-module.exports = {
-  stories: ['../app/**/*.stories.mdx', '../app/**/*.stories.@(ts|tsx)'],
-  addons: [
-    '@storybook/addon-links',
-    '@storybook/addon-essentials',
-    {
-      name: '@storybook/addon-postcss',
-      options: {
-        cssLoaderOptions: {
-          importLoaders: 1,
-          modules: {
-            localIdentName: '[path][name]__[local]',
-            auto: true,
-            exportLocalsConvention: 'camelCaseOnly',
-          },
-        },
-        postcssLoaderOptions: {
-          implementation: require('postcss'),
-        },
-      },
-    },
-  ],
-  webpackFinal: (config) => {
-    config.resolve.plugins = [
-      ...(config.resolve.plugins || []),
-      new TsconfigPathsPlugin({
-        extensions: config.resolve.extensions,
-      }),
-    ];
-
-    const svgRule = config.module.rules.find((rule) => rule.test && typeof rule.test.test === 'function' && rule.test.test('.svg'));
-    svgRule.test = new RegExp(svgRule.test.source.replace('svg|', ''));
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      type: 'asset',
-      resourceQuery: { not: [/c/] }, // exclude react component if *.svg?url
-    });
-
-    config.module.rules.unshift({
-      test: /\.svg$/i,
-      issuer: /\.(js|ts)(x)?$/,
-      resourceQuery: /c/, // *.svg?c
-      use: [{ loader: '@svgr/webpack', options: { icon: true } }],
-    });
-
-    return config;
-  },
-  core: {
-    builder: 'webpack5',
-  },
-  staticDirs: ['./public'],
-  typescript: {
-    reactDocgen: 'react-docgen-typescript-plugin',
-  },
-};

.storybook/main.ts

@@ -0,0 +1,95 @@
+import { StorybookConfig } from '@storybook/react-webpack5';
+
+import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
+import { Configuration } from 'webpack';
+import postcss from 'postcss';
+const config: StorybookConfig = {
+  stories: ['../app/**/*.stories.@(ts|tsx)'],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    {
+      name: '@storybook/addon-styling',
+      options: {
+        cssLoaderOptions: {
+          importLoaders: 1,
+          modules: {
+            localIdentName: '[path][name]__[local]',
+            auto: true,
+            exportLocalsConvention: 'camelCaseOnly',
+          },
+        },
+        postCss: {
+          implementation: postcss,
+        },
+      },
+    },
+  ],
+  webpackFinal: (config) => {
+    const rules = config?.module?.rules || [];
+
+    const imageRule = rules.find((rule) => {
+      const test = (rule as { test: RegExp }).test;
+
+      if (!test) {
+        return false;
+      }
+
+      return test.test('.svg');
+    }) as { [key: string]: any };
+
+    imageRule.exclude = /\.svg$/;
+
+    rules.unshift({
+      test: /\.svg$/i,
+      type: 'asset',
+      resourceQuery: {
+        not: [/c/],
+      }, // exclude react component if *.svg?url
+    });
+
+    rules.unshift({
+      test: /\.svg$/i,
+      issuer: /\.(js|ts)(x)?$/,
+      resourceQuery: /c/,
+      // *.svg?c
+      use: [
+        {
+          loader: '@svgr/webpack',
+          options: {
+            icon: true,
+          },
+        },
+      ],
+    });
+    return {
+      ...config,
+      resolve: {
+        ...config.resolve,
+        plugins: [
+          ...(config.resolve?.plugins || []),
+          new TsconfigPathsPlugin({
+            extensions: config.resolve?.extensions,
+          }),
+        ],
+      },
+      module: {
+        ...config.module,
+        rules,
+      },
+    } satisfies Configuration;
+  },
+  staticDirs: ['./public'],
+  typescript: {
+    reactDocgen: 'react-docgen-typescript',
+  },
+  framework: {
+    name: '@storybook/react-webpack5',
+    options: {},
+  },
+  docs: {
+    autodocs: true,
+  },
+};
+
+export default config;

README.md

@@ -9,7 +9,7 @@ Portainer consists of a single container that can run on any cluster. It can be
 **Portainer Business Edition** builds on the open-source base and includes a range of advanced features and functions (like RBAC and Support) that are specific to the needs of business users.
 
 - [Compare Portainer CE and Compare Portainer BE](https://portainer.io/products)
-- [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
+- [Take3 – get 3 free nodes of Portainer Business for as long as you want them](https://www.portainer.io/take-3)
 - [Portainer BE install guide](https://install.portainer.io)
 
 ## Latest Version
@@ -21,8 +21,8 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
 ## Getting started
 
 - [Deploy Portainer](https://docs.portainer.io/start/install)
-- [Documentation](https://documentation.portainer.io)
-- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
+- [Documentation](https://docs.portainer.io)
+- [Contribute to the project](https://docs.portainer.io/contribute/contribute)
 
 ## Features & Functions
 
@@ -30,23 +30,22 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 - [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
 - [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
-- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
 
 ## Getting help
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
+Learn more about Portainer's community support channels [here.](https://www.portainer.io/get-support-for-portainer)
 
 - Issues: https://github.com/portainer/portainer/issues
 - Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
 
-You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
+You can join the Portainer Community by visiting [https://www.portainer.io/join-our-community](https://www.portainer.io/join-our-community). This will give you advance notice of events, content and other related Portainer content.
 
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
-- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://documentation.portainer.io/contributing/instructions/) to build it locally and make a pull request.
+- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
 
 ## Security
 
@@ -60,7 +59,7 @@ If you are a developer, and our code in this repo makes sense to you, we would l
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
 
-When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/documentation/in-app-analytics-and-privacy-policy/). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
+When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
 
 ## Limitations
 

api/.golangci.yaml

@@ -10,15 +10,17 @@ linters:
     - exportloopref
 linters-settings:
   depguard:
-    list-type: denylist
-    include-go-root: true
-    packages:
-      - github.com/sirupsen/logrus
-      - golang.org/x/exp
-    packages-with-error-message:
-      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
-    ignore-file-rules:
-      - '**/*_test.go'
+    rules:
+      main:
+        deny:
+          - pkg: 'github.com/sirupsen/logrus'
+            desc: 'logging is allowed only by github.com/rs/zerolog'
+          - pkg: 'golang.org/x/exp'
+            desc: 'exp is not allowed'
+        files:
+          - '!**/*_test.go'
+          - '!**/base.go'
+          - '!**/base_tx.go'
 
 # errorlint is causing a typecheck error for some reason. The go compiler will report these
 # anyway, so ignore them from the linter

api/apikey/apikey.go

@@ -1,9 +1,6 @@
 package apikey
 
 import (
-	"crypto/rand"
-	"io"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -18,13 +15,3 @@ type APIKeyService interface {
 	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
 	InvalidateUserKeyCache(userId portainer.UserID) bool
 }
-
-// generateRandomKey generates a random key of specified length
-// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
-func generateRandomKey(length int) []byte {
-	k := make([]byte, length)
-	if _, err := io.ReadFull(rand.Reader, k); err != nil {
-		return nil
-	}
-	return k
-}

api/apikey/apikey_test.go

@@ -3,6 +3,7 @@ package apikey
 import (
 	"testing"
 
+	"github.com/portainer/portainer/api/internal/securecookie"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -33,7 +34,7 @@ func Test_generateRandomKey(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := generateRandomKey(tt.wantLenth)
+			got := securecookie.GenerateRandomKey(tt.wantLenth)
 			is.Equal(tt.wantLenth, len(got))
 		})
 	}
@@ -41,7 +42,7 @@ func Test_generateRandomKey(t *testing.T) {
 	t.Run("Generated keys are unique", func(t *testing.T) {
 		keys := make(map[string]bool)
 		for i := 0; i < 100; i++ {
-			key := generateRandomKey(8)
+			key := securecookie.GenerateRandomKey(8)
 			_, ok := keys[string(key)]
 			is.False(ok)
 			keys[string(key)] = true

api/apikey/service.go

@@ -8,6 +8,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/securecookie"
 
 	"github.com/pkg/errors"
 )
@@ -39,7 +40,7 @@ func (a *apiKeyService) HashRaw(rawKey string) []byte {
 // GenerateApiKey generates a raw API key for a user (for one-time display).
 // The generated API key is stored in the cache and database.
 func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
-	randKey := generateRandomKey(32)
+	randKey := securecookie.GenerateRandomKey(32)
 	encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
 	prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
 
@@ -53,7 +54,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 		Digest:      hashDigest,
 	}
 
-	err := a.apiKeyRepository.CreateAPIKey(apiKey)
+	err := a.apiKeyRepository.Create(apiKey)
 	if err != nil {
 		return "", nil, errors.Wrap(err, "Unable to create API key")
 	}
@@ -66,7 +67,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
 
 // GetAPIKey returns an API key by its ID.
 func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	return a.apiKeyRepository.GetAPIKey(apiKeyID)
+	return a.apiKeyRepository.Read(apiKeyID)
 }
 
 // GetAPIKeys returns all the API keys associated to a user.
@@ -88,7 +89,7 @@ func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, port
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve API key")
 	}
 
-	user, err := a.userRepository.User(apiKey.UserID)
+	user, err := a.userRepository.Read(apiKey.UserID)
 	if err != nil {
 		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve digest user")
 	}
@@ -106,20 +107,20 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
 		return errors.Wrap(err, "Unable to retrieve API key")
 	}
 	a.cache.Set(apiKey.Digest, user, *apiKey)
-	return a.apiKeyRepository.UpdateAPIKey(apiKey)
+	return a.apiKeyRepository.Update(apiKey.ID, apiKey)
 }
 
 // DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
 func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
 	// get api-key digest to remove from cache
-	apiKey, err := a.apiKeyRepository.GetAPIKey(apiKeyID)
+	apiKey, err := a.apiKeyRepository.Read(apiKeyID)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
 	}
 
 	// delete the user/api-key from cache
 	a.cache.Delete(apiKey.Digest)
-	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
+	return a.apiKeyRepository.Delete(apiKeyID)
 }
 
 func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {

api/backup/backup.go

@@ -30,6 +30,7 @@ var filesToBackup = []string{
 	"portainer.key",
 	"portainer.pub",
 	"tls",
+	"chisel",
 }
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.

api/chisel/crypto/crypto.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+
+	chshare "github.com/jpillora/chisel/share"
+)
+
+var one = new(big.Int).SetInt64(1)
+
+// GenerateGo119CompatibleKey This function is basically copied from chshare.GenerateKey.
+func GenerateGo119CompatibleKey(seed string) ([]byte, error) {
+	r := chshare.NewDetermRand([]byte(seed))
+	priv, err := ecdsaGenerateKey(elliptic.P256(), r)
+	if err != nil {
+		return nil, err
+	}
+	b, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to marshal ECDSA private key: %w", err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil
+}
+
+// This function is copied from Go1.19
+func randFieldElement(c elliptic.Curve, rand io.Reader) (k *big.Int, err error) {
+	params := c.Params()
+	// Note that for P-521 this will actually be 63 bits more than the order, as
+	// division rounds down, but the extra bit is inconsequential.
+	b := make([]byte, params.N.BitLen()/8+8)
+	_, err = io.ReadFull(rand, b)
+	if err != nil {
+		return
+	}
+
+	k = new(big.Int).SetBytes(b)
+	n := new(big.Int).Sub(params.N, one)
+	k.Mod(k, n)
+	k.Add(k, one)
+	return
+}
+
+// This function is copied from Go1.19
+func ecdsaGenerateKey(c elliptic.Curve, rand io.Reader) (*ecdsa.PrivateKey, error) {
+	k, err := randFieldElement(c, rand)
+	if err != nil {
+		return nil, err
+	}
+
+	priv := new(ecdsa.PrivateKey)
+	priv.PublicKey.Curve = c
+	priv.D = k
+	priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
+	return priv, nil
+}

api/chisel/crypto/crypto_test.go

@@ -0,0 +1,37 @@
+package crypto
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGenerateGo119CompatibleKey(t *testing.T) {
+	type args struct {
+		seed string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name:    "Generate Go 1.19 compatible private key with a given seed",
+			args:    args{seed: "94qh17MCIk8BOkiI"},
+			want:    []byte("-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIHeohwk0Gy3RHVVViaHz7pz/HOiqA7fkv1FTM3mGgfT3oAoGCCqGSM49\nAwEHoUQDQgAEN7riX06xDsLNPuUmOvYFluNEakcFwZZRVvOcIYk/9VYnanDzW0Km\n8/BUUiKyJDuuGdS4fj9SlQ4iL8yBK01uKg==\n-----END EC PRIVATE KEY-----\n"),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GenerateGo119CompatibleKey(tt.args.seed)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GenerateGo119CompatibleKey() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GenerateGo119CompatibleKey()\ngot: Z %v\nwant: %v", got, tt.want)
+			}
+		})
+	}
+}

api/chisel/schedules.go

@@ -18,6 +18,7 @@ func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portai
 	for idx, existingJob := range tunnel.Jobs {
 		if existingJob.ID == edgeJob.ID {
 			existingJobIndex = idx
+
 			break
 		}
 	}

api/chisel/service.go

@@ -12,8 +12,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 
-	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
+	"github.com/jpillora/chisel/share/ccrypto"
 	"github.com/rs/zerolog/log"
 )
 
@@ -36,14 +36,16 @@ type Service struct {
 	shutdownCtx       context.Context
 	ProxyManager      *proxy.Manager
 	mu                sync.Mutex
+	fileService       portainer.FileService
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
+func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
 	return &Service{
 		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
 		dataStore:        dataStore,
 		shutdownCtx:      shutdownCtx,
+		fileService:      fileService,
 	}
 }
 
@@ -73,10 +75,11 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 		log.Debug().
 			Int("endpoint_id", int(endpointID)).
 			Float64("max_alive_minutes", maxAlive.Minutes()).
-			Msg("start")
+			Msg("KeepTunnelAlive: start")
 
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
+
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
 		defer pingTicker.Stop()
 
@@ -89,13 +92,13 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 					log.Debug().
 						Int("endpoint_id", int(endpointID)).
 						Err(err).
-						Msg("ping agent")
+						Msg("KeepTunnelAlive: ping agent")
 				}
 			case <-maxAliveTicker.C:
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Float64("timeout_minutes", maxAlive.Minutes()).
-					Msg("tunnel keep alive timeout")
+					Msg("KeepTunnelAlive: tunnel keep alive timeout")
 
 				return
 			case <-ctx.Done():
@@ -103,7 +106,7 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				log.Debug().
 					Int("endpoint_id", int(endpointID)).
 					Err(err).
-					Msg("tunnel stop")
+					Msg("KeepTunnelAlive: tunnel stop")
 
 				return
 			}
@@ -117,14 +120,15 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 // It starts the tunnel status verification process in the background.
 // The snapshotter is used in the tunnel status verification process.
 func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
-	keySeed, err := service.retrievePrivateKeySeed()
+	privateKeyFile, err := service.retrievePrivateKeyFile()
+
 	if err != nil {
 		return err
 	}
 
 	config := &chserver.Config{
-		Reverse: true,
-		KeySeed: keySeed,
+		Reverse:        true,
+		PrivateKeyFile: privateKeyFile,
 	}
 
 	chiselServer, err := chserver.NewServer(config)
@@ -160,26 +164,41 @@ func (service *Service) StopTunnelServer() error {
 	return service.chiselServer.Close()
 }
 
-func (service *Service) retrievePrivateKeySeed() (string, error) {
-	var serverInfo *portainer.TunnelServerInfo
+func (service *Service) retrievePrivateKeyFile() (string, error) {
+	privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
 
-	serverInfo, err := service.dataStore.TunnelServer().Info()
-	if service.dataStore.IsErrObjectNotFound(err) {
-		keySeed := uniuri.NewLen(16)
+	exist, _ := service.fileService.FileExists(privateKeyFile)
+	if !exist {
+		log.Debug().
+			Str("private-key", privateKeyFile).
+			Msg("Chisel private key file does not exist")
 
-		serverInfo = &portainer.TunnelServerInfo{
-			PrivateKeySeed: keySeed,
-		}
-
-		err := service.dataStore.TunnelServer().UpdateInfo(serverInfo)
+		privateKey, err := ccrypto.GenerateKey("")
 		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to generate chisel private key")
 			return "", err
 		}
-	} else if err != nil {
-		return "", err
+
+		err = service.fileService.StoreChiselPrivateKey(privateKey)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to save Chisel private key to disk")
+			return "", err
+		} else {
+			log.Info().
+				Str("private-key", privateKeyFile).
+				Msg("Generated a new Chisel private key file")
+		}
+	} else {
+		log.Info().
+			Str("private-key", privateKeyFile).
+			Msg("Found Chisel private key file on disk")
 	}
 
-	return serverInfo.PrivateKeySeed, nil
+	return privateKeyFile, nil
 }
 
 func (service *Service) startTunnelVerificationLoop() {
@@ -271,14 +290,7 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 		return err
 	}
 
-	endpointURL := endpoint.URL
-
 	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
-	err = service.snapshotService.SnapshotEndpoint(endpoint)
-	if err != nil {
-		return err
-	}
 
-	endpoint.URL = endpointURL
-	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
+	return service.snapshotService.SnapshotEndpoint(endpoint)
 }

api/chisel/tunnel.go

@@ -126,7 +126,10 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 	credentials := tunnel.Credentials
 	if credentials != "" {
 		tunnel.Credentials = ""
-		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
+
+		if service.chiselServer != nil {
+			service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
+		}
 	}
 
 	service.ProxyManager.DeleteEndpointProxy(endpointID)
@@ -161,9 +164,12 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
 
 		username, password := generateRandomCredentials()
 		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-		err = service.chiselServer.AddUser(username, password, authorizedRemote)
-		if err != nil {
-			return err
+
+		if service.chiselServer != nil {
+			err = service.chiselServer.AddUser(username, password, authorizedRemote)
+			if err != nil {
+				return err
+			}
 		}
 
 		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)

api/cli/cli.go

@@ -49,7 +49,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),

api/cli/confirm.go

@@ -9,7 +9,7 @@ import (
 
 // Confirm starts a rollback db cli application
 func Confirm(message string) (bool, error) {
-	fmt.Printf("%s [y/N]", message)
+	fmt.Printf("%s [y/N] ", message)
 
 	reader := bufio.NewReader(os.Stdin)
 

api/cmd/portainer/main.go

@@ -9,8 +9,6 @@ import (
 	"strings"
 	"time"
 
-	libstack "github.com/portainer/docker-compose-wrapper"
-	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
 	"github.com/portainer/portainer/api/build"
@@ -22,6 +20,7 @@ import (
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
 	dockerclient "github.com/portainer/portainer/api/docker/client"
@@ -48,6 +47,8 @@ import (
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/pkg/featureflags"
 	"github.com/portainer/portainer/pkg/libhelm"
+	"github.com/portainer/portainer/pkg/libstack"
+	"github.com/portainer/portainer/pkg/libstack/compose"
 
 	"github.com/gofrs/uuid"
 	"github.com/rs/zerolog/log"
@@ -119,11 +120,15 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 			log.Fatal().Err(err).Msg("failed generating instance id")
 		}
 
+		migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{})
+		migratorCount := migratorInstance.GetMigratorCountOfCurrentAPIVersion()
+
 		// from MigrateData
 		v := models.Version{
 			SchemaVersion: portainer.APIVersion,
 			Edition:       int(portainer.PortainerCE),
 			InstanceID:    instanceId.String(),
+			MigratorCount: migratorCount,
 		}
 		store.VersionService.UpdateVersion(&v)
 
@@ -152,7 +157,17 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 	return store
 }
 
-func initComposeStackManager(composeDeployer libstack.Deployer, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+// checkDBSchemaServerVersionMatch checks if the server version matches the db scehma version
+func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersion string, serverEdition int) bool {
+	v, err := dbStore.Version().Version()
+	if err != nil {
+		return false
+	}
+
+	return v.SchemaVersion == serverVersion && v.Edition == serverEdition
+}
+
+func initComposeStackManager(composeDeployer libstack.Deployer, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed creating compose manager")
@@ -383,6 +398,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("")
 	}
 
+	// check if the db schema version matches with server version
+	if !checkDBSchemaServerVersionMatch(dataStore, portainer.APIVersion, int(portainer.Edition)) {
+		log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
+	}
+
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed getting instance id")
@@ -429,7 +449,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
@@ -458,7 +478,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing compose deployer")
 	}
 
-	composeStackManager := initComposeStackManager(composeDeployer, reverseTunnelService, proxyManager)
+	composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {

api/database/boltdb/db.go

@@ -144,6 +144,8 @@ func (connection *DbConnection) Open() error {
 // Close closes the BoltDB database.
 // Safe to being called multiple times.
 func (connection *DbConnection) Close() error {
+	log.Info().Msg("closing PortainerDB")
+
 	if connection.DB != nil {
 		return connection.DB.Close()
 	}

api/database/boltdb/tx.go

@@ -28,10 +28,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
 		return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
 	}
 
-	data := make([]byte, len(value))
-	copy(data, value)
-
-	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
+	return tx.conn.UnmarshalObjectWithJsoniter(value, object)
 }
 
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
@@ -49,7 +46,9 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
 	return bucket.Delete(key)
 }
 
-func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
+	var ids []int
+
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
 	cursor := bucket.Cursor()
@@ -59,11 +58,14 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
 			return err
 		}
 
-		if id, ok := matching(obj); ok {
-			err := bucket.Delete(tx.conn.ConvertToKey(id))
-			if err != nil {
-				return err
-			}
+		if id, ok := matchingFn(obj); ok {
+			ids = append(ids, id)
+		}
+	}
+
+	for _, id := range ids {
+		if err := bucket.Delete(tx.conn.ConvertToKey(id)); err != nil {
+			return err
 		}
 	}
 
@@ -115,45 +117,33 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
 	return bucket.Put(id, data)
 }
 
-func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObject(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	bucket := tx.tx.Bucket([]byte(bucketName))
 
-	cursor := bucket.Cursor()
-	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+	return bucket.ForEach(func(k []byte, v []byte) error {
 		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
-		if err != nil {
-			return err
+		if err == nil {
+			obj, err = appendFn(obj)
 		}
 
-		obj, err = append(obj)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+		return err
+	})
 }
 
-func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
 	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
 
 	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
@@ -162,7 +152,7 @@ func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte
 			return err
 		}
 
-		obj, err = append(obj)
+		obj, err = appendFn(obj)
 		if err != nil {
 			return err
 		}

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -6,19 +6,18 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "api_key"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "api_key"
 
 // Service represents a service for managing api-key data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]
 }
 
 // NewService creates a new instance of a service.
@@ -29,7 +28,10 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.APIKey, portainer.APIKeyID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
@@ -37,14 +39,14 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error) {
 	var result = make([]portainer.APIKey, 0)
 
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
 			}
 
 			if record.UserID == userID {
@@ -62,14 +64,14 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
 func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error) {
 	var k *portainer.APIKey
 	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.APIKey{},
 		func(obj interface{}) (interface{}, error) {
 			key, ok := obj.(*portainer.APIKey)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
-				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
 			}
 			if bytes.Equal(key.Digest, digest) {
 				k = key
@@ -90,9 +92,9 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 	return nil, err
 }
 
-// CreateAPIKey creates a new APIKey object.
-func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
-	return service.connection.CreateObject(
+// Create creates a new APIKey object.
+func (service *Service) Create(record *portainer.APIKey) error {
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.APIKeyID(id)
@@ -101,26 +103,3 @@ func (service *Service) CreateAPIKey(record *portainer.APIKey) error {
 		},
 	)
 }
-
-// GetAPIKey retrieves an existing APIKey object by api key ID.
-func (service *Service) GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	var key portainer.APIKey
-	identifier := service.connection.ConvertToKey(int(keyID))
-
-	err := service.connection.GetObject(BucketName, identifier, &key)
-	if err != nil {
-		return nil, err
-	}
-
-	return &key, nil
-}
-
-func (service *Service) UpdateAPIKey(key *portainer.APIKey) error {
-	identifier := service.connection.ConvertToKey(int(key.ID))
-	return service.connection.UpdateObject(BucketName, identifier, key)
-}
-
-func (service *Service) DeleteAPIKey(ID portainer.APIKeyID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/base.go

@@ -0,0 +1,66 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseCRUD[T any, I constraints.Integer] interface {
+	Create(element *T) error
+	Read(ID I) (*T, error)
+	ReadAll() ([]T, error)
+	Update(ID I, element *T) error
+	Delete(ID I) error
+}
+
+type BaseDataService[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+}
+
+func (s *BaseDataService[T, I]) BucketName() string {
+	return s.Bucket
+}
+
+func (service *BaseDataService[T, I]) Tx(tx portainer.Transaction) BaseDataServiceTx[T, I] {
+	return BaseDataServiceTx[T, I]{
+		Bucket:     service.Bucket,
+		Connection: service.Connection,
+		Tx:         tx,
+	}
+}
+
+func (service BaseDataService[T, I]) Read(ID I) (*T, error) {
+	var element *T
+
+	return element, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		element, err = service.Tx(tx).Read(ID)
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		collection, err = service.Tx(tx).ReadAll()
+
+		return err
+	})
+}
+
+func (service BaseDataService[T, I]) Update(ID I, element *T) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Update(ID, element)
+	})
+}
+
+func (service BaseDataService[T, I]) Delete(ID I) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Delete(ID)
+	})
+}

api/dataservices/base_tx.go

@@ -0,0 +1,49 @@
+package dataservices
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"golang.org/x/exp/constraints"
+)
+
+type BaseDataServiceTx[T any, I constraints.Integer] struct {
+	Bucket     string
+	Connection portainer.Connection
+	Tx         portainer.Transaction
+}
+
+func (service BaseDataServiceTx[T, I]) BucketName() string {
+	return service.Bucket
+}
+
+func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
+	var element T
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	err := service.Tx.GetObject(service.Bucket, identifier, &element)
+	if err != nil {
+		return nil, err
+	}
+
+	return &element, nil
+}
+
+func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
+	var collection = make([]T, 0)
+
+	return collection, service.Tx.GetAllWithJsoniter(
+		service.Bucket,
+		new(T),
+		AppendFn(&collection),
+	)
+}
+
+func (service BaseDataServiceTx[T, I]) Update(ID I, element *T) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.UpdateObject(service.Bucket, identifier, element)
+}
+
+func (service BaseDataServiceTx[T, I]) Delete(ID I) error {
+	identifier := service.Connection.ConvertToKey(int(ID))
+	return service.Tx.DeleteObject(service.Bucket, identifier)
+}

api/dataservices/customtemplate/customtemplate.go

@@ -1,25 +1,16 @@
 package customtemplate
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "customtemplates"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "customtemplates"
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,64 +21,20 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.CustomTemplate, portainer.CustomTemplateID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// CustomTemplates return an array containing all the custom templates.
-func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
-	var customTemplates = make([]portainer.CustomTemplate, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.CustomTemplate{},
-		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
-			customTemplate, ok := obj.(*portainer.CustomTemplate)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
-				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
-			}
-			customTemplates = append(customTemplates, *customTemplate)
-
-			return &portainer.CustomTemplate{}, nil
-		})
-
-	return customTemplates, err
-}
-
-// CustomTemplate returns an custom template by ID.
-func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
-	var customTemplate portainer.CustomTemplate
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &customTemplate)
-	if err != nil {
-		return nil, err
-	}
-
-	return &customTemplate, nil
-}
-
-// UpdateCustomTemplate updates an custom template.
-func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, customTemplate)
-}
-
-// DeleteCustomTemplate deletes an custom template.
-func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // CreateCustomTemplate uses the existing id and saves it.
 // TODO: where does the ID come from, and is it safe?
 func (service *Service) Create(customTemplate *portainer.CustomTemplate) error {
-	return service.connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
+	return service.Connection.CreateObjectWithId(BucketName, int(customTemplate.ID), customTemplate)
 }
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgegroup/edgegroup.go

@@ -2,6 +2,7 @@ package edgegroup
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -9,7 +10,7 @@ const BucketName = "edgegroups"
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection portainer.Connection
+	dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 func (service *Service) BucketName() string {
@@ -24,69 +25,36 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeGroups return a slice containing all the Edge groups.
-func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups []portainer.EdgeGroup
-	var err error
-
-	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-		groups, err = service.Tx(tx).EdgeGroups()
-		return err
-	})
-
-	return groups, err
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group *portainer.EdgeGroup
-	var err error
-
-	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
-		group, err = service.Tx(tx).EdgeGroup(ID)
-		return err
-	})
-
-	return group, err
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, group)
-}
-
 // Deprecated: UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	edgeGroup := &portainer.EdgeGroup{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeGroup, func() {
 		updateFunc(edgeGroup)
 	})
 }
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
-		return service.Tx(tx).DeleteEdgeGroup(ID)
-	})
-}
-
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) Create(group *portainer.EdgeGroup) error {
-	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+	return service.Connection.UpdateTx(func(tx portainer.Transaction) error {
 		return service.Tx(tx).Create(group)
 	})
 }

api/dataservices/edgegroup/tx.go

@@ -2,60 +2,13 @@ package edgegroup
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeGroups return a slice containing all the Edge groups.
-func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups = make([]portainer.EdgeGroup, 0)
-
-	err := service.tx.GetAllWithJsoniter(
-		BucketName,
-		&portainer.EdgeGroup{},
-		func(obj interface{}) (interface{}, error) {
-			group, ok := obj.(*portainer.EdgeGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
-				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
-			}
-			groups = append(groups, *group)
-
-			return &portainer.EdgeGroup{}, nil
-		})
-
-	return groups, err
-}
-
-// EdgeGroup returns an Edge group by ID.
-func (service ServiceTx) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group portainer.EdgeGroup
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &group)
-	if err != nil {
-		return nil, err
-	}
-
-	return &group, nil
-}
-
-// UpdateEdgeGroup updates an edge group.
-func (service ServiceTx) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, group)
+	dataservices.BaseDataServiceTx[portainer.EdgeGroup, portainer.EdgeGroupID]
 }
 
 // UpdateEdgeGroupFunc is a no-op inside a transaction.
@@ -63,14 +16,8 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeGroup deletes an Edge group.
-func (service ServiceTx) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			group.ID = portainer.EdgeGroupID(id)

api/dataservices/edgejob/edgejob.go

@@ -1,11 +1,8 @@
 package edgejob
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "edgejobs"
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,86 +21,50 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EdgeJobs returns a list of Edge jobs
-func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
-}
-
-// EdgeJob returns an Edge job by ID
-func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &edgeJob)
-	if err != nil {
-		return nil, err
-	}
-
-	return &edgeJob, nil
-}
-
 // Create creates a new EdgeJob
-func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+func (service *Service) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
+}
+
+// CreateWithID creates a new EdgeJob
+func (service *Service) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	edgeJob.ID = ID
 
-	return service.connection.CreateObjectWithId(
+	return service.Connection.CreateObjectWithId(
 		BucketName,
 		int(edgeJob.ID),
 		edgeJob,
 	)
 }
 
-// Deprecated: use UpdateEdgeJobFunc instead
-func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
-}
-
 // UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	edgeJob := &portainer.EdgeJob{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
 		updateFunc(edgeJob)
 	})
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/edgejob/tx.go

@@ -2,68 +2,25 @@ package edgejob
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EdgeJobs returns a list of Edge jobs
-func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
-	var edgeJobs = make([]portainer.EdgeJob, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.EdgeJob{},
-		func(obj interface{}) (interface{}, error) {
-			job, ok := obj.(*portainer.EdgeJob)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
-				return nil, fmt.Errorf("failed to convert to EdgeJob object: %s", obj)
-			}
-
-			edgeJobs = append(edgeJobs, *job)
-
-			return &portainer.EdgeJob{}, nil
-		})
-
-	return edgeJobs, err
-}
-
-// EdgeJob returns an Edge job by ID
-func (service ServiceTx) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
-	var edgeJob portainer.EdgeJob
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &edgeJob)
-	if err != nil {
-		return nil, err
-	}
-
-	return &edgeJob, nil
+	dataservices.BaseDataServiceTx[portainer.EdgeJob, portainer.EdgeJobID]
 }
 
 // Create creates a new EdgeJob
-func (service ServiceTx) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	edgeJob.ID = ID
-
-	return service.tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+func (service ServiceTx) Create(edgeJob *portainer.EdgeJob) error {
+	return service.CreateWithID(portainer.EdgeJobID(service.GetNextIdentifier()), edgeJob)
 }
 
-// UpdateEdgeJob updates an edge job
-func (service ServiceTx) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, edgeJob)
+// CreateWithID creates a new EdgeJob
+func (service ServiceTx) CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.Tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
 }
 
 // UpdateEdgeJobFunc is a no-op inside a transaction.
@@ -71,14 +28,7 @@ func (service ServiceTx) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc fu
 	return errors.New("cannot be called inside a transaction")
 }
 
-// DeleteEdgeJob deletes an Edge job
-func (service ServiceTx) DeleteEdgeJob(ID portainer.EdgeJobID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
-	return service.tx.GetNextIdentifier(BucketName)
+	return service.Tx.GetNextIdentifier(BucketName)
 }

api/dataservices/edgestack/edgestack.go

@@ -1,12 +1,10 @@
 package edgestack
 
 import (
-	"fmt"
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -64,22 +62,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.connection.GetAll(
 		BucketName,
 		&portainer.EdgeStack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.EdgeStack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
-				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.EdgeStack{}, nil
-		})
-
-	return stacks, err
+		dataservices.AppendFn(&stacks),
+	)
 }
 
 // EdgeStack returns an Edge stack by ID.

api/dataservices/endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -144,6 +145,23 @@ func (service *Service) Create(endpoint *portainer.Endpoint) error {
 	})
 }
 
+func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.connection.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
 	var identifier int

api/dataservices/endpoint/tx.go

@@ -1,9 +1,8 @@
 package endpoint
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -28,6 +27,8 @@ func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 		return nil, err
 	}
 
+	endpoint.LastCheckInDate, _ = service.service.Heartbeat(ID)
+
 	return &endpoint, nil
 }
 
@@ -65,6 +66,7 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 	for edgeID, endpointID := range service.service.idxEdgeID {
 		if endpointID == ID {
 			delete(service.service.idxEdgeID, edgeID)
+
 			break
 		}
 	}
@@ -80,22 +82,11 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
 func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.tx.GetAllWithJsoniter(
+	return endpoints, service.tx.GetAllWithJsoniter(
 		BucketName,
 		&portainer.Endpoint{},
-		func(obj interface{}) (interface{}, error) {
-			endpoint, ok := obj.(*portainer.Endpoint)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
-				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
-			}
-
-			endpoints = append(endpoints, *endpoint)
-
-			return &portainer.Endpoint{}, nil
-		})
-
-	return endpoints, err
+		dataservices.AppendFn(&endpoints),
+	)
 }
 
 func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {
@@ -131,6 +122,23 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
 	return nil
 }
 
+func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	return endpoints, service.tx.GetAll(
+		BucketName,
+		&portainer.Endpoint{},
+		dataservices.FilterFn(&endpoints, func(e portainer.Endpoint) bool {
+			for t := range e.TeamAccessPolicies {
+				if t == teamID {
+					return true
+				}
+			}
+			return false
+		}),
+	)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service ServiceTx) GetNextIdentifier() int {
 	return service.tx.GetNextIdentifier(BucketName)

api/dataservices/endpointgroup/endpointgroup.go

@@ -1,11 +1,8 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -15,11 +12,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,67 +23,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &endpointGroup)
-	if err != nil {
-		return nil, err
-	}
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, endpointGroup)
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
-}
-
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service *Service) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointgroup/tx.go

@@ -1,72 +1,17 @@
 package endpointgroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// EndpointGroup returns an environment(endpoint) group by ID.
-func (service ServiceTx) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
-	var endpointGroup portainer.EndpointGroup
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &endpointGroup)
-	if err != nil {
-		return nil, err
-	}
-
-	return &endpointGroup, nil
-}
-
-// UpdateEndpointGroup updates an environment(endpoint) group.
-func (service ServiceTx) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, endpointGroup)
-}
-
-// DeleteEndpointGroup deletes an environment(endpoint) group.
-func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
-// EndpointGroups return an array containing all the environment(endpoint) groups.
-func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
-	var endpointGroups = make([]portainer.EndpointGroup, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.EndpointGroup{},
-		func(obj interface{}) (interface{}, error) {
-			endpointGroup, ok := obj.(*portainer.EndpointGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
-				return nil, fmt.Errorf("failed to convert to EndpointGroup object: %s", obj)
-			}
-
-			endpointGroups = append(endpointGroups, *endpointGroup)
-
-			return &portainer.EndpointGroup{}, nil
-		})
-
-	return endpointGroups, err
+	dataservices.BaseDataServiceTx[portainer.EndpointGroup, portainer.EndpointGroupID]
 }
 
 // CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
 func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			endpointGroup.ID = portainer.EndpointGroupID(id)

api/dataservices/endpointrelation/endpointrelation.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -54,22 +53,11 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.connection.GetAll(
+	return all, service.connection.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID

api/dataservices/endpointrelation/tx.go

@@ -1,9 +1,8 @@
 package endpointrelation
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
@@ -22,22 +21,11 @@ func (service ServiceTx) BucketName() string {
 func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
-	err := service.tx.GetAll(
+	return all, service.tx.GetAll(
 		BucketName,
 		&portainer.EndpointRelation{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.EndpointRelation)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
-				return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
-			}
-
-			all = append(all, *r)
-
-			return &portainer.EndpointRelation{}, nil
-		})
-
-	return all, err
+		dataservices.AppendFn(&all),
+	)
 }
 
 // EndpointRelation returns an Environment(Endpoint) relation object by EndpointID

api/dataservices/extension/extension.go

@@ -1,17 +1,12 @@
 package extension
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "extension"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "extension"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -51,22 +46,12 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.GetAll(
+	return extensions, service.connection.GetAll(
 		BucketName,
 		&portainer.Extension{},
-		func(obj interface{}) (interface{}, error) {
-			extension, ok := obj.(*portainer.Extension)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
-				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
-			}
+		dataservices.AppendFn(&extensions),
+	)
 
-			extensions = append(extensions, *extension)
-
-			return &portainer.Extension{}, nil
-		})
-
-	return extensions, err
 }
 
 // Persist persists a extension inside the database.

api/dataservices/fdoprofile/fdoprofile.go

@@ -1,25 +1,16 @@
 package fdoprofile
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "fdo_profiles"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "fdo_profiles"
 
 // Service represents a service for managingFDO Profiles data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,66 +21,23 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// FDOProfiles return an array containing all the FDO Profiles.
-func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
-	var fdoProfiles = make([]portainer.FDOProfile, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.FDOProfile{},
-		func(obj interface{}) (interface{}, error) {
-			fdoProfile, ok := obj.(*portainer.FDOProfile)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
-
-				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
-			}
-			fdoProfiles = append(fdoProfiles, *fdoProfile)
-			return &portainer.FDOProfile{}, nil
-		})
-
-	return fdoProfiles, err
-}
-
-// FDOProfile returns an FDO Profile by ID.
-func (service *Service) FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error) {
-	var FDOProfile portainer.FDOProfile
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &FDOProfile)
-	if err != nil {
-		return nil, err
-	}
-
-	return &FDOProfile, nil
-}
-
 // Create assign an ID to a new FDO Profile and saves it.
 func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
-	return service.connection.CreateObjectWithId(
+	return service.Connection.CreateObjectWithId(
 		BucketName,
 		int(FDOProfile.ID),
 		FDOProfile,
 	)
 }
 
-// Update updates an FDO Profile.
-func (service *Service) Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, FDOProfile)
-}
-
-// Delete deletes an FDO Profile.
-func (service *Service) Delete(ID portainer.FDOProfileID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // GetNextIdentifier returns the next identifier for a FDO Profile.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -1,25 +1,16 @@
 package helmuserrepository
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "helm_user_repository"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "helm_user_repository"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,59 +21,29 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// HelmUserRepository returns an array of all HelmUserRepository
-func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
-	var repos = make([]portainer.HelmUserRepository, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			r, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			repos = append(repos, *r)
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return repos, err
-}
-
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
 func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
-	err := service.connection.GetAll(
+	return result, service.Connection.GetAll(
 		BucketName,
 		&portainer.HelmUserRepository{},
-		func(obj interface{}) (interface{}, error) {
-			record, ok := obj.(*portainer.HelmUserRepository)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
-				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
-			}
-
-			if record.UserID == userID {
-				result = append(result, *record)
-			}
-
-			return &portainer.HelmUserRepository{}, nil
-		})
-
-	return result, err
+		dataservices.FilterFn(&result, func(e portainer.HelmUserRepository) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // CreateHelmUserRepository creates a new HelmUserRepository object.
 func (service *Service) Create(record *portainer.HelmUserRepository) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			record.ID = portainer.HelmUserRepositoryID(id)
@@ -90,15 +51,3 @@ func (service *Service) Create(record *portainer.HelmUserRepository) error {
 		},
 	)
 }
-
-// UpdateHelmUserRepostory updates an registry.
-func (service *Service) UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, registry *portainer.HelmUserRepository) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteHelmUserRepository deletes an registry.
-func (service *Service) DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/helpers.go

@@ -0,0 +1,68 @@
+package dataservices
+
+import (
+	"errors"
+	"fmt"
+
+	perrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+)
+
+// ErrStop signals the stop of computation when filtering results
+var ErrStop = errors.New("stop")
+
+func IsErrObjectNotFound(e error) bool {
+	return errors.Is(e, perrors.ErrObjectNotFound)
+}
+
+// AppendFn appends elements to the given collection slice
+func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		*collection = append(*collection, *element)
+
+		return new(T), nil
+	}
+}
+
+// FilterFn appends elements to the given collection when the predicate is true
+func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		element, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*element) {
+			*collection = append(*collection, *element)
+		}
+
+		return new(T), nil
+	}
+}
+
+// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
+// success
+func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
+	return func(obj interface{}) (interface{}, error) {
+		e, ok := obj.(*T)
+		if !ok {
+			log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
+			return nil, fmt.Errorf("failed to convert to %T object: %#v", new(T), obj)
+		}
+
+		if predicate(*e) {
+			*element = *e
+			return new(T), ErrStop
+		}
+
+		return new(T), nil
+	}
+}

api/dataservices/interface.go

@@ -1,13 +1,11 @@
 package dataservices
 
 import (
-	"errors"
 	"io"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
-	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 )
 
 type (
@@ -39,7 +37,6 @@ type (
 		Webhook() WebhookService
 	}
 
-	// DataStore defines the interface to manage the data
 	DataStore interface {
 		Open() (newStore bool, err error)
 		Init() error
@@ -57,36 +54,22 @@ type (
 
 	// CustomTemplateService represents a service to manage custom templates
 	CustomTemplateService interface {
+		BaseCRUD[portainer.CustomTemplate, portainer.CustomTemplateID]
 		GetNextIdentifier() int
-		CustomTemplates() ([]portainer.CustomTemplate, error)
-		CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error)
-		Create(customTemplate *portainer.CustomTemplate) error
-		UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error
-		DeleteCustomTemplate(ID portainer.CustomTemplateID) error
-		BucketName() string
 	}
 
 	// EdgeGroupService represents a service to manage Edge groups
 	EdgeGroupService interface {
-		EdgeGroups() ([]portainer.EdgeGroup, error)
-		EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error)
-		Create(group *portainer.EdgeGroup) error
-		UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error
+		BaseCRUD[portainer.EdgeGroup, portainer.EdgeGroupID]
 		UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(group *portainer.EdgeGroup)) error
-		DeleteEdgeGroup(ID portainer.EdgeGroupID) error
-		BucketName() string
 	}
 
 	// EdgeJobService represents a service to manage Edge jobs
 	EdgeJobService interface {
-		EdgeJobs() ([]portainer.EdgeJob, error)
-		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
-		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
-		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		BaseCRUD[portainer.EdgeJob, portainer.EdgeJobID]
+		CreateWithID(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
-		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
-		BucketName() string
 	}
 
 	// EdgeStackService represents a service to manage Edge stacks
@@ -95,7 +78,6 @@ type (
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
 		EdgeStackVersion(ID portainer.EdgeStackID) (int, bool)
 		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
-		// Deprecated: Use UpdateEdgeStackFunc instead.
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
@@ -107,6 +89,7 @@ type (
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool)
+		EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.Endpoint, error)
 		Heartbeat(endpointID portainer.EndpointID) (int64, bool)
 		UpdateHeartbeat(endpointID portainer.EndpointID)
 		Endpoints() ([]portainer.Endpoint, error)
@@ -119,12 +102,7 @@ type (
 
 	// EndpointGroupService represents a service for managing environment(endpoint) group data
 	EndpointGroupService interface {
-		EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error)
-		EndpointGroups() ([]portainer.EndpointGroup, error)
-		Create(group *portainer.EndpointGroup) error
-		UpdateEndpointGroup(ID portainer.EndpointGroupID, group *portainer.EndpointGroup) error
-		DeleteEndpointGroup(ID portainer.EndpointGroupID) error
-		BucketName() string
+		BaseCRUD[portainer.EndpointGroup, portainer.EndpointGroupID]
 	}
 
 	// EndpointRelationService represents a service for managing environment(endpoint) relations data
@@ -139,23 +117,14 @@ type (
 
 	// FDOProfileService represents a service to manage FDO Profiles
 	FDOProfileService interface {
-		FDOProfiles() ([]portainer.FDOProfile, error)
-		FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error)
-		Create(FDOProfile *portainer.FDOProfile) error
-		Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error
-		Delete(ID portainer.FDOProfileID) error
+		BaseCRUD[portainer.FDOProfile, portainer.FDOProfileID]
 		GetNextIdentifier() int
-		BucketName() string
 	}
 
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
+		BaseCRUD[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
-		Create(record *portainer.HelmUserRepository) error
-		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
-		DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error
-		BucketName() string
 	}
 
 	// JWTService represents a service for managing JWT tokens
@@ -169,40 +138,23 @@ type (
 
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
-		Registry(ID portainer.RegistryID) (*portainer.Registry, error)
-		Registries() ([]portainer.Registry, error)
-		Create(registry *portainer.Registry) error
-		UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error
-		DeleteRegistry(ID portainer.RegistryID) error
-		BucketName() string
+		BaseCRUD[portainer.Registry, portainer.RegistryID]
 	}
 
 	// ResourceControlService represents a service for managing resource control data
 	ResourceControlService interface {
-		ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error)
+		BaseCRUD[portainer.ResourceControl, portainer.ResourceControlID]
 		ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error)
-		ResourceControls() ([]portainer.ResourceControl, error)
-		Create(rc *portainer.ResourceControl) error
-		UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error
-		DeleteResourceControl(ID portainer.ResourceControlID) error
-		BucketName() string
 	}
 
 	// RoleService represents a service for managing user roles
 	RoleService interface {
-		Role(ID portainer.RoleID) (*portainer.Role, error)
-		Roles() ([]portainer.Role, error)
-		Create(role *portainer.Role) error
-		UpdateRole(ID portainer.RoleID, role *portainer.Role) error
-		BucketName() string
+		BaseCRUD[portainer.Role, portainer.RoleID]
 	}
 
 	// APIKeyRepositoryService
 	APIKeyRepository interface {
-		CreateAPIKey(key *portainer.APIKey) error
-		GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error)
-		UpdateAPIKey(key *portainer.APIKey) error
-		DeleteAPIKey(ID portainer.APIKeyID) error
+		BaseCRUD[portainer.APIKey, portainer.APIKeyID]
 		GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error)
 		GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error)
 	}
@@ -215,12 +167,7 @@ type (
 	}
 
 	SnapshotService interface {
-		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
-		Snapshots() ([]portainer.Snapshot, error)
-		UpdateSnapshot(snapshot *portainer.Snapshot) error
-		DeleteSnapshot(endpointID portainer.EndpointID) error
-		Create(snapshot *portainer.Snapshot) error
-		BucketName() string
+		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
 	}
 
 	// SSLSettingsService represents a service for managing application settings
@@ -232,53 +179,33 @@ type (
 
 	// StackService represents a service for managing stack data
 	StackService interface {
-		Stack(ID portainer.StackID) (*portainer.Stack, error)
+		BaseCRUD[portainer.Stack, portainer.StackID]
 		StackByName(name string) (*portainer.Stack, error)
 		StacksByName(name string) ([]portainer.Stack, error)
-		Stacks() ([]portainer.Stack, error)
-		Create(stack *portainer.Stack) error
-		UpdateStack(ID portainer.StackID, stack *portainer.Stack) error
-		DeleteStack(ID portainer.StackID) error
 		GetNextIdentifier() int
 		StackByWebhookID(ID string) (*portainer.Stack, error)
 		RefreshableStacks() ([]portainer.Stack, error)
-		BucketName() string
 	}
 
 	// TagService represents a service for managing tag data
 	TagService interface {
-		Tags() ([]portainer.Tag, error)
-		Tag(ID portainer.TagID) (*portainer.Tag, error)
-		Create(tag *portainer.Tag) error
-		UpdateTag(ID portainer.TagID, tag *portainer.Tag) error
+		BaseCRUD[portainer.Tag, portainer.TagID]
 		UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error
-		DeleteTag(ID portainer.TagID) error
-		BucketName() string
 	}
 
 	// TeamService represents a service for managing user data
 	TeamService interface {
-		Team(ID portainer.TeamID) (*portainer.Team, error)
+		BaseCRUD[portainer.Team, portainer.TeamID]
 		TeamByName(name string) (*portainer.Team, error)
-		Teams() ([]portainer.Team, error)
-		Create(team *portainer.Team) error
-		UpdateTeam(ID portainer.TeamID, team *portainer.Team) error
-		DeleteTeam(ID portainer.TeamID) error
-		BucketName() string
 	}
 
 	// TeamMembershipService represents a service for managing team membership data
 	TeamMembershipService interface {
-		TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error)
-		TeamMemberships() ([]portainer.TeamMembership, error)
+		BaseCRUD[portainer.TeamMembership, portainer.TeamMembershipID]
 		TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error)
 		TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error)
-		Create(membership *portainer.TeamMembership) error
-		UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error
-		DeleteTeamMembership(ID portainer.TeamMembershipID) error
 		DeleteTeamMembershipByUserID(userID portainer.UserID) error
 		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
-		BucketName() string
 		DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error
 	}
 
@@ -291,38 +218,24 @@ type (
 
 	// UserService represents a service for managing user data
 	UserService interface {
-		User(ID portainer.UserID) (*portainer.User, error)
+		BaseCRUD[portainer.User, portainer.UserID]
 		UserByUsername(username string) (*portainer.User, error)
-		Users() ([]portainer.User, error)
 		UsersByRole(role portainer.UserRole) ([]portainer.User, error)
-		Create(user *portainer.User) error
-		UpdateUser(ID portainer.UserID, user *portainer.User) error
-		DeleteUser(ID portainer.UserID) error
-		BucketName() string
 	}
 
 	// VersionService represents a service for managing version data
 	VersionService interface {
-		Edition() (portainer.SoftwareEdition, error)
 		InstanceID() (string, error)
 		UpdateInstanceID(ID string) error
+		Edition() (portainer.SoftwareEdition, error)
 		Version() (*models.Version, error)
 		UpdateVersion(*models.Version) error
 	}
 
 	// WebhookService represents a service for managing webhook data.
 	WebhookService interface {
-		Webhooks() ([]portainer.Webhook, error)
-		Webhook(ID portainer.WebhookID) (*portainer.Webhook, error)
-		Create(portainer *portainer.Webhook) error
-		UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error
+		BaseCRUD[portainer.Webhook, portainer.WebhookID]
 		WebhookByResourceID(resourceID string) (*portainer.Webhook, error)
 		WebhookByToken(token string) (*portainer.Webhook, error)
-		DeleteWebhook(ID portainer.WebhookID) error
-		BucketName() string
 	}
 )
-
-func IsErrObjectNotFound(e error) bool {
-	return errors.Is(e, dserrors.ErrObjectNotFound)
-}

api/dataservices/registry/registry.go

@@ -1,11 +1,8 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "registries"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,55 +21,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Registry returns a registry by ID.
-func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &registry)
-	if err != nil {
-		return nil, err
-	}
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service *Service) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
-}
-
 // Create creates a new registry.
 func (service *Service) Create(registry *portainer.Registry) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -84,15 +48,3 @@ func (service *Service) Create(registry *portainer.Registry) error {
 		},
 	)
 }
-
-// UpdateRegistry updates a registry.
-func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteRegistry deletes a registry.
-func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/registry/tx.go

@@ -1,59 +1,17 @@
 package registry
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Registry returns a registry by ID.
-func (service ServiceTx) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
-	var registry portainer.Registry
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &registry)
-	if err != nil {
-		return nil, err
-	}
-
-	return &registry, nil
-}
-
-// Registries returns an array containing all the registries.
-func (service ServiceTx) Registries() ([]portainer.Registry, error) {
-	var registries = make([]portainer.Registry, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Registry{},
-		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*portainer.Registry)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
-				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
-			}
-
-			registries = append(registries, *registry)
-
-			return &portainer.Registry{}, nil
-		})
-
-	return registries, err
+	dataservices.BaseDataServiceTx[portainer.Registry, portainer.RegistryID]
 }
 
 // Create creates a new registry.
 func (service ServiceTx) Create(registry *portainer.Registry) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			registry.ID = portainer.RegistryID(id)
@@ -61,15 +19,3 @@ func (service ServiceTx) Create(registry *portainer.Registry) error {
 		},
 	)
 }
-
-// UpdateRegistry updates a registry.
-func (service ServiceTx) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, registry)
-}
-
-// DeleteRegistry deletes a registry.
-func (service ServiceTx) DeleteRegistry(ID portainer.RegistryID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
@@ -14,11 +15,7 @@ const BucketName = "resource_control"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // NewService creates a new instance of a service.
@@ -29,44 +26,37 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// ResourceControl returns a ResourceControl object by ID
-func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &resourceControl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &resourceControl, nil
-}
-
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
 // to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
 // if no ResourceControl was found.
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -90,31 +80,9 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 	return nil, err
 }
 
-// ResourceControls returns all the ResourceControl objects
-func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
-}
-
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -122,15 +90,3 @@ func (service *Service) Create(resourceControl *portainer.ResourceControl) error
 		},
 	)
 }
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, resourceControl)
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/resourcecontrol/tx.go

@@ -5,30 +5,13 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// ResourceControl returns a ResourceControl object by ID
-func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var resourceControl portainer.ResourceControl
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &resourceControl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &resourceControl, nil
+	dataservices.BaseDataServiceTx[portainer.ResourceControl, portainer.ResourceControlID]
 }
 
 // ResourceControlByResourceIDAndType returns a ResourceControl object by checking if the resourceID is equal
@@ -37,14 +20,14 @@ func (service ServiceTx) ResourceControl(ID portainer.ResourceControlID) (*porta
 func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 	stop := fmt.Errorf("ok")
-	err := service.tx.GetAll(
+	err := service.Tx.GetAll(
 		BucketName,
 		&portainer.ResourceControl{},
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
 				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
@@ -68,31 +51,9 @@ func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, r
 	return nil, err
 }
 
-// ResourceControls returns all the ResourceControl objects
-func (service ServiceTx) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
-			}
-
-			rcs = append(rcs, *rc)
-
-			return &portainer.ResourceControl{}, nil
-		})
-
-	return rcs, err
-}
-
 // CreateResourceControl creates a new ResourceControl object
 func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			resourceControl.ID = portainer.ResourceControlID(id)
@@ -100,15 +61,3 @@ func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) erro
 		},
 	)
 }
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service ServiceTx) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, resourceControl)
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service ServiceTx) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/role/role.go

@@ -1,11 +1,8 @@
 package role
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -13,11 +10,7 @@ const BucketName = "roles"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Role, portainer.RoleID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,55 +21,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Role returns a Role by ID
-func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &set)
-	if err != nil {
-		return nil, err
-	}
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service *Service) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Role{},
-		func(obj interface{}) (interface{}, error) {
-			set, ok := obj.(*portainer.Role)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-				return nil, fmt.Errorf("failed to convert to Role object: %s", obj)
-			}
-
-			sets = append(sets, *set)
-
-			return &portainer.Role{}, nil
-		})
-
-	return sets, err
-}
-
 // CreateRole creates a new Role.
 func (service *Service) Create(role *portainer.Role) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			role.ID = portainer.RoleID(id)
@@ -84,9 +48,3 @@ func (service *Service) Create(role *portainer.Role) error {
 		},
 	)
 }
-
-// UpdateRole updates a role.
-func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, role)
-}

api/dataservices/role/tx.go

@@ -1,61 +1,17 @@
 package role
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-// Service represents a service for managing environment(endpoint) data.
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Role returns a Role by ID
-func (service ServiceTx) Role(ID portainer.RoleID) (*portainer.Role, error) {
-	var set portainer.Role
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &set)
-	if err != nil {
-		return nil, err
-	}
-
-	return &set, nil
-}
-
-// Roles returns an array containing all the sets.
-func (service ServiceTx) Roles() ([]portainer.Role, error) {
-	var sets = make([]portainer.Role, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Role{},
-		func(obj interface{}) (interface{}, error) {
-			set, ok := obj.(*portainer.Role)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
-				return nil, fmt.Errorf("failed to convert to Role object: %s", obj)
-			}
-
-			sets = append(sets, *set)
-
-			return &portainer.Role{}, nil
-		})
-
-	return sets, err
+	dataservices.BaseDataServiceTx[portainer.Role, portainer.RoleID]
 }
 
 // CreateRole creates a new Role.
 func (service ServiceTx) Create(role *portainer.Role) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			role.ID = portainer.RoleID(id)
@@ -63,9 +19,3 @@ func (service ServiceTx) Create(role *portainer.Role) error {
 		},
 	)
 }
-
-// UpdateRole updates a role.
-func (service ServiceTx) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, role)
-}

api/dataservices/schedule/schedule.go

@@ -1,17 +1,12 @@
 package schedule
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "schedules"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "schedules"
 
 // Service represents a service for managing schedule data.
 type Service struct {
@@ -63,22 +58,11 @@ func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			schedules = append(schedules, *schedule)
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.AppendFn(&schedules),
+	)
 }
 
 // SchedulesByJobType return a array containing all the schedules
@@ -86,24 +70,13 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.GetAll(
+	return schedules, service.connection.GetAll(
 		BucketName,
 		&portainer.Schedule{},
-		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*portainer.Schedule)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
-				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
-			}
-
-			if schedule.JobType == jobType {
-				schedules = append(schedules, *schedule)
-			}
-
-			return &portainer.Schedule{}, nil
-		})
-
-	return schedules, err
+		dataservices.FilterFn(&schedules, func(e portainer.Schedule) bool {
+			return e.JobType == jobType
+		}),
+	)
 }
 
 // Create assign an ID to a new schedule and saves it.

api/dataservices/snapshot/snapshot.go

@@ -1,11 +1,8 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -13,11 +10,7 @@ const (
 )
 
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]
 }
 
 func NewService(connection portainer.Connection) (*Service, error) {
@@ -27,58 +20,23 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-	identifier := service.connection.ConvertToKey(int(endpointID))
-
-	err := service.connection.GetObject(BucketName, identifier, &snapshot)
-	if err != nil {
-		return nil, err
-	}
-
-	return &snapshot, nil
-}
-
-func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	err := service.connection.GetAllWithJsoniter(
-		BucketName,
-		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
-}
-
-func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
-	return service.connection.UpdateObject(BucketName, identifier, snapshot)
-}
-
-func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	identifier := service.connection.ConvertToKey(int(endpointID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 func (service *Service) Create(snapshot *portainer.Snapshot) error {
-	return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/snapshot/tx.go

@@ -1,63 +1,14 @@
 package snapshot
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
-	var snapshot portainer.Snapshot
-	identifier := service.service.connection.ConvertToKey(int(endpointID))
-
-	err := service.tx.GetObject(BucketName, identifier, &snapshot)
-	if err != nil {
-		return nil, err
-	}
-
-	return &snapshot, nil
-}
-
-func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
-	var snapshots = make([]portainer.Snapshot, 0)
-
-	err := service.tx.GetAllWithJsoniter(
-		BucketName,
-		&portainer.Snapshot{},
-		func(obj interface{}) (interface{}, error) {
-			snapshot, ok := obj.(*portainer.Snapshot)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
-				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
-			}
-			snapshots = append(snapshots, *snapshot)
-			return &portainer.Snapshot{}, nil
-		})
-
-	return snapshots, err
-}
-
-func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {
-	identifier := service.service.connection.ConvertToKey(int(snapshot.EndpointID))
-	return service.tx.UpdateObject(BucketName, identifier, snapshot)
-}
-
-func (service ServiceTx) DeleteSnapshot(endpointID portainer.EndpointID) error {
-	identifier := service.service.connection.ConvertToKey(int(endpointID))
-	return service.tx.DeleteObject(BucketName, identifier)
+	dataservices.BaseDataServiceTx[portainer.Snapshot, portainer.EndpointID]
 }
 
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
-	return service.tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }

api/dataservices/stack/stack.go

@@ -2,27 +2,19 @@ package stack
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "stacks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "stacks"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Stack, portainer.StackID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,35 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Stack, portainer.StackID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Stack returns a stack object by ID.
-func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
-	var stack portainer.Stack
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &stack)
-	if err != nil {
-		return nil, err
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: service.BaseDataService.Tx(tx),
 	}
-
-	return &stack, nil
 }
 
 // StackByName returns a stack object by name.
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
-	var s *portainer.Stack
+	var s portainer.Stack
 
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 
-			if stack.Name == name {
-				s = stack
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if errors.Is(err, stop) {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
@@ -86,97 +65,42 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.Name == name {
-				stacks = append(stacks, stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
-}
-
-// Stacks returns an array containing all the stacks.
-func (service *Service) Stacks() ([]portainer.Stack, error) {
-	var stacks = make([]portainer.Stack, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			stacks = append(stacks, *stack)
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
 }
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	return service.Connection.GetNextIdentifier(BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) Create(stack *portainer.Stack) error {
-	return service.connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
-}
-
-// UpdateStack updates a stack.
-func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, stack)
-}
-
-// DeleteStack deletes a stack.
-func (service *Service) DeleteStack(ID portainer.StackID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
+	return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
 }
 
 // StackByWebhookID returns a pointer to a stack object by webhook ID.
 // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
 func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	var s *portainer.Stack
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var s portainer.Stack
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			var ok bool
-			s, ok = obj.(*portainer.Stack)
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
 
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-
-				return &portainer.Stack{}, nil
-			}
-
-			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
-				return nil, stop
-			}
-
-			return &portainer.Stack{}, nil
-		})
-	if errors.Is(err, stop) {
-		return s, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
@@ -189,22 +113,11 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 	stacks := make([]portainer.Stack, 0)
 
-	err := service.connection.GetAll(
+	return stacks, service.Connection.GetAll(
 		BucketName,
 		&portainer.Stack{},
-		func(obj interface{}) (interface{}, error) {
-			stack, ok := obj.(*portainer.Stack)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
-				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
-			}
-
-			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
-				stacks = append(stacks, *stack)
-			}
-
-			return &portainer.Stack{}, nil
-		})
-
-	return stacks, err
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
 }

api/dataservices/stack/tx.go

@@ -0,0 +1,98 @@
+package stack
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.Stack, portainer.StackID]
+}
+
+// StackByName returns a stack object by name.
+func (service ServiceTx) StackByName(name string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// Stacks returns an array containing all the stacks with same name
+func (service ServiceTx) StacksByName(name string) ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.Name == name
+		}),
+	)
+}
+
+// GetNextIdentifier returns the next identifier for a stack.
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.Tx.GetNextIdentifier(BucketName)
+}
+
+// CreateStack creates a new stack.
+func (service ServiceTx) Create(stack *portainer.Stack) error {
+	return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack)
+}
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
+	var s portainer.Stack
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FirstFn(&s, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && strings.EqualFold(e.AutoUpdate.Webhook, id)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &s, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+
+	return stacks, service.Tx.GetAll(
+		BucketName,
+		&portainer.Stack{},
+		dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
+			return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
+		}),
+	)
+}

api/dataservices/tag/tag.go

@@ -1,11 +1,8 @@
 package tag
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 const (
@@ -15,11 +12,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Tag, portainer.TagID]
 }
 
 // NewService creates a new instance of a service.
@@ -30,55 +23,26 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// Tags return an array containing all the tags.
-func (service *Service) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
-}
-
-// Tag returns a tag by ID.
-func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &tag)
-	if err != nil {
-		return nil, err
-	}
-
-	return &tag, nil
-}
-
 // CreateTag creates a new tag.
 func (service *Service) Create(tag *portainer.Tag) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -87,24 +51,12 @@ func (service *Service) Create(tag *portainer.Tag) error {
 	)
 }
 
-// Deprecated: Use UpdateTagFunc instead.
-func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, tag)
-}
-
 // UpdateTagFunc updates a tag inside a transaction avoiding data races.
 func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
-	id := service.connection.ConvertToKey(int(ID))
+	id := service.Connection.ConvertToKey(int(ID))
 	tag := &portainer.Tag{}
 
-	return service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
+	return service.Connection.UpdateObjectFunc(BucketName, id, tag, func() {
 		updateFunc(tag)
 	})
 }
-
-// DeleteTag deletes a tag.
-func (service *Service) DeleteTag(ID portainer.TagID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/tag/tx.go

@@ -2,60 +2,18 @@ package tag
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
+	"github.com/portainer/portainer/api/dataservices"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// Tags return an array containing all the tags.
-func (service ServiceTx) Tags() ([]portainer.Tag, error) {
-	var tags = make([]portainer.Tag, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.Tag{},
-		func(obj interface{}) (interface{}, error) {
-			tag, ok := obj.(*portainer.Tag)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
-				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
-			}
-
-			tags = append(tags, *tag)
-
-			return &portainer.Tag{}, nil
-		})
-
-	return tags, err
-}
-
-// Tag returns a tag by ID.
-func (service ServiceTx) Tag(ID portainer.TagID) (*portainer.Tag, error) {
-	var tag portainer.Tag
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &tag)
-	if err != nil {
-		return nil, err
-	}
-
-	return &tag, nil
+	dataservices.BaseDataServiceTx[portainer.Tag, portainer.TagID]
 }
 
 // CreateTag creates a new tag.
 func (service ServiceTx) Create(tag *portainer.Tag) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			tag.ID = portainer.TagID(id)
@@ -64,19 +22,7 @@ func (service ServiceTx) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTag updates a tag
-func (service ServiceTx) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, tag)
-}
-
 // UpdateTagFunc is a no-op inside a transaction
 func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
 	return errors.New("cannot be called inside a transaction")
 }
-
-// DeleteTag deletes a tag.
-func (service ServiceTx) DeleteTag(ID portainer.TagID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}

api/dataservices/team/team.go

@@ -2,27 +2,19 @@ package team
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "teams"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "teams"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Team, portainer.TeamID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,29 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Team, portainer.TeamID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Team returns a Team by ID
-func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
-	var team portainer.Team
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &team)
-	if err != nil {
-		return nil, err
-	}
-
-	return &team, nil
-}
-
 // TeamByName returns a team by name.
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
-	var t *portainer.Team
+	var t portainer.Team
 
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
+		dataservices.FirstFn(&t, func(e portainer.Team) bool {
+			return strings.EqualFold(e.Name, name)
+		}),
+	)
 
-			if strings.EqualFold(team.Name, name) {
-				t = team
-				return nil, stop
-			}
-
-			return &portainer.Team{}, nil
-		})
-	if errors.Is(err, stop) {
-		return t, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &t, nil
 	}
+
 	if err == nil {
 		return nil, dserrors.ErrObjectNotFound
 	}
@@ -82,37 +55,9 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	return nil, err
 }
 
-// Teams return an array containing all the teams.
-func (service *Service) Teams() ([]portainer.Team, error) {
-	var teams = make([]portainer.Team, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Team{},
-		func(obj interface{}) (interface{}, error) {
-			team, ok := obj.(*portainer.Team)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
-				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
-			}
-
-			teams = append(teams, *team)
-
-			return &portainer.Team{}, nil
-		})
-
-	return teams, err
-}
-
-// UpdateTeam saves a Team.
-func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, team)
-}
-
 // CreateTeam creates a new Team.
 func (service *Service) Create(team *portainer.Team) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			team.ID = portainer.TeamID(id)
@@ -120,9 +65,3 @@ func (service *Service) Create(team *portainer.Team) error {
 		},
 	)
 }
-
-// DeleteTeam deletes a Team.
-func (service *Service) DeleteTeam(ID portainer.TeamID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/teammembership/teammembership.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
@@ -13,11 +14,7 @@ const BucketName = "team_membership"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // NewService creates a new instance of a service.
@@ -28,109 +25,52 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
 func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 	return ServiceTx{
-		service: service,
-		tx:      tx,
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
 }
 
-// TeamMembership returns a TeamMembership object by ID
-func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var membership portainer.TeamMembership
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &membership)
-	if err != nil {
-		return nil, err
-	}
-
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			memberships = append(memberships, *membership)
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.Connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.UserID == userID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.GetAll(
+	return memberships, service.Connection.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.TeamID == teamID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, membership)
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) Create(membership *portainer.TeamMembership) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			membership.ID = portainer.TeamMembershipID(id)
@@ -139,15 +79,9 @@ func (service *Service) Create(membership *portainer.TeamMembership) error {
 	)
 }
 
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -168,7 +102,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -188,7 +122,7 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 }
 
 func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.connection.DeleteAllObjects(
+	return service.Connection.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/teammembership/tx.go

@@ -4,111 +4,44 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 
 	"github.com/rs/zerolog/log"
 )
 
 type ServiceTx struct {
-	service *Service
-	tx      portainer.Transaction
-}
-
-func (service ServiceTx) BucketName() string {
-	return BucketName
-}
-
-// TeamMembership returns a TeamMembership object by ID
-func (service ServiceTx) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var membership portainer.TeamMembership
-	identifier := service.service.connection.ConvertToKey(int(ID))
-
-	err := service.tx.GetObject(BucketName, identifier, &membership)
-	if err != nil {
-		return nil, err
-	}
-
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service ServiceTx) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-
-	err := service.tx.GetAll(
-		BucketName,
-		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			memberships = append(memberships, *membership)
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+	dataservices.BaseDataServiceTx[portainer.TeamMembership, portainer.TeamMembershipID]
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
 func (service ServiceTx) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.tx.GetAll(
+	return memberships, service.Tx.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.UserID == userID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.UserID == userID
+		}),
+	)
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
 func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.tx.GetAll(
+	return memberships, service.Tx.GetAll(
 		BucketName,
 		&portainer.TeamMembership{},
-		func(obj interface{}) (interface{}, error) {
-			membership, ok := obj.(*portainer.TeamMembership)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
-				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
-			}
-
-			if membership.TeamID == teamID {
-				memberships = append(memberships, *membership)
-			}
-
-			return &portainer.TeamMembership{}, nil
-		})
-
-	return memberships, err
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service ServiceTx) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.UpdateObject(BucketName, identifier, membership)
+		dataservices.FilterFn(&memberships, func(e portainer.TeamMembership) bool {
+			return e.TeamID == teamID
+		}),
+	)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
-	return service.tx.CreateObject(
+	return service.Tx.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			membership.ID = portainer.TeamMembershipID(id)
@@ -117,15 +50,9 @@ func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
 	)
 }
 
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service ServiceTx) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	identifier := service.service.connection.ConvertToKey(int(ID))
-	return service.tx.DeleteObject(BucketName, identifier)
-}
-
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -146,7 +73,7 @@ func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) e
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {
@@ -166,7 +93,7 @@ func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) e
 }
 
 func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.TeamID, userID portainer.UserID) error {
-	return service.tx.DeleteAllObjects(
+	return service.Tx.DeleteAllObjects(
 		BucketName,
 		&portainer.TeamMembership{},
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/user/tx.go

@@ -0,0 +1,63 @@
+package user
+
+import (
+	"errors"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+type ServiceTx struct {
+	dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]
+}
+
+// UserByUsername returns a user by username.
+func (service ServiceTx) UserByUsername(username string) (*portainer.User, error) {
+	var u portainer.User
+
+	err := service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
+
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
+	}
+
+	if err == nil {
+		return nil, dserrors.ErrObjectNotFound
+	}
+
+	return nil, err
+}
+
+// UsersByRole return an array containing all the users with the specified role.
+func (service ServiceTx) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+
+	return users, service.Tx.GetAll(
+		BucketName,
+		&portainer.User{},
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
+}
+
+// CreateUser creates a new user.
+func (service ServiceTx) Create(user *portainer.User) error {
+	return service.Tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			user.ID = portainer.UserID(id)
+			user.Username = strings.ToLower(user.Username)
+
+			return int(user.ID), user
+		},
+	)
+}

api/dataservices/user/user.go

@@ -2,27 +2,19 @@ package user
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "users"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "users"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.User, portainer.UserID]
 }
 
 // NewService creates a new instance of a service.
@@ -33,48 +25,37 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// User returns a user by ID
-func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
-	var user portainer.User
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &user)
-	if err != nil {
-		return nil, err
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.User, portainer.UserID]{
+			Bucket:     BucketName,
+			Connection: service.Connection,
+			Tx:         tx,
+		},
 	}
-
-	return &user, nil
 }
 
 // UserByUsername returns a user by username.
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
-	var u *portainer.User
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var u portainer.User
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+		dataservices.FirstFn(&u, func(e portainer.User) bool {
+			return strings.EqualFold(e.Username, username)
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if strings.EqualFold(user.Username, username) {
-				u = user
-				return nil, stop
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return u, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &u, nil
 	}
 
 	if err == nil {
@@ -84,64 +65,22 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 	return nil, err
 }
 
-// Users return an array containing all the users.
-func (service *Service) Users() ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			users = append(users, *user)
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
-}
-
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.GetAll(
+	return users, service.Connection.GetAll(
 		BucketName,
 		&portainer.User{},
-		func(obj interface{}) (interface{}, error) {
-			user, ok := obj.(*portainer.User)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
-
-				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
-			}
-
-			if user.Role == role {
-				users = append(users, *user)
-			}
-
-			return &portainer.User{}, nil
-		})
-
-	return users, err
-}
-
-// UpdateUser saves a user.
-func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	user.Username = strings.ToLower(user.Username)
-	return service.connection.UpdateObject(BucketName, identifier, user)
+		dataservices.FilterFn(&users, func(e portainer.User) bool {
+			return e.Role == role
+		}),
+	)
 }
 
 // CreateUser creates a new user.
 func (service *Service) Create(user *portainer.User) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			user.ID = portainer.UserID(id)
@@ -151,9 +90,3 @@ func (service *Service) Create(user *portainer.User) error {
 		},
 	)
 }
-
-// DeleteUser deletes a user.
-func (service *Service) DeleteUser(ID portainer.UserID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}

api/dataservices/webhook/webhook.go

@@ -2,26 +2,18 @@ package webhook
 
 import (
 	"errors"
-	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "webhooks"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "webhooks"
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]
 }
 
 // NewService creates a new instance of a service.
@@ -32,70 +24,27 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}
 
 	return &Service{
-		connection: connection,
+		BaseDataService: dataservices.BaseDataService[portainer.Webhook, portainer.WebhookID]{
+			Bucket:     BucketName,
+			Connection: connection,
+		},
 	}, nil
 }
 
-// Webhooks returns an array of all webhooks
-func (service *Service) Webhooks() ([]portainer.Webhook, error) {
-	var webhooks = make([]portainer.Webhook, 0)
-
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			webhooks = append(webhooks, *webhook)
-
-			return &portainer.Webhook{}, nil
-		})
-
-	return webhooks, err
-}
-
-// Webhook returns a webhook by ID.
-func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
-	var webhook portainer.Webhook
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.GetObject(BucketName, identifier, &webhook)
-	if err != nil {
-		return nil, err
-	}
-
-	return &webhook, nil
-}
-
 // WebhookByResourceID returns a webhook by the ResourceID it is associated with.
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var w portainer.Webhook
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.ResourceID == ID
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.ResourceID == ID {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
@@ -107,29 +56,18 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 
 // WebhookByToken returns a webhook by the random token it is associated with.
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
-	var w *portainer.Webhook
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
+	var w portainer.Webhook
+
+	err := service.Connection.GetAll(
 		BucketName,
 		&portainer.Webhook{},
-		func(obj interface{}) (interface{}, error) {
-			webhook, ok := obj.(*portainer.Webhook)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+		dataservices.FirstFn(&w, func(e portainer.Webhook) bool {
+			return e.Token == token
+		}),
+	)
 
-				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
-			}
-
-			if webhook.Token == token {
-				w = webhook
-				return nil, stop
-			}
-
-			return &portainer.Webhook{}, nil
-		})
-
-	if errors.Is(err, stop) {
-		return w, nil
+	if errors.Is(err, dataservices.ErrStop) {
+		return &w, nil
 	}
 
 	if err == nil {
@@ -139,15 +77,9 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 	return nil, err
 }
 
-// DeleteWebhook deletes a webhook.
-func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
-}
-
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) Create(webhook *portainer.Webhook) error {
-	return service.connection.CreateObject(
+	return service.Connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
 			webhook.ID = portainer.WebhookID(id)
@@ -155,9 +87,3 @@ func (service *Service) Create(webhook *portainer.Webhook) error {
 		},
 	)
 }
-
-// UpdateWebhook update a webhook.
-func (service *Service) UpdateWebhook(ID portainer.WebhookID, webhook *portainer.Webhook) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, webhook)
-}

api/datastore/backup.go

@@ -4,186 +4,82 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"time"
 
-	"github.com/portainer/portainer/api/database/models"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/rs/zerolog/log"
 )
 
-var backupDefaults = struct {
-	backupDir string
-	commonDir string
-}{
-	"backups",
-	"common",
+func (store *Store) Backup() (string, error) {
+	if err := store.createBackupPath(); err != nil {
+		return "", err
+	}
+
+	backupFilename := store.backupFilename()
+	log.Info().Str("from", store.connection.GetDatabaseFilePath()).Str("to", backupFilename).Msgf("Backing up database")
+
+	// Close the store before backing up
+	err := store.Close()
+	if err != nil {
+		return "", fmt.Errorf("failed to close store before backup: %w", err)
+	}
+
+	err = store.fileService.Copy(store.connection.GetDatabaseFilePath(), backupFilename, true)
+	if err != nil {
+		return "", fmt.Errorf("failed to create backup file: %w", err)
+	}
+
+	// reopen the store
+	_, err = store.Open()
+	if err != nil {
+		return "", fmt.Errorf("failed to reopen store after backup: %w", err)
+	}
+
+	return backupFilename, nil
 }
 
-//
-// Backup Helpers
-//
+func (store *Store) Restore() error {
+	backupFilename := store.backupFilename()
+	return store.RestoreFromFile(backupFilename)
+}
 
-// createBackupFolders create initial folders for backups
-func (store *Store) createBackupFolders() {
-	// create common dir
-	commonDir := store.commonBackupDir()
-	if exists, _ := store.fileService.FileExists(commonDir); !exists {
-		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			log.Error().Err(err).Msg("error while creating common backup folder")
+func (store *Store) RestoreFromFile(backupFilename string) error {
+	store.Close()
+	if err := store.fileService.Copy(backupFilename, store.connection.GetDatabaseFilePath(), true); err != nil {
+		return fmt.Errorf("unable to restore backup file %q. err: %w", backupFilename, err)
+	}
+
+	log.Info().Str("from", backupFilename).Str("to", store.connection.GetDatabaseFilePath()).Msgf("database restored")
+
+	_, err := store.Open()
+	if err != nil {
+		return fmt.Errorf("unable to determine version of restored portainer backup file: %w", err)
+	}
+
+	// determine the db version
+	version, err := store.VersionService.Version()
+	if err != nil {
+		return fmt.Errorf("unable to determine restored database version. err: %w", err)
+	}
+
+	editionLabel := portainer.SoftwareEdition(version.Edition).GetEditionLabel()
+	log.Info().Msgf("Restored database version: Portainer %s %s", editionLabel, version.SchemaVersion)
+	return nil
+}
+
+func (store *Store) createBackupPath() error {
+	backupDir := path.Join(store.connection.GetStorePath(), "backups")
+	if exists, _ := store.fileService.FileExists(backupDir); !exists {
+		if err := os.MkdirAll(backupDir, 0700); err != nil {
+			return fmt.Errorf("unable to create backup folder: %w", err)
 		}
 	}
+	return nil
+}
+
+func (store *Store) backupFilename() string {
+	return path.Join(store.connection.GetStorePath(), "backups", store.connection.GetDatabaseFileName()+".bak")
 }
 
 func (store *Store) databasePath() string {
 	return store.connection.GetDatabaseFilePath()
 }
-
-func (store *Store) commonBackupDir() string {
-	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
-}
-
-func (store *Store) copyDBFile(from string, to string) error {
-	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
-
-	err := store.fileService.Copy(from, to, true)
-	if err != nil {
-		log.Error().Err(err).Msg("failed")
-	}
-
-	return err
-}
-
-// BackupOptions provide a helper to inject backup options
-type BackupOptions struct {
-	Version        string
-	BackupDir      string
-	BackupFileName string
-	BackupPath     string
-}
-
-// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// - db backup prior to version upgrade
-// - db rollback
-func getBackupRestoreOptions(backupDir string) *BackupOptions {
-	return &BackupOptions{
-		BackupDir:      backupDir, //connection.commonBackupDir(),
-		BackupFileName: beforePortainerVersionUpgradeBackup,
-	}
-}
-
-// Backup current database with default options
-func (store *Store) Backup(version *models.Version) (string, error) {
-	if version == nil {
-		return store.backupWithOptions(nil)
-	}
-
-	return store.backupWithOptions(&BackupOptions{
-		Version: version.SchemaVersion,
-	})
-}
-
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-	if options == nil {
-		options = &BackupOptions{}
-	}
-	if options.Version == "" {
-		v, err := store.VersionService.Version()
-		if err != nil {
-			options.Version = ""
-		}
-		options.Version = v.SchemaVersion
-	}
-	if options.BackupDir == "" {
-		options.BackupDir = store.commonBackupDir()
-	}
-	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), options.Version, time.Now().Format("20060102150405"))
-	}
-	if options.BackupPath == "" {
-		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-	}
-	return options
-}
-
-// BackupWithOptions backup current database with options
-func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
-	log.Info().Msg("creating DB backup")
-
-	store.createBackupFolders()
-
-	options = store.setupOptions(options)
-	dbPath := store.databasePath()
-
-	if err := store.Close(); err != nil {
-		return options.BackupPath, fmt.Errorf(
-			"error closing datastore before creating backup: %w",
-			err,
-		)
-	}
-
-	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
-		return options.BackupPath, err
-	}
-
-	if _, err := store.Open(); err != nil {
-		return options.BackupPath, fmt.Errorf(
-			"error opening datastore after creating backup: %w",
-			err,
-		)
-	}
-
-	return options.BackupPath, nil
-}
-
-// RestoreWithOptions previously saved backup for the current Edition  with options
-// Restore strategies:
-// - default: restore latest from current edition
-// - restore a specific
-func (store *Store) restoreWithOptions(options *BackupOptions) error {
-	options = store.setupOptions(options)
-
-	// Check if backup file exist before restoring
-	_, err := os.Stat(options.BackupPath)
-	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
-
-		return err
-	}
-
-	err = store.Close()
-	if err != nil {
-		log.Error().Err(err).Msg("error while closing store before restore")
-
-		return err
-	}
-
-	log.Info().Msg("restoring DB backup")
-	err = store.copyDBFile(options.BackupPath, store.databasePath())
-	if err != nil {
-		return err
-	}
-
-	_, err = store.Open()
-	return err
-}
-
-// RemoveWithOptions removes backup database based on supplied options
-func (store *Store) removeWithOptions(options *BackupOptions) error {
-	log.Info().Msg("removing DB backup")
-
-	options = store.setupOptions(options)
-	_, err := os.Stat(options.BackupPath)
-
-	if os.IsNotExist(err) {
-		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
-		return err
-	}
-
-	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
-	err = os.Remove(options.BackupPath)
-	if err != nil {
-		log.Error().Err(err).Msg("failed")
-		return err
-	}
-
-	return nil
-}

api/datastore/backup_test.go

@@ -2,106 +2,79 @@ package datastore
 
 import (
 	"fmt"
-	"os"
-	"path"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
+
+	"github.com/rs/zerolog/log"
 )
 
-func TestCreateBackupFolders(t *testing.T) {
-	_, store := MustNewTestStore(t, true, true)
-
-	connection := store.GetConnection()
-	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
-
-	if isFileExist(backupPath) {
-		t.Error("Expect backups folder to not exist")
-	}
-
-	store.createBackupFolders()
-	if !isFileExist(backupPath) {
-		t.Error("Expect backups folder to exist")
-	}
-}
-
 func TestStoreCreation(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
 	if store == nil {
-		t.Error("Expect to create a store")
+		t.Fatal("Expect to create a store")
 	}
 
-	if store.CheckCurrentEdition() != nil {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	if portainer.SoftwareEdition(v.Edition) != portainer.PortainerCE {
 		t.Error("Expect to get CE Edition")
 	}
+
+	if v.SchemaVersion != portainer.APIVersion {
+		t.Error("Expect to get APIVersion")
+	}
 }
 
 func TestBackup(t *testing.T) {
 	_, store := MustNewTestStore(t, true, true)
-	connection := store.GetConnection()
-
-	t.Run("Backup should create default db backup", func(t *testing.T) {
+	backupFileName := store.backupFilename()
+	t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) {
 		v := models.Version{
+			Edition:       int(portainer.PortainerCE),
 			SchemaVersion: portainer.APIVersion,
 		}
 		store.VersionService.UpdateVersion(&v)
-		store.backupWithOptions(nil)
+		store.Backup()
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%s.*", portainer.APIVersion))
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-
-	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-		store.backupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerVersionUpgradeBackup,
-			BackupDir:      store.commonBackupDir(),
-		})
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
 	})
 }
 
-func TestRemoveWithOptions(t *testing.T) {
-	_, store := MustNewTestStore(t, true, true)
+func TestRestore(t *testing.T) {
+	_, store := MustNewTestStore(t, true, false)
 
-	t.Run("successfully removes file if existent", func(t *testing.T) {
-		store.createBackupFolders()
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
+	t.Run(fmt.Sprintf("Basic Restore"), func(t *testing.T) {
+		// override and set initial db version and edition
+		updateEdition(store, portainer.PortainerCE)
+		updateVersion(store, "2.4")
 
-		filePath := path.Join(options.BackupDir, options.BackupFileName)
-		f, err := os.Create(filePath)
-		if err != nil {
-			t.Fatalf("file should be created; err=%s", err)
-		}
-		f.Close()
+		store.Backup()
+		updateVersion(store, "2.16")
+		testVersion(store, "2.16", t)
+		store.Restore()
 
-		err = store.removeWithOptions(options)
-		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
-		}
-
-		if isFileExist(f.Name()) {
-			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
-		}
+		// check if the restore is successful and the version is correct
+		testVersion(store, "2.4", t)
 	})
 
-	t.Run("fails to removes file if non-existent", func(t *testing.T) {
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
+	t.Run(fmt.Sprintf("Basic Restore After Multiple Backups"), func(t *testing.T) {
+		// override and set initial db version and edition
+		updateEdition(store, portainer.PortainerCE)
+		updateVersion(store, "2.4")
+		store.Backup()
+		updateVersion(store, "2.14")
+		updateVersion(store, "2.16")
+		testVersion(store, "2.16", t)
+		store.Restore()
 
-		err := store.removeWithOptions(options)
-		if err == nil {
-			t.Error("RemoveWithOptions should fail for non-existent file")
-		}
+		// check if the restore is successful and the version is correct
+		testVersion(store, "2.4", t)
 	})
 }

api/datastore/datastore.go

@@ -31,8 +31,14 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	if encryptionReq {
+		backupFilename, err := store.Backup()
+		if err != nil {
+			return false, fmt.Errorf("failed to backup database prior to encrypting: %w", err)
+		}
+
 		err = store.encryptDB()
 		if err != nil {
+			store.RestoreFromFile(backupFilename) // restore from backup if encryption fails
 			return false, err
 		}
 	}

api/datastore/datastore_test.go

@@ -150,7 +150,7 @@ func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType porta
 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
 
 	case portainer.EdgeAgentOnKubernetesEnvironment:
-		cs := chisel.NewService(store, nil)
+		cs := chisel.NewService(store, nil, nil)
 		expectedEndpoint = newEndpoint(endpointType, id, name, URL, tls)
 		edgeKey := cs.GenerateEdgeKey(URL, "", int(id))
 		expectedEndpoint.EdgeKey = edgeKey
@@ -316,7 +316,7 @@ func (store *Store) testCustomTemplates(t *testing.T) {
 
 	customTemplate.Create(expectedTemplate)
 
-	actualTemplate, err := customTemplate.CustomTemplate(expectedTemplate.ID)
+	actualTemplate, err := customTemplate.Read(expectedTemplate.ID)
 	is.NoError(err, "CustomTemplate should not return an error")
 	is.Equal(expectedTemplate, actualTemplate, "expected and actual template do not match")
 }
@@ -350,11 +350,11 @@ func (store *Store) testRegistries(t *testing.T) {
 	err = regService.Create(reg2)
 	is.NoError(err)
 
-	actualReg1, err := regService.Registry(reg1.ID)
+	actualReg1, err := regService.Read(reg1.ID)
 	is.NoError(err)
 	is.Equal(reg1, actualReg1, "registries differ")
 
-	actualReg2, err := regService.Registry(reg2.ID)
+	actualReg2, err := regService.Read(reg2.ID)
 	is.NoError(err)
 	is.Equal(reg2, actualReg2, "registries differ")
 }
@@ -406,11 +406,11 @@ func (store *Store) testTags(t *testing.T) {
 	err = tags.Create(tag2)
 	is.NoError(err, "Tags.Create should succeed")
 
-	actual, err := tags.Tag(tag1.ID)
+	actual, err := tags.Read(tag1.ID)
 	is.NoError(err, "tag1 should be found")
 	is.Equal(tag1, actual, "tags differ")
 
-	actual, err = tags.Tag(tag2.ID)
+	actual, err = tags.Read(tag2.ID)
 	is.NoError(err, "tag2 should be found")
 	is.Equal(tag2, actual, "tags differ")
 }

api/datastore/helpers_test.go

@@ -0,0 +1,68 @@
+package datastore
+
+import (
+	"path/filepath"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+// isFileExist is helper function to check for file existence
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
+func updateVersion(store *Store, v string) {
+	version, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	version.SchemaVersion = v
+
+	err = store.VersionService.UpdateVersion(version)
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+}
+
+func updateEdition(store *Store, edition portainer.SoftwareEdition) {
+	version, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	version.Edition = int(edition)
+
+	err = store.VersionService.UpdateVersion(version)
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+}
+
+// testVersion is a helper which tests current store version against wanted version
+func testVersion(store *Store, versionWant string, t *testing.T) {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+	if v.SchemaVersion != versionWant {
+		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
+	}
+}
+
+func testEdition(store *Store, editionWant portainer.SoftwareEdition, t *testing.T) {
+	v, err := store.VersionService.Version()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+	if portainer.SoftwareEdition(v.Edition) != editionWant {
+		t.Errorf("Expect store edition to be %s but was %s", editionWant.GetEditionLabel(), portainer.SoftwareEdition(v.Edition).GetEditionLabel())
+	}
+}

api/datastore/init.go

@@ -22,7 +22,6 @@ func (store *Store) Init() error {
 }
 
 func (store *Store) checkOrCreateDefaultSettings() error {
-
 	isDDExtention := false
 	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {
 		isDDExtention = true
@@ -73,6 +72,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
 		return store.Settings().UpdateSettings(settings)
 	}
+
 	return nil
 }
 
@@ -90,7 +90,7 @@ func (store *Store) checkOrCreateDefaultSSLSettings() error {
 }
 
 func (store *Store) checkOrCreateDefaultData() error {
-	groups, err := store.EndpointGroupService.EndpointGroups()
+	groups, err := store.EndpointGroupService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -110,5 +110,6 @@ func (store *Store) checkOrCreateDefaultData() error {
 			return err
 		}
 	}
+
 	return nil
 }

api/datastore/migrate_data.go

@@ -2,6 +2,7 @@ package datastore
 
 import (
 	"fmt"
+	"os"
 	"runtime/debug"
 
 	portainer "github.com/portainer/portainer/api"
@@ -15,8 +16,6 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
-
 func (store *Store) MigrateData() error {
 	updating, err := store.VersionService.IsUpdating()
 	if err != nil {
@@ -41,7 +40,7 @@ func (store *Store) MigrateData() error {
 	}
 
 	// before we alter anything in the DB, create a backup
-	backupPath, err := store.Backup(version)
+	_, err = store.Backup()
 	if err != nil {
 		return errors.Wrap(err, "while backing up database")
 	}
@@ -50,10 +49,10 @@ func (store *Store) MigrateData() error {
 	if err != nil {
 		err = errors.Wrap(err, "failed to migrate database")
 
-		log.Warn().Msg("migration failed, restoring database to previous version")
-		err = store.restoreWithOptions(&BackupOptions{BackupPath: backupPath})
-		if err != nil {
-			return errors.Wrap(err, "failed to restore database")
+		log.Warn().Err(err).Msg("migration failed, restoring database to previous version")
+		restoreErr := store.Restore()
+		if restoreErr != nil {
+			return errors.Wrap(restoreErr, "failed to restore database")
 		}
 
 		log.Info().Msg("database restored to previous version")
@@ -86,6 +85,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		AuthorizationService:    authorization.NewService(store),
 		EdgeStackService:        store.EdgeStackService,
 		EdgeJobService:          store.EdgeJobService,
+		TunnelServerService:     store.TunnelServerService,
 	}
 }
 
@@ -116,6 +116,11 @@ func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version *models
 		return err
 	}
 
+	// Special test code to simulate a failure (used by migrate_data_test.go).  Do not remove...
+	if os.Getenv("PORTAINER_TEST_MIGRATE_FAIL") == "FAIL" {
+		panic("test migration failure")
+	}
+
 	err = store.VersionService.StoreIsUpdating(false)
 	if err != nil {
 		return errors.Wrap(err, "failed to update the store")
@@ -134,9 +139,7 @@ func (store *Store) connectionRollback(force bool) error {
 		}
 	}
 
-	options := getBackupRestoreOptions(store.commonBackupDir())
-
-	err := store.restoreWithOptions(options)
+	err := store.Restore()
 	if err != nil {
 		return err
 	}

api/datastore/migrate_data_test.go

@@ -7,29 +7,20 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
+	"github.com/Masterminds/semver"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
+	"github.com/portainer/portainer/api/database/models"
+	"github.com/portainer/portainer/api/datastore/migrator"
+	"github.com/rs/zerolog/log"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/portainer/portainer/api/database/models"
-	"github.com/rs/zerolog/log"
 )
 
-// testVersion is a helper which tests current store version against wanted version
-func testVersion(store *Store, versionWant string, t *testing.T) {
-	v, err := store.VersionService.Version()
-	if err != nil {
-		t.Errorf("Expect store version to be %s but was %s with error: %s", versionWant, v.SchemaVersion, err)
-	}
-	if v.SchemaVersion != versionWant {
-		t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
-	}
-}
-
 func TestMigrateData(t *testing.T) {
-	snapshotTests := []struct {
+	tests := []struct {
 		testName           string
 		srcPath            string
 		wantPath           string
@@ -42,7 +33,7 @@ func TestMigrateData(t *testing.T) {
 			overrideInstanceId: true,
 		},
 	}
-	for _, test := range snapshotTests {
+	for _, test := range tests {
 		t.Run(test.testName, func(t *testing.T) {
 			err := migrateDBTestHelper(t, test.srcPath, test.wantPath, test.overrideInstanceId)
 			if err != nil {
@@ -55,147 +46,133 @@ func TestMigrateData(t *testing.T) {
 		})
 	}
 
-	// t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-	// 	newStore, store, teardown := MustNewTestStore(t, true, false)
-	// 	defer teardown()
+	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
+		newStore, store := MustNewTestStore(t, true, false)
+		if !newStore {
+			t.Error("Expect a new DB")
+		}
 
-	// 	if !newStore {
-	// 		t.Error("Expect a new DB")
-	// 	}
+		testVersion(store, portainer.APIVersion, t)
+		store.Close()
 
-	// 	testVersion(store, portainer.APIVersion, t)
-	// 	store.Close()
+		newStore, _ = store.Open()
+		if newStore {
+			t.Error("Expect store to NOT be new DB")
+		}
+	})
 
-	// 	newStore, _ = store.Open()
-	// 	if newStore {
-	// 		t.Error("Expect store to NOT be new DB")
-	// 	}
-	// })
+	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
+		_, store := MustNewTestStore(t, true, false)
+		store.VersionService.UpdateVersion(&models.Version{SchemaVersion: "1.0", Edition: int(portainer.PortainerCE)})
+		store.MigrateData()
 
-	// tests := []struct {
-	// 	version         string
-	// 	expectedVersion string
-	// }{
-	// 	{version: "1.24.1", expectedVersion: portainer.APIVersion},
-	// 	{version: "2.0.0", expectedVersion: portainer.APIVersion},
-	// }
-	// for _, tc := range tests {
-	// 	_, store, teardown := MustNewTestStore(t, true, true)
-	// 	defer teardown()
+		backupfilename := store.backupFilename()
+		if exists, _ := store.fileService.FileExists(backupfilename); !exists {
+			t.Errorf("Expect backup file to be created %s", backupfilename)
+		}
+	})
 
-	// 	// Setup data
-	// 	v := models.Version{SchemaVersion: tc.version}
-	// 	store.VersionService.UpdateVersion(&v)
+	t.Run("MigrateData should recover and restore backup during migration critical failure", func(t *testing.T) {
+		os.Setenv("PORTAINER_TEST_MIGRATE_FAIL", "FAIL")
 
-	// 	// Required roles by migrations 22.2
-	// 	store.RoleService.Create(&portainer.Role{ID: 1})
-	// 	store.RoleService.Create(&portainer.Role{ID: 2})
-	// 	store.RoleService.Create(&portainer.Role{ID: 3})
-	// 	store.RoleService.Create(&portainer.Role{ID: 4})
+		version := "2.15"
+		_, store := MustNewTestStore(t, true, false)
+		store.VersionService.UpdateVersion(&models.Version{SchemaVersion: version, Edition: int(portainer.PortainerCE)})
+		store.MigrateData()
 
-	// 	t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
-	// 		store.MigrateData()
-	// 		testVersion(store, tc.expectedVersion, t)
-	// 	})
+		store.Open()
+		testVersion(store, version, t)
+	})
 
-	// 	t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
-	// 		store.Rollback(true)
-	// 		store.Open()
-	// 		testVersion(store, tc.version, t)
-	// 	})
-	// }
+	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
+		_, store := MustNewTestStore(t, true, false)
+		store.VersionService.StoreIsUpdating(true)
+		store.MigrateData()
 
-	// t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
+		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
+		// only create a backup when the version changes.
+		backupfilename := store.backupFilename()
+		if exists, _ := store.fileService.FileExists(backupfilename); exists {
+			t.Errorf("Backup file should not exist for dirty database")
+		}
+	})
 
-	// 	v := models.Version{SchemaVersion: "1.24.1"}
-	// 	store.VersionService.UpdateVersion(&v)
+	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
+		_, store := MustNewTestStore(t, true, false)
 
-	// 	store.MigrateData()
+		// Set migrator the count to match our migrations array (simulate no changes).
+		// Should not create a backup
+		v, err := store.VersionService.Version()
+		if err != nil {
+			t.Errorf("Unable to read version from db: %s", err)
+			t.FailNow()
+		}
 
-	// 	testVersion(store, v.SchemaVersion, t)
-	// })
+		migratorParams := store.newMigratorParameters(v)
+		m := migrator.NewMigrator(migratorParams)
+		latestMigrations := m.LatestMigrations()
 
-	// t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+		if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
+			v.MigratorCount = len(latestMigrations.MigrationFuncs)
+			store.VersionService.UpdateVersion(v)
+		}
 
-	// 	v := models.Version{SchemaVersion: "0.0.0"}
-	// 	store.VersionService.UpdateVersion(&v)
+		store.MigrateData()
 
-	// 	store.MigrateData()
+		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
+		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
+		// only create a backup when the version changes.
+		backupfilename := store.backupFilename()
+		if exists, _ := store.fileService.FileExists(backupfilename); exists {
+			t.Errorf("Backup file should not exist for dirty database")
+		}
+	})
 
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+	t.Run("MigrateData should create backup on startup if portainer version matches db and migrationFuncs counts differ", func(t *testing.T) {
+		_, store := MustNewTestStore(t, true, false)
 
-	// 	if !isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should exist; file=%s", options.BackupPath)
-	// 	}
-	// })
+		// Set migrator count very large to simulate changes
+		// Should not create a backup
+		v, err := store.VersionService.Version()
+		if err != nil {
+			t.Errorf("Unable to read version from db: %s", err)
+			t.FailNow()
+		}
 
-	// t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
+		v.MigratorCount = 1000
+		store.VersionService.UpdateVersion(v)
+		store.MigrateData()
 
-	// 	store.VersionService.StoreIsUpdating(true)
-
-	// 	store.MigrateData()
-
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
-
-	// 	if isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-	// 	}
-	// })
-
-	// t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-	// 	_, store, teardown := MustNewTestStore(t, false, true)
-	// 	defer teardown()
-
-	// 	store.MigrateData()
-
-	// 	options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
-
-	// 	if isFileExist(options.BackupPath) {
-	// 		t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-	// 	}
-	// })
-}
-
-func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store := MustNewTestStore(t, false, true)
-
-	options := getBackupRestoreOptions(store.commonBackupDir())
-
-	wantDir := store.commonBackupDir()
-	if !strings.HasSuffix(options.BackupDir, wantDir) {
-		log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir")
-	}
-
-	wantFilename := "portainer.db.bak"
-	if options.BackupFileName != wantFilename {
-		log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file")
-	}
+		// If you get an error, it usually means that the backup folder doesn't exist (no backups). Expected!
+		// If the backup file is not blank, then it means a backup was created.  We don't want that because we
+		// only create a backup when the version changes.
+		backupfilename := store.backupFilename()
+		if exists, _ := store.fileService.FileExists(backupfilename); !exists {
+			t.Errorf("DB backup should exist and there should be no error")
+		}
+	})
 }
 
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := models.Version{SchemaVersion: "2.4.0"}
-		_, store := MustNewTestStore(t, true, false)
+		version := "2.11"
 
-		err := store.VersionService.UpdateVersion(&version)
-		if err != nil {
-			t.Errorf("Failed updating version: %v", err)
+		v := models.Version{
+			SchemaVersion: version,
 		}
 
-		_, err = store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		_, store := MustNewTestStore(t, false, false)
+		store.VersionService.UpdateVersion(&v)
+
+		_, err := store.Backup()
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
 
-		// Change the current version
-		version2 := models.Version{SchemaVersion: "2.6.0"}
-		err = store.VersionService.UpdateVersion(&version2)
+		v.SchemaVersion = "2.14"
+		// Change the current edition
+		err = store.VersionService.UpdateVersion(&v)
 		if err != nil {
 			log.Fatal().Err(err).Msg("")
 		}
@@ -207,26 +184,45 @@ func TestRollback(t *testing.T) {
 			return
 		}
 
-		_, err = store.Open()
+		store.Open()
+		testVersion(store, version, t)
+	})
+
+	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
+		version := "2.15"
+
+		v := models.Version{
+			SchemaVersion: version,
+			Edition:       int(portainer.PortainerCE),
+		}
+
+		_, store := MustNewTestStore(t, true, false)
+		store.VersionService.UpdateVersion(&v)
+
+		_, err := store.Backup()
 		if err != nil {
-			t.Logf("Open failed: %s", err)
+			log.Fatal().Err(err).Msg("")
+		}
+
+		v.SchemaVersion = "2.14"
+		// Change the current edition
+		err = store.VersionService.UpdateVersion(&v)
+		if err != nil {
+			log.Fatal().Err(err).Msg("")
+		}
+
+		err = store.Rollback(true)
+		if err != nil {
+			t.Logf("Rollback failed: %s", err)
 			t.Fail()
 			return
 		}
 
-		testVersion(store, version.SchemaVersion, t)
+		store.Open()
+		testVersion(store, version, t)
 	})
 }
 
-// isFileExist is helper function to check for file existence
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}
-
 // migrateDBTestHelper loads a json representation of a bolt database from srcPath,
 // parses it into a database, runs a migration on that database, and then
 // compares it with an expected output database.

api/datastore/migrate_dbversion33_test.go

@@ -31,22 +31,22 @@ func TestMigrateStackEntryPoint(t *testing.T) {
 		assert.NoError(t, err, "failed to create stack")
 	}
 
-	s, err := stackService.Stack(1)
+	s, err := stackService.Read(1)
 	assert.NoError(t, err)
 	assert.Nil(t, s.GitConfig, "first stack should not have git config")
 
-	s, err = stackService.Stack(2)
+	s, err = stackService.Read(2)
 	assert.NoError(t, err)
 	assert.Equal(t, "", s.GitConfig.ConfigFilePath, "not migrated yet migrated")
 
 	err = migrator.MigrateStackEntryPoint(stackService)
 	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
 
-	s, err = stackService.Stack(1)
+	s, err = stackService.Read(1)
 	assert.NoError(t, err)
 	assert.Nil(t, s.GitConfig, "first stack should not have git config")
 
-	s, err = stackService.Stack(2)
+	s, err = stackService.Read(2)
 	assert.NoError(t, err)
 	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")
 }

api/datastore/migrate_legacyversion.go

@@ -1,7 +1,7 @@
 package datastore
 
 import (
-	portaineree "github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices"
 )
@@ -72,7 +72,7 @@ func dbVersionToSemanticVersion(dbVersion int) string {
 func (store *Store) getOrMigrateLegacyVersion() (*models.Version, error) {
 	// Very old versions of portainer did not have a version bucket, lets set some defaults
 	dbVersion := 24
-	edition := int(portaineree.PortainerCE)
+	edition := int(portainer.PortainerCE)
 	instanceId := ""
 
 	// If we already have a version key, we don't need to migrate

api/datastore/migrator/migrate_dbversion100.go

@@ -2,7 +2,11 @@ package migrator
 
 import (
 	"os"
+	"time"
 
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/chisel/crypto"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/rs/zerolog/log"
 )
 
@@ -20,10 +24,133 @@ func (m *Migrator) migrateDockerDesktopExtentionSetting() error {
 	}
 
 	settings.IsDockerDesktopExtension = isDDExtension
-	err = m.settingsService.UpdateSettings(settings)
+
+	return m.settingsService.UpdateSettings(settings)
+}
+
+func (m *Migrator) convertSeedToPrivateKeyForDB100() error {
+	var serverInfo *portainer.TunnelServerInfo
+
+	serverInfo, err := m.TunnelServerService.Info()
+	if err != nil {
+		if dataservices.IsErrObjectNotFound(err) {
+			log.Info().Msg("ServerInfo object not found")
+			return nil
+		}
+		log.Error().
+			Err(err).
+			Msg("Failed to read ServerInfo from DB")
+		return err
+	}
+
+	if serverInfo.PrivateKeySeed != "" {
+		key, err := crypto.GenerateGo119CompatibleKey(serverInfo.PrivateKeySeed)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to read ServerInfo from DB")
+			return err
+		}
+
+		err = m.fileService.StoreChiselPrivateKey(key)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("Failed to save Chisel private key to disk")
+			return err
+		}
+	} else {
+		log.Info().Msg("PrivateKeySeed is blank")
+	}
+
+	serverInfo.PrivateKeySeed = ""
+	err = m.TunnelServerService.UpdateInfo(serverInfo)
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("Failed to clean private key seed in DB")
+	} else {
+		log.Info().Msg("Success to migrate private key seed to private key file")
+	}
+	return err
+}
+
+func (m *Migrator) updateEdgeStackStatusForDB100() error {
+	log.Info().Msg("update edge stack status to have deployment steps")
+
+	edgeStacks, err := m.edgeStackService.EdgeStacks()
 	if err != nil {
 		return err
 	}
 
+	for _, edgeStack := range edgeStacks {
+
+		for environmentID, environmentStatus := range edgeStack.Status {
+			// skip if status is already updated
+			if len(environmentStatus.Status) > 0 {
+				continue
+			}
+
+			statusArray := []portainer.EdgeStackDeploymentStatus{}
+			if environmentStatus.Details.Pending {
+				statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{
+					Type: portainer.EdgeStackStatusPending,
+					Time: time.Now().Unix(),
+				})
+			}
+
+			if environmentStatus.Details.Acknowledged {
+				statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{
+					Type: portainer.EdgeStackStatusAcknowledged,
+					Time: time.Now().Unix(),
+				})
+			}
+
+			if environmentStatus.Details.Error {
+				statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{
+					Type:  portainer.EdgeStackStatusError,
+					Error: environmentStatus.Error,
+					Time:  time.Now().Unix(),
+				})
+			}
+
+			if environmentStatus.Details.Ok {
+				statusArray = append(statusArray,
+					portainer.EdgeStackDeploymentStatus{
+						Type: portainer.EdgeStackStatusDeploymentReceived,
+						Time: time.Now().Unix(),
+					},
+					portainer.EdgeStackDeploymentStatus{
+						Type: portainer.EdgeStackStatusRunning,
+						Time: time.Now().Unix(),
+					},
+				)
+			}
+
+			if environmentStatus.Details.ImagesPulled {
+				statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{
+					Type: portainer.EdgeStackStatusImagesPulled,
+					Time: time.Now().Unix(),
+				})
+			}
+
+			if environmentStatus.Details.Remove {
+				statusArray = append(statusArray, portainer.EdgeStackDeploymentStatus{
+					Type: portainer.EdgeStackStatusRemoving,
+					Time: time.Now().Unix(),
+				})
+			}
+
+			environmentStatus.Status = statusArray
+
+			edgeStack.Status[environmentID] = environmentStatus
+		}
+
+		err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

api/datastore/migrator/migrate_dbversion17.go

@@ -9,7 +9,7 @@ import (
 func (m *Migrator) updateUsersToDBVersion18() error {
 	log.Info().Msg("updating users")
 
-	legacyUsers, err := m.userService.Users()
+	legacyUsers, err := m.userService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -33,7 +33,7 @@ func (m *Migrator) updateUsersToDBVersion18() error {
 			portainer.OperationPortainerUserMemberships:         true,
 		}
 
-		err = m.userService.UpdateUser(user.ID, &user)
+		err = m.userService.Update(user.ID, &user)
 		if err != nil {
 			return err
 		}
@@ -77,7 +77,7 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
 func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 	log.Info().Msg("updating endpoint groups")
 
-	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
+	legacyEndpointGroups, err := m.endpointGroupService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -97,7 +97,7 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 			}
 		}
 
-		err = m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		err = m.endpointGroupService.Update(endpointGroup.ID, &endpointGroup)
 		if err != nil {
 			return err
 		}
@@ -109,7 +109,7 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 func (m *Migrator) updateRegistriesToDBVersion18() error {
 	log.Info().Msg("updating registries")
 
-	legacyRegistries, err := m.registryService.Registries()
+	legacyRegistries, err := m.registryService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -125,7 +125,7 @@ func (m *Migrator) updateRegistriesToDBVersion18() error {
 			registry.TeamAccessPolicies[teamID] = portainer.AccessPolicy{}
 		}
 
-		err = m.registryService.UpdateRegistry(registry.ID, &registry)
+		err = m.registryService.Update(registry.ID, &registry)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion20.go

@@ -10,7 +10,7 @@ import (
 func (m *Migrator) updateResourceControlsToDBVersion22() error {
 	log.Info().Msg("updating resource controls")
 
-	legacyResourceControls, err := m.resourceControlService.ResourceControls()
+	legacyResourceControls, err := m.resourceControlService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -18,7 +18,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 	for _, resourceControl := range legacyResourceControls {
 		resourceControl.AdministratorsOnly = false
 
-		err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, &resourceControl)
+		err := m.resourceControlService.Update(resourceControl.ID, &resourceControl)
 		if err != nil {
 			return err
 		}
@@ -30,7 +30,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 	log.Info().Msg("updating users and roles")
 
-	legacyUsers, err := m.userService.Users()
+	legacyUsers, err := m.userService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -42,47 +42,47 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 
 	for _, user := range legacyUsers {
 		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
-		err = m.userService.UpdateUser(user.ID, &user)
+		err = m.userService.Update(user.ID, &user)
 		if err != nil {
 			return err
 		}
 	}
 
-	endpointAdministratorRole, err := m.roleService.Role(portainer.RoleID(1))
+	endpointAdministratorRole, err := m.roleService.Read(portainer.RoleID(1))
 	if err != nil {
 		return err
 	}
 	endpointAdministratorRole.Priority = 1
 	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
 
-	err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
+	err = m.roleService.Update(endpointAdministratorRole.ID, endpointAdministratorRole)
 
-	helpDeskRole, err := m.roleService.Role(portainer.RoleID(2))
+	helpDeskRole, err := m.roleService.Read(portainer.RoleID(2))
 	if err != nil {
 		return err
 	}
 	helpDeskRole.Priority = 2
 	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
+	err = m.roleService.Update(helpDeskRole.ID, helpDeskRole)
 
-	standardUserRole, err := m.roleService.Role(portainer.RoleID(3))
+	standardUserRole, err := m.roleService.Read(portainer.RoleID(3))
 	if err != nil {
 		return err
 	}
 	standardUserRole.Priority = 3
 	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
+	err = m.roleService.Update(standardUserRole.ID, standardUserRole)
 
-	readOnlyUserRole, err := m.roleService.Role(portainer.RoleID(4))
+	readOnlyUserRole, err := m.roleService.Read(portainer.RoleID(4))
 	if err != nil {
 		return err
 	}
 	readOnlyUserRole.Priority = 4
 	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
+	err = m.roleService.Update(readOnlyUserRole.ID, readOnlyUserRole)
 	if err != nil {
 		return err
 	}

api/datastore/migrator/migrate_dbversion22.go

@@ -9,7 +9,7 @@ import (
 func (m *Migrator) updateTagsToDBVersion23() error {
 	log.Info().Msg("updating tags")
 
-	tags, err := m.tagService.Tags()
+	tags, err := m.tagService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -17,7 +17,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 	for _, tag := range tags {
 		tag.EndpointGroups = make(map[portainer.EndpointGroupID]bool)
 		tag.Endpoints = make(map[portainer.EndpointID]bool)
-		err = m.tagService.UpdateTag(tag.ID, &tag)
+		err = m.tagService.Update(tag.ID, &tag)
 		if err != nil {
 			return err
 		}
@@ -29,7 +29,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 	log.Info().Msg("updating endpoints and endpoint groups")
 
-	tags, err := m.tagService.Tags()
+	tags, err := m.tagService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -70,7 +70,7 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 		}
 	}
 
-	endpointGroups, err := m.endpointGroupService.EndpointGroups()
+	endpointGroups, err := m.endpointGroupService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -85,14 +85,14 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 			}
 		}
 		endpointGroup.TagIDs = endpointGroupTags
-		err = m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		err = m.endpointGroupService.Update(endpointGroup.ID, &endpointGroup)
 		if err != nil {
 			return err
 		}
 	}
 
 	for _, tag := range tagsNameMap {
-		err = m.tagService.UpdateTag(tag.ID, &tag)
+		err = m.tagService.Update(tag.ID, &tag)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion23.go

@@ -24,7 +24,7 @@ func (m *Migrator) updateSettingsToDB24() error {
 func (m *Migrator) updateStacksToDB24() error {
 	log.Info().Msg("updating stacks")
 
-	stacks, err := m.stackService.Stacks()
+	stacks, err := m.stackService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -32,7 +32,7 @@ func (m *Migrator) updateStacksToDB24() error {
 	for idx := range stacks {
 		stack := &stacks[idx]
 		stack.Status = portainer.StackStatusActive
-		err := m.stackService.UpdateStack(stack.ID, stack)
+		err := m.stackService.Update(stack.ID, stack)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion26.go

@@ -11,7 +11,7 @@ import (
 func (m *Migrator) updateStackResourceControlToDB27() error {
 	log.Info().Msg("updating stack resource controls")
 
-	resourceControls, err := m.resourceControlService.ResourceControls()
+	resourceControls, err := m.resourceControlService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -34,7 +34,7 @@ func (m *Migrator) updateStackResourceControlToDB27() error {
 
 		resource.ResourceID = stackutils.ResourceControlID(stack.EndpointID, stack.Name)
 
-		err = m.resourceControlService.UpdateResourceControl(resource.ID, &resource)
+		err = m.resourceControlService.Update(resource.ID, &resource)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion31.go

@@ -41,7 +41,7 @@ func (m *Migrator) migrateDBVersionToDB32() error {
 func (m *Migrator) updateRegistriesToDB32() error {
 	log.Info().Msg("updating registries")
 
-	registries, err := m.registryService.Registries()
+	registries, err := m.registryService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -77,7 +77,7 @@ func (m *Migrator) updateRegistriesToDB32() error {
 				Namespaces:         []string{},
 			}
 		}
-		m.registryService.UpdateRegistry(registry.ID, &registry)
+		m.registryService.Update(registry.ID, &registry)
 	}
 	return nil
 }
@@ -111,7 +111,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 	// we only have one migrated registry entry. Duplicates will be removed
 	// if they exist and which has been happening due to earlier migration bugs
 	migrated := false
-	registries, _ := m.registryService.Registries()
+	registries, _ := m.registryService.ReadAll()
 	for _, r := range registries {
 		if r.Type == registry.Type &&
 			r.Name == registry.Name &&
@@ -123,7 +123,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 				migrated = true
 			} else {
 				// delete subsequent duplicates
-				m.registryService.DeleteRegistry(portainer.RegistryID(r.ID))
+				m.registryService.Delete(portainer.RegistryID(r.ID))
 			}
 		}
 	}
@@ -180,7 +180,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 		return fmt.Errorf("failed fetching environments: %w", err)
 	}
 
-	resourceControls, err := m.resourceControlService.ResourceControls()
+	resourceControls, err := m.resourceControlService.ReadAll()
 	if err != nil {
 		return fmt.Errorf("failed fetching resource controls: %w", err)
 	}
@@ -228,12 +228,12 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
 			resourceControl.ResourceID = newResourceID
 
-			err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
+			err := m.resourceControlService.Update(resourceControl.ID, resourceControl)
 			if err != nil {
 				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
 			}
 		} else {
-			err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			err := m.resourceControlService.Delete(resourceControl.ID)
 			if err != nil {
 				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
 			}

api/datastore/migrator/migrate_dbversion33.go

@@ -14,7 +14,7 @@ func (m *Migrator) migrateDBVersionToDB34() error {
 
 // MigrateStackEntryPoint exported for testing
 func MigrateStackEntryPoint(stackService dataservices.StackService) error {
-	stacks, err := stackService.Stacks()
+	stacks, err := stackService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -26,7 +26,7 @@ func MigrateStackEntryPoint(stackService dataservices.StackService) error {
 		}
 
 		stack.GitConfig.ConfigFilePath = stack.EntryPoint
-		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
+		if err := stackService.Update(stack.ID, stack); err != nil {
 			return err
 		}
 	}

api/datastore/migrator/migrate_dbversion35.go

@@ -16,7 +16,7 @@ func (m *Migrator) migrateDBVersionToDB36() error {
 func (m *Migrator) migrateUsersToDB36() error {
 	log.Info().Msg("updating user authorizations")
 
-	users, err := m.userService.Users()
+	users, err := m.userService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -27,7 +27,7 @@ func (m *Migrator) migrateUsersToDB36() error {
 		currentAuthorizations[portainer.OperationPortainerUserCreateToken] = true
 		currentAuthorizations[portainer.OperationPortainerUserRevokeToken] = true
 		user.PortainerAuthorizations = currentAuthorizations
-		err = m.userService.UpdateUser(user.ID, &user)
+		err = m.userService.Update(user.ID, &user)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion71.go

@@ -9,7 +9,7 @@ import (
 func (m *Migrator) migrateDBVersionToDB71() error {
 	log.Info().Msg("removing orphaned snapshots")
 
-	snapshots, err := m.snapshotService.Snapshots()
+	snapshots, err := m.snapshotService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -26,7 +26,7 @@ func (m *Migrator) migrateDBVersionToDB71() error {
 
 		log.Debug().Int("endpoint_id", int(s.EndpointID)).Msg("removing snapshot")
 
-		err = m.snapshotService.DeleteSnapshot(s.EndpointID)
+		err = m.snapshotService.Delete(s.EndpointID)
 		if err != nil {
 			return err
 		}

api/datastore/migrator/migrate_dbversion80.go

@@ -78,13 +78,13 @@ func (m *Migrator) updateEdgeStackStatusForDB80() error {
 			switch status.Type {
 			case portainer.EdgeStackStatusPending:
 				status.Details.Pending = true
-			case portainer.EdgeStackStatusOk:
+			case portainer.EdgeStackStatusDeploymentReceived:
 				status.Details.Ok = true
 			case portainer.EdgeStackStatusError:
 				status.Details.Error = true
 			case portainer.EdgeStackStatusAcknowledged:
 				status.Details.Acknowledged = true
-			case portainer.EdgeStackStatusRemove:
+			case portainer.EdgeStackStatusRemoved:
 				status.Details.Remove = true
 			case portainer.EdgeStackStatusRemoteUpdateSuccess:
 				status.Details.RemoteUpdateSuccess = true

api/datastore/migrator/migrate_dbversion90.go

@@ -22,7 +22,7 @@ func (m *Migrator) migrateDBVersionToDB90() error {
 func (m *Migrator) updateEdgeStackStatusForDB90() error {
 	log.Info().Msg("clean up deleted endpoints from edge jobs")
 
-	edgeJobs, err := m.edgeJobService.EdgeJobs()
+	edgeJobs, err := m.edgeJobService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -33,7 +33,7 @@ func (m *Migrator) updateEdgeStackStatusForDB90() error {
 			if dataservices.IsErrObjectNotFound(err) {
 				delete(edgeJob.Endpoints, endpointId)
 
-				err = m.edgeJobService.UpdateEdgeJob(edgeJob.ID, &edgeJob)
+				err = m.edgeJobService.Update(edgeJob.ID, &edgeJob)
 				if err != nil {
 					return err
 				}
@@ -47,7 +47,7 @@ func (m *Migrator) updateEdgeStackStatusForDB90() error {
 func (m *Migrator) updateUserThemeForDB90() error {
 	log.Info().Msg("updating existing user theme settings")
 
-	users, err := m.userService.Users()
+	users, err := m.userService.ReadAll()
 	if err != nil {
 		return err
 	}
@@ -58,7 +58,7 @@ func (m *Migrator) updateUserThemeForDB90() error {
 			user.ThemeSettings.Color = user.UserTheme
 		}
 
-		if err := m.userService.UpdateUser(user.ID, user); err != nil {
+		if err := m.userService.Update(user.ID, user); err != nil {
 			return err
 		}
 	}

api/datastore/migrator/migrator.go

@@ -3,15 +3,12 @@ package migrator
 import (
 	"errors"
 
-	"github.com/portainer/portainer/api/dataservices/edgejob"
-	"github.com/portainer/portainer/api/dataservices/edgestack"
-
 	"github.com/Masterminds/semver"
-	"github.com/rs/zerolog/log"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices/dockerhub"
+	"github.com/portainer/portainer/api/dataservices/edgejob"
+	"github.com/portainer/portainer/api/dataservices/edgestack"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
@@ -26,9 +23,11 @@ import (
 	"github.com/portainer/portainer/api/dataservices/stack"
 	"github.com/portainer/portainer/api/dataservices/tag"
 	"github.com/portainer/portainer/api/dataservices/teammembership"
+	"github.com/portainer/portainer/api/dataservices/tunnelserver"
 	"github.com/portainer/portainer/api/dataservices/user"
 	"github.com/portainer/portainer/api/dataservices/version"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/rs/zerolog/log"
 )
 
 type (
@@ -58,6 +57,7 @@ type (
 		dockerhubService        *dockerhub.Service
 		edgeStackService        *edgestack.Service
 		edgeJobService          *edgejob.Service
+		TunnelServerService     *tunnelserver.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -84,6 +84,7 @@ type (
 		DockerhubService        *dockerhub.Service
 		EdgeStackService        *edgestack.Service
 		EdgeJobService          *edgejob.Service
+		TunnelServerService     *tunnelserver.Service
 	}
 )
 
@@ -112,6 +113,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		dockerhubService:        parameters.DockerhubService,
 		edgeStackService:        parameters.EdgeStackService,
 		edgeJobService:          parameters.EdgeJobService,
+		TunnelServerService:     parameters.TunnelServerService,
 	}
 
 	migrator.initMigrations()
@@ -146,6 +148,17 @@ func (m *Migrator) LatestMigrations() Migrations {
 	return m.migrations[len(m.migrations)-1]
 }
 
+func (m *Migrator) GetMigratorCountOfCurrentAPIVersion() int {
+	migratorCount := 0
+	latestMigrations := m.LatestMigrations()
+
+	if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
+		migratorCount = len(latestMigrations.MigrationFuncs)
+	}
+
+	return migratorCount
+}
+
 // !NOTE: Migration funtions should ideally be idempotent.
 // !      Which simply means the function can run over the same data many times but only transform it once.
 // !      In practice this really just means an extra check or two to ensure we're not destroying valid data.
@@ -210,8 +223,11 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
 	m.addMigrations("2.17", m.migrateDBVersionToDB80)
 	m.addMigrations("2.18", m.migrateDBVersionToDB90)
-
-	m.addMigrations("2.19", m.migrateDockerDesktopExtentionSetting)
+	m.addMigrations("2.19",
+		m.convertSeedToPrivateKeyForDB100,
+		m.migrateDockerDesktopExtentionSetting,
+		m.updateEdgeStackStatusForDB100,
+	)
 
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.

api/datastore/services.go

@@ -392,7 +392,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	backup := storeExport{}
 
-	if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
+	if c, err := store.CustomTemplate().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Custom Templates")
 		}
@@ -400,7 +400,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.CustomTemplate = c
 	}
 
-	if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
+	if e, err := store.EdgeGroup().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Edge Groups")
 		}
@@ -408,7 +408,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.EdgeGroup = e
 	}
 
-	if e, err := store.EdgeJob().EdgeJobs(); err != nil {
+	if e, err := store.EdgeJob().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Edge Jobs")
 		}
@@ -432,7 +432,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Endpoint = e
 	}
 
-	if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
+	if e, err := store.EndpointGroup().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Endpoint Groups")
 		}
@@ -456,7 +456,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Extensions = r
 	}
 
-	if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
+	if r, err := store.HelmUserRepository().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Helm User Repositories")
 		}
@@ -464,7 +464,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.HelmUserRepository = r
 	}
 
-	if r, err := store.Registry().Registries(); err != nil {
+	if r, err := store.Registry().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Registries")
 		}
@@ -472,7 +472,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Registry = r
 	}
 
-	if c, err := store.ResourceControl().ResourceControls(); err != nil {
+	if c, err := store.ResourceControl().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Resource Controls")
 		}
@@ -480,7 +480,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.ResourceControl = c
 	}
 
-	if role, err := store.Role().Roles(); err != nil {
+	if role, err := store.Role().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Roles")
 		}
@@ -504,7 +504,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Settings = *settings
 	}
 
-	if snapshot, err := store.Snapshot().Snapshots(); err != nil {
+	if snapshot, err := store.Snapshot().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Snapshots")
 		}
@@ -520,7 +520,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.SSLSettings = *settings
 	}
 
-	if t, err := store.Stack().Stacks(); err != nil {
+	if t, err := store.Stack().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Stacks")
 		}
@@ -528,7 +528,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Stack = t
 	}
 
-	if t, err := store.Tag().Tags(); err != nil {
+	if t, err := store.Tag().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Tags")
 		}
@@ -536,7 +536,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.Tag = t
 	}
 
-	if t, err := store.TeamMembership().TeamMemberships(); err != nil {
+	if t, err := store.TeamMembership().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Team Memberships")
 		}
@@ -544,7 +544,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.TeamMembership = t
 	}
 
-	if t, err := store.Team().Teams(); err != nil {
+	if t, err := store.Team().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Teams")
 		}
@@ -560,7 +560,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.TunnelServer = *info
 	}
 
-	if users, err := store.User().Users(); err != nil {
+	if users, err := store.User().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Users")
 		}
@@ -568,7 +568,7 @@ func (store *Store) Export(filename string) (err error) {
 		backup.User = users
 	}
 
-	if webhooks, err := store.Webhook().Webhooks(); err != nil {
+	if webhooks, err := store.Webhook().ReadAll(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
 			log.Error().Err(err).Msg("exporting Webhooks")
 		}
@@ -611,15 +611,15 @@ func (store *Store) Import(filename string) (err error) {
 	store.Version().UpdateVersion(&backup.Version)
 
 	for _, v := range backup.CustomTemplate {
-		store.CustomTemplate().UpdateCustomTemplate(v.ID, &v)
+		store.CustomTemplate().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.EdgeGroup {
-		store.EdgeGroup().UpdateEdgeGroup(v.ID, &v)
+		store.EdgeGroup().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.EdgeJob {
-		store.EdgeJob().UpdateEdgeJob(v.ID, &v)
+		store.EdgeJob().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.EdgeStack {
@@ -631,7 +631,7 @@ func (store *Store) Import(filename string) (err error) {
 	}
 
 	for _, v := range backup.EndpointGroup {
-		store.EndpointGroup().UpdateEndpointGroup(v.ID, &v)
+		store.EndpointGroup().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.EndpointRelation {
@@ -639,54 +639,54 @@ func (store *Store) Import(filename string) (err error) {
 	}
 
 	for _, v := range backup.HelmUserRepository {
-		store.HelmUserRepository().UpdateHelmUserRepository(v.ID, &v)
+		store.HelmUserRepository().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.Registry {
-		store.Registry().UpdateRegistry(v.ID, &v)
+		store.Registry().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.ResourceControl {
-		store.ResourceControl().UpdateResourceControl(v.ID, &v)
+		store.ResourceControl().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.Role {
-		store.Role().UpdateRole(v.ID, &v)
+		store.Role().Update(v.ID, &v)
 	}
 
 	store.Settings().UpdateSettings(&backup.Settings)
 	store.SSLSettings().UpdateSettings(&backup.SSLSettings)
 
 	for _, v := range backup.Snapshot {
-		store.Snapshot().UpdateSnapshot(&v)
+		store.Snapshot().Update(v.EndpointID, &v)
 	}
 
 	for _, v := range backup.Stack {
-		store.Stack().UpdateStack(v.ID, &v)
+		store.Stack().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.Tag {
-		store.Tag().UpdateTag(v.ID, &v)
+		store.Tag().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.TeamMembership {
-		store.TeamMembership().UpdateTeamMembership(v.ID, &v)
+		store.TeamMembership().Update(v.ID, &v)
 	}
 
 	for _, v := range backup.Team {
-		store.Team().UpdateTeam(v.ID, &v)
+		store.Team().Update(v.ID, &v)
 	}
 
 	store.TunnelServer().UpdateInfo(&backup.TunnelServer)
 
 	for _, user := range backup.User {
-		if err := store.User().UpdateUser(user.ID, &user); err != nil {
+		if err := store.User().Update(user.ID, &user); err != nil {
 			log.Debug().Str("user", fmt.Sprintf("%+v", user)).Err(err).Msg("failed to update the user in the database")
 		}
 	}
 
 	for _, v := range backup.Webhook {
-		store.Webhook().UpdateWebhook(v.ID, &v)
+		store.Webhook().Update(v.ID, &v)
 	}
 
 	return store.connection.RestoreMetadata(backup.Metadata)

api/datastore/services_tx.go

@@ -78,6 +78,10 @@ func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
 
 func (tx *StoreTx) Team() dataservices.TeamService                 { return nil }
 func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }
-func (tx *StoreTx) User() dataservices.UserService                 { return nil }
-func (tx *StoreTx) Version() dataservices.VersionService           { return nil }
-func (tx *StoreTx) Webhook() dataservices.WebhookService           { return nil }
+
+func (tx *StoreTx) User() dataservices.UserService {
+	return tx.store.UserService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) Version() dataservices.VersionService { return nil }
+func (tx *StoreTx) Webhook() dataservices.WebhookService { return nil }

api/datastore/test_data/output_24_to_latest.json

@@ -879,7 +879,7 @@
     }
   ],
   "tunnel_server": {
-    "PrivateKeySeed": "IvX6ZPRuWtLS5zyg"
+    "PrivateKeySeed": ""
   },
   "users": [
     {
@@ -944,6 +944,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.19.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.19.4\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/demo/demo.go

@@ -73,7 +73,7 @@ func isCleanStore(store dataservices.DataStore) (bool, error) {
 		return false, nil
 	}
 
-	users, err := store.User().Users()
+	users, err := store.User().ReadAll()
 	if err != nil {
 		return false, err
 	}

api/docker/client/client.go

@@ -57,20 +57,20 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
 func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	return client.NewClientWithOpts(
 		client.WithHost(endpoint.URL),
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 	)
 }
 
 func CreateClientFromEnv() (*client.Client, error) {
 	return client.NewClientWithOpts(
 		client.FromEnv,
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 	)
 }
 
 func CreateSimpleClient() (*client.Client, error) {
 	return client.NewClientWithOpts(
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 	)
 }
 
@@ -82,7 +82,7 @@ func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*cli
 
 	return client.NewClientWithOpts(
 		client.WithHost(endpoint.URL),
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
 	)
 }
@@ -116,7 +116,7 @@ func createEdgeClient(endpoint *portainer.Endpoint, signatureService portainer.D
 
 	return client.NewClientWithOpts(
 		client.WithHost(endpointURL),
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
 		client.WithHTTPHeaders(headers),
 	)
@@ -144,7 +144,7 @@ func createAgentClient(endpoint *portainer.Endpoint, signatureService portainer.
 
 	return client.NewClientWithOpts(
 		client.WithHost(endpoint.URL),
-		client.WithVersion(dockerClientVersion),
+		client.WithAPIVersionNegotiation(),
 		client.WithHTTPClient(httpCli),
 		client.WithHTTPHeaders(headers),
 	)

api/docker/images/registry.go

@@ -4,29 +4,26 @@ import (
 	"strings"
 	"time"
 
-	"github.com/patrickmn/go-cache"
-	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/registryutils"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/pkg/errors"
 )
 
-var (
-	_registriesCache = cache.New(5*time.Minute, 5*time.Minute)
-)
+var registriesCache = cache.New(5*time.Minute, 5*time.Minute)
 
-type (
-	RegistryClient struct {
-		dataStore dataservices.DataStore
-	}
-)
+type RegistryClient struct {
+	dataStore dataservices.DataStore
+}
 
 func NewRegistryClient(dataStore dataservices.DataStore) *RegistryClient {
 	return &RegistryClient{dataStore: dataStore}
 }
 
 func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
-	registries, err := c.dataStore.Registry().Registries()
+	registries, err := c.dataStore.Registry().ReadAll()
 	if err != nil {
 		return "", "", err
 	}
@@ -57,7 +54,7 @@ func (c *RegistryClient) CertainRegistryAuth(registry *portainer.Registry) (stri
 }
 
 func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
-	registries, err := c.dataStore.Registry().Registries()
+	registries, err := c.dataStore.Registry().ReadAll()
 	if err != nil {
 		return "", err
 	}
@@ -130,12 +127,14 @@ func findBestMatchRegistry(repository string, registries []portainer.Registry) (
 	if match == nil {
 		return nil, errors.New("no registries matched")
 	}
-	_registriesCache.Set(repository, match, 0)
+
+	registriesCache.Set(repository, match, 0)
+
 	return match, nil
 }
 
 func cachedRegistry(cacheKey string) (*portainer.Registry, error) {
-	r, ok := _registriesCache.Get(cacheKey)
+	r, ok := registriesCache.Get(cacheKey)
 	if ok {
 		registry, ok := r.(portainer.Registry)
 		if ok {

api/docker/images/status.go

@@ -7,12 +7,12 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
-	"github.com/opencontainers/go-digest"
-	"github.com/patrickmn/go-cache"
 	portainer "github.com/portainer/portainer/api"
 	consts "github.com/portainer/portainer/api/docker/consts"
 	"github.com/portainer/portainer/api/internal/slices"
 
+	"github.com/opencontainers/go-digest"
+	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
@@ -40,6 +40,7 @@ func (c *DigestClient) ContainersImageStatus(ctx context.Context, containers []t
 	cli, err := c.clientFactory.CreateClient(endpoint, "", nil)
 	if err != nil {
 		log.Error().Err(err).Msg("cannot create docker client")
+
 		return Error
 	}
 

api/edge/edge.go

@@ -0,0 +1,62 @@
+package edge
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+type (
+
+	// StackPayload represents the payload sent to the agent
+	StackPayload struct {
+		// ID of the stack
+		ID int
+		// Name of the stack
+		Name string
+
+		// Content of the stack file (for compatibility to agent version less than 2.19.0)
+		StackFileContent string
+
+		// Content of stack folder
+		DirEntries []filesystem.DirEntry
+		// Name of the stack entry file
+		EntryFileName string
+		// Namespace to use for kubernetes stack. Keep empty to use the manifest namespace.
+		Namespace string
+		// Version of the stack file
+		Version int
+		// RollbackTo specifies the stack file version to rollback to (only support to rollback to the last version currently)
+		RollbackTo *int
+
+		// RegistryCredentials holds the credentials for a Docker registry.
+		// Used only for EE
+		RegistryCredentials []RegistryCredentials
+		// PrePullImage is a flag indicating if the agent should pull the image before deploying the stack.
+		// Used only for EE
+		PrePullImage bool
+		// RePullImage is a flag indicating if the agent should pull the image if it is already present on the node.
+		// Used only for EE
+		RePullImage bool
+		// RetryDeploy is a flag indicating if the agent should retry to deploy the stack if it fails.
+		// Used only for EE
+		RetryDeploy bool
+		// EdgeUpdateID is the ID of the edge update related to this stack.
+		// Used only for EE
+		EdgeUpdateID int
+
+		// Is relative path supported
+		SupportRelativePath bool
+		// Mount point for relative path
+		FilesystemPath string
+		// Used only for EE
+		// EnvVars is a list of environment variables to inject into the stack
+		EnvVars []portainer.Pair
+	}
+
+	// RegistryCredentials holds the credentials for a Docker registry.
+	RegistryCredentials struct {
+		ServerURL string
+		Username  string
+		Secret    string
+	}
+)

api/exec/compose_stack.go

@@ -8,11 +8,11 @@ import (
 	"path"
 	"strings"
 
-	libstack "github.com/portainer/docker-compose-wrapper"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/stacks/stackutils"
+	"github.com/portainer/portainer/pkg/libstack"
 
 	"github.com/pkg/errors"
 )

api/exec/compose_stack_integration_test.go

@@ -9,9 +9,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/portainer/docker-compose-wrapper/compose"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/pkg/libstack/compose"
 
 	"github.com/rs/zerolog/log"
 )

api/exec/kubernetes_deploy.go

@@ -56,7 +56,7 @@ func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *
 		return "", err
 	}
 
-	user, err := deployer.dataStore.User().User(userID)
+	user, err := deployer.dataStore.User().Read(userID)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to fetch the user")
 	}

api/exec/swarm_stack.go

@@ -15,6 +15,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/registryutils"
 	"github.com/portainer/portainer/api/stacks/stackutils"
+	"github.com/rs/zerolog/log"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -64,16 +65,35 @@ func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoin
 		if registry.Authentication {
 			err = registryutils.EnsureRegTokenValid(manager.dataStore, &registry)
 			if err != nil {
-				return err
+				log.
+					Warn().
+					Err(err).
+					Str("RegistryName", registry.Name).
+					Msg("Failed to validate registry token. Skip logging with this registry.")
+
+				continue
 			}
 
 			username, password, err := registryutils.GetRegEffectiveCredential(&registry)
 			if err != nil {
-				return err
+				log.
+					Warn().
+					Err(err).
+					Str("RegistryName", registry.Name).
+					Msg("Failed to get effective credential. Skip logging with this registry.")
+
+				continue
 			}
 
 			registryArgs := append(args, "login", "--username", username, "--password", password, registry.URL)
-			runCommandAndCaptureStdErr(command, registryArgs, nil, "")
+			err = runCommandAndCaptureStdErr(command, registryArgs, nil, "")
+			if err != nil {
+				log.
+					Warn().
+					Err(err).
+					Str("RegistryName", registry.Name).
+					Msg("Failed to login.")
+			}
 		}
 	}
 

api/filesystem/filesystem.go

@@ -6,14 +6,15 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/gofrs/uuid"
 	portainer "github.com/portainer/portainer/api"
 
-	"io"
-	"os"
+	"github.com/gofrs/uuid"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -66,6 +67,11 @@ const (
 	MTLSCertFilename   = "mtls-cert.pem"
 	MTLSCACertFilename = "mtls-ca-cert.pem"
 	MTLSKeyFilename    = "mtls-key.pem"
+
+	// ChiselPath represents the default chisel path
+	ChiselPath = "chisel"
+	// ChiselPrivateKeyFilename represents the chisel private key file name
+	ChiselPrivateKeyFilename = "private-key.pem"
 )
 
 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -158,6 +164,20 @@ func (service *Service) GetStackProjectPath(stackIdentifier string) string {
 	return JoinPaths(service.wrapFileStore(ComposeStorePath), stackIdentifier)
 }
 
+// GetStackProjectPathByVersion returns the absolute path on the FS for a stack based
+// on its identifier and version.
+func (service *Service) GetStackProjectPathByVersion(stackIdentifier string, version int, commitHash string) string {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+
+	if commitHash != "" {
+		versionStr = fmt.Sprintf("%s", commitHash)
+	}
+	return JoinPaths(service.wrapFileStore(ComposeStorePath), stackIdentifier, versionStr)
+}
+
 // Copy copies the file on fromFilePath to toFilePath
 // if toFilePath exists func will fail unless deleteIfExists is true
 func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error {
@@ -238,6 +258,31 @@ func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string
 	return service.wrapFileStore(stackStorePath), nil
 }
 
+// StoreStackFileFromBytesByVersion creates a version subfolder in the ComposeStorePath and stores a new file from bytes.
+// It returns the path to the folder where version folders are stored.
+func (service *Service) StoreStackFileFromBytesByVersion(stackIdentifier, fileName string, version int, data []byte) (string, error) {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+	stackVersionPath := JoinPaths(stackStorePath, versionStr)
+	err := service.createDirectoryInStore(stackVersionPath)
+	if err != nil {
+		return "", err
+	}
+
+	composeFilePath := JoinPaths(stackVersionPath, fileName)
+	r := bytes.NewReader(data)
+
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return service.wrapFileStore(stackStorePath), nil
+}
+
 // UpdateStoreStackFileFromBytes makes stack file backup and updates a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) UpdateStoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
@@ -257,6 +302,38 @@ func (service *Service) UpdateStoreStackFileFromBytes(stackIdentifier, fileName
 	return service.wrapFileStore(stackStorePath), nil
 }
 
+// UpdateStoreStackFileFromBytesByVersion makes stack file backup and updates a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) UpdateStoreStackFileFromBytesByVersion(stackIdentifier, fileName string, version int, commitHash string, data []byte) (string, error) {
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
+
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	if commitHash != "" {
+		versionStr = commitHash
+	}
+
+	if versionStr != "" {
+		stackStorePath = JoinPaths(stackStorePath, versionStr)
+	}
+
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	err := service.createBackupFileInStore(composeFilePath)
+	if err != nil {
+		return "", err
+	}
+
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return service.wrapFileStore(stackStorePath), nil
+}
+
 // RemoveStackFileBackup removes the stack file backup in the ComposeStorePath.
 func (service *Service) RemoveStackFileBackup(stackIdentifier, fileName string) error {
 	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
@@ -265,6 +342,18 @@ func (service *Service) RemoveStackFileBackup(stackIdentifier, fileName string)
 	return service.removeBackupFileInStore(composeFilePath)
 }
 
+// RemoveStackFileBackupByVersion removes the stack file backup by version in the ComposeStorePath.
+func (service *Service) RemoveStackFileBackupByVersion(stackIdentifier string, version int, fileName string) error {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier, versionStr)
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+
+	return service.removeBackupFileInStore(composeFilePath)
+}
+
 // RollbackStackFile rollbacks the stack file backup in the ComposeStorePath.
 func (service *Service) RollbackStackFile(stackIdentifier, fileName string) error {
 	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier)
@@ -290,6 +379,35 @@ func (service *Service) RollbackStackFile(stackIdentifier, fileName string) erro
 	return os.Remove(backupPath)
 }
 
+// RollbackStackFileByVersion rollbacks the stack file backup by version in the ComposeStorePath.
+func (service *Service) RollbackStackFileByVersion(stackIdentifier string, version int, fileName string) error {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	stackStorePath := JoinPaths(ComposeStorePath, stackIdentifier, versionStr)
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	path := service.wrapFileStore(composeFilePath)
+	backupPath := fmt.Sprintf("%s.bak", path)
+
+	exists, err := service.FileExists(backupPath)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		// keep the updated/failed stack file
+		return nil
+	}
+
+	err = service.Copy(backupPath, path, true)
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(backupPath)
+}
+
 // GetEdgeStackProjectPath returns the absolute path on the FS for a edge stack based
 // on its identifier.
 func (service *Service) GetEdgeStackProjectPath(edgeStackIdentifier string) string {
@@ -316,6 +434,62 @@ func (service *Service) StoreEdgeStackFileFromBytes(edgeStackIdentifier, fileNam
 	return service.wrapFileStore(stackStorePath), nil
 }
 
+// GetEdgeStackProjectPathByVersion returns the absolute path on the FS for a edge stack based
+// on its identifier and version.
+// EE only feature
+func (service *Service) GetEdgeStackProjectPathByVersion(edgeStackIdentifier string, version int, commitHash string) string {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+
+	if commitHash != "" {
+		versionStr = commitHash
+	}
+
+	return JoinPaths(service.wrapFileStore(EdgeStackStorePath), edgeStackIdentifier, versionStr)
+}
+
+// StoreEdgeStackFileFromBytesByVersion creates a subfolder in the EdgeStackStorePath with version and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+// EE only feature
+func (service *Service) StoreEdgeStackFileFromBytesByVersion(edgeStackIdentifier, fileName string, version int, data []byte) (string, error) {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+	stackStorePath := JoinPaths(EdgeStackStorePath, edgeStackIdentifier, versionStr)
+
+	err := service.createDirectoryInStore(stackStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	composeFilePath := JoinPaths(stackStorePath, fileName)
+	r := bytes.NewReader(data)
+
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return service.wrapFileStore(stackStorePath), nil
+}
+
+// FormProjectPathByVersion returns the absolute path on the FS for a project based with version
+func (service *Service) FormProjectPathByVersion(path string, version int, commitHash string) string {
+	versionStr := ""
+	if version != 0 {
+		versionStr = fmt.Sprintf("v%d", version)
+	}
+
+	if commitHash != "" {
+		versionStr = commitHash
+	}
+
+	return JoinPaths(path, versionStr)
+}
+
 // StoreRegistryManagementFileFromBytes creates a subfolder in the
 // ExtensionRegistryManagementStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
@@ -675,6 +849,28 @@ func defaultMTLSCertPathUnderFileStore() (string, string, string) {
 	return certPath, caCertPath, keyPath
 }
 
+// GetDefaultChiselPrivateKeyPath returns the chisle private key path
+func (service *Service) GetDefaultChiselPrivateKeyPath() string {
+	privateKeyPath := defaultChiselPrivateKeyPathUnderFileStore()
+	return service.wrapFileStore(privateKeyPath)
+}
+
+func defaultChiselPrivateKeyPathUnderFileStore() string {
+	return JoinPaths(ChiselPath, ChiselPrivateKeyFilename)
+}
+
+// StoreChiselPrivateKey store the specified chisel private key content on disk.
+func (service *Service) StoreChiselPrivateKey(privateKey []byte) error {
+	err := service.createDirectoryInStore(ChiselPath)
+	if err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	r := bytes.NewReader(privateKey)
+	privateKeyPath := defaultChiselPrivateKeyPathUnderFileStore()
+	return service.createFileInStore(privateKeyPath, r)
+}
+
 // StoreSSLCertPair stores a ssl certificate pair
 func (service *Service) StoreSSLCertPair(cert, key []byte) (string, string, error) {
 	certPath, keyPath := defaultCertPathUnderFileStore()
@@ -734,6 +930,56 @@ func FileExists(filePath string) (bool, error) {
 	return true, nil
 }
 
+// SafeCopyDirectory copies a directory from src to dst in a safe way.
+func (service *Service) SafeMoveDirectory(originalPath, newPath string) error {
+	// 1. Backup the source directory to a different folder
+	backupDir := fmt.Sprintf("%s-%s", filepath.Dir(originalPath), "backup")
+	err := MoveDirectory(originalPath, backupDir)
+	if err != nil {
+		return fmt.Errorf("failed to backup source directory: %w", err)
+	}
+
+	defer func() {
+		if err != nil {
+			// If an error occurred, rollback the backup directory
+			restoreErr := restoreBackup(originalPath, backupDir)
+			if restoreErr != nil {
+				log.Warn().Err(restoreErr).Msg("failed to restore backup during creating versioning folder")
+			}
+		}
+	}()
+
+	// 2. Copy the backup directory to the destination directory
+	err = CopyDir(backupDir, newPath, false)
+	if err != nil {
+		return fmt.Errorf("failed to copy backup directory to destination directory: %w", err)
+	}
+
+	// 3. Delete the backup directory
+	err = os.RemoveAll(backupDir)
+	if err != nil {
+		return fmt.Errorf("failed to delete backup directory: %w", err)
+	}
+
+	return nil
+}
+
+func restoreBackup(src, backupDir string) error {
+	// Rollback by deleting the original directory and copying the
+	// backup direcotry back to the source directory, and then deleting
+	// the backup directory
+	err := os.RemoveAll(src)
+	if err != nil {
+		return fmt.Errorf("failed to delete destination directory: %w", err)
+	}
+
+	err = MoveDirectory(backupDir, src)
+	if err != nil {
+		return fmt.Errorf("failed to restore backup directory: %w", err)
+	}
+	return nil
+}
+
 func MoveDirectory(originalPath, newPath string) error {
 	if _, err := os.Stat(originalPath); err != nil {
 		return err

api/filesystem/filesystem_move_test.go

@@ -8,6 +8,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var content = []byte("content")
+
 func Test_movePath_shouldFailIfSourceDirDoesNotExist(t *testing.T) {
 	sourceDir := "missing"
 	destinationDir := t.TempDir()
@@ -52,8 +54,6 @@ func Test_movePath_successWhenSourceExistsAndDestinationIsMissing(t *testing.T)
 	assertFileContent(t, path.Join(destinationDir, "dir", "file"))
 }
 
-var content []byte = []byte("content")
-
 func addFile(fileParts ...string) (filepath string) {
 	if len(fileParts) > 2 {
 		dir := path.Join(fileParts[:len(fileParts)-1]...)
@@ -62,6 +62,7 @@ func addFile(fileParts ...string) (filepath string) {
 
 	p := path.Join(fileParts...)
 	os.WriteFile(p, content, 0766)
+
 	return p
 }
 

api/filesystem/serialize.go

@@ -0,0 +1,169 @@
+package filesystem
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/mod/semver"
+)
+
+type DirEntry struct {
+	Name        string
+	Content     string
+	IsFile      bool
+	Permissions os.FileMode
+}
+
+// FilterDirForEntryFile filers the given dirEntries, returns entries of the entryFile and .env file
+func FilterDirForEntryFile(dirEntries []DirEntry, entryFile string) []DirEntry {
+	var filteredDirEntries []DirEntry
+
+	dotEnvFile := filepath.Join(filepath.Dir(entryFile), ".env")
+	filters := []string{entryFile, dotEnvFile}
+
+	for _, dirEntry := range dirEntries {
+		match := false
+		if dirEntry.IsFile {
+			for _, filter := range filters {
+				if filter == dirEntry.Name {
+					match = true
+					break
+				}
+			}
+		} else {
+			for _, filter := range filters {
+				if strings.HasPrefix(filter, dirEntry.Name) {
+					match = true
+					break
+				}
+			}
+		}
+		if match {
+			filteredDirEntries = append(filteredDirEntries, dirEntry)
+		}
+	}
+
+	return filteredDirEntries
+}
+
+func FilterDirForCompatibility(dirEntries []DirEntry, entryFilePath, agentVersion string) (string, error) {
+
+	if semver.Compare(fmt.Sprintf("v%s", agentVersion), "v2.19.0") == -1 {
+		for _, dirEntry := range dirEntries {
+			if dirEntry.IsFile {
+				if dirEntry.Name == entryFilePath {
+					return DecodeFileContent(dirEntry.Content)
+				}
+			}
+		}
+		return "", fmt.Errorf("Entry file %s not found in dir entries", entryFilePath)
+	}
+
+	return "", nil
+}
+
+// LoadDir reads all files and folders recursively from the given directory
+// File content is base64-encoded
+func LoadDir(dir string) ([]DirEntry, error) {
+	var dirEntries []DirEntry
+
+	err := filepath.WalkDir(
+		dir,
+		func(path string, d os.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			fileInfo, err := d.Info()
+			if err != nil {
+				return err
+			}
+
+			relativePath, err := filepath.Rel(dir, path)
+			if err != nil {
+				return err
+			}
+			if relativePath == "." {
+				return nil
+			}
+
+			dirEntry := DirEntry{
+				Name:        relativePath,
+				Permissions: fileInfo.Mode().Perm(),
+			}
+
+			if !fileInfo.IsDir() {
+				// Read file contents
+				fileContent, err := os.ReadFile(path)
+				if err != nil {
+					return err
+				}
+
+				dirEntry.Content = base64.StdEncoding.EncodeToString(fileContent)
+				dirEntry.IsFile = true
+			}
+
+			dirEntries = append(dirEntries, dirEntry)
+			return nil
+		})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return dirEntries, nil
+}
+
+// PersistDir writes the provided array of files and folders back to the given directory.
+func PersistDir(dir string, dirEntries []DirEntry) error {
+	for _, dirEntry := range dirEntries {
+		path := filepath.Join(dir, dirEntry.Name)
+
+		if dirEntry.IsFile {
+			// Create the directory path if it doesn't exist
+			err := os.MkdirAll(filepath.Dir(path), 0744)
+			if err != nil {
+				return err
+			}
+
+			// Write file contents
+			err = os.WriteFile(path, []byte(dirEntry.Content), dirEntry.Permissions)
+			if err != nil {
+				return err
+			}
+		} else {
+			// Create the directory
+			err := os.MkdirAll(path, dirEntry.Permissions)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func DecodeFileContent(encodedFileContent string) (string, error) {
+	decodedBytes, err := base64.StdEncoding.DecodeString(encodedFileContent)
+	if err != nil {
+		return "", err
+	}
+	return string(decodedBytes), nil
+}
+
+func DecodeDirEntries(dirEntries []DirEntry) error {
+	for index, dirEntry := range dirEntries {
+		if dirEntry.IsFile && dirEntry.Content != "" {
+			decodedFile, err := DecodeFileContent(dirEntry.Content)
+			if err != nil {
+				return err
+			}
+			dirEntries[index].Content = decodedFile
+		}
+	}
+
+	return nil
+}