fix namespace refresh flag issue

Co-authored-by: testa113 <testa113>

.git-blame-ignore-revs

@@ -2,4 +2,7 @@
 cf5056d9c03b62d91a25c3b9127caac838695f98
 
 # prettier v2
-42e7db0ae7897d3cb72b0ea1ecf57ee2dd694169
+42e7db0ae7897d3cb72b0ea1ecf57ee2dd694169
+
+# tailwind prettier
+58d66d3142950bb90a7d85511c034ac9fabba9ba

.github/workflows/lint.yml

@@ -23,6 +23,9 @@ jobs:
         with:
           node-version: '14'
           cache: 'yarn'
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.4
       - run: yarn --frozen-lockfile
 
       - name: Run linters
@@ -36,3 +39,9 @@ jobs:
           gofmt_dir: api/
       - name: Typecheck
         uses: icrawl/action-tsc@v1
+      - name: GolangCI-Lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: latest
+          working-directory: api
+          args: -c .golangci.yaml

.github/workflows/nightly-security-scan.yml

@@ -56,17 +56,20 @@ jobs:
     steps:
       - uses: actions/checkout@master
 
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19.4'
+
       - name: Download go modules
         run: cd ./api && go get -t -v -d ./...
 
       - name: Run Snyk to check for vulnerabilities
-        uses: snyk/actions/golang@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          args: --file=./api/go.mod
-          json: true
+        run: |
+          yarn global add snyk
+          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
       - name: Upload go security scan result as artifact
         uses: actions/upload-artifact@v3
@@ -101,15 +104,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@master
 
-      - name: Use golang 1.18
+      - name: Use golang 1.19.4
         uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19.4'
 
-      - name: Use Node.js 12.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
-          node-version: 12.x
+          node-version: 18.x
 
       - name: Install packages and build
         run: yarn install && yarn build

.github/workflows/pr-security.yml

@@ -78,17 +78,20 @@ jobs:
     steps:
       - uses: actions/checkout@master
 
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19.4'
+
       - name: Download go modules
         run: cd ./api && go get -t -v -d ./...
 
       - name: Run Snyk to check for vulnerabilities
-        uses: snyk/actions/golang@master
         continue-on-error: true # To make sure that artifact upload gets called
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          args: --file=./api/go.mod
-          json: true
+        run: |
+          yarn global add snyk
+          snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
 
       - name: Upload go security scan result as artifact
         uses: actions/upload-artifact@v3
@@ -136,15 +139,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@master
 
-      - name: Use golang 1.18
+      - name: Use golang 1.19.4
         uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19.4'
 
-      - name: Use Node.js 12.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
-          node-version: 12.x
+          node-version: 18.x
 
       - name: Install packages and build
         run: yarn install && yarn build

.storybook/main.js

@@ -29,6 +29,23 @@ module.exports = {
         extensions: config.resolve.extensions,
       }),
     ];
+
+    const svgRule = config.module.rules.find((rule) => rule.test && typeof rule.test.test === 'function' && rule.test.test('.svg'));
+    svgRule.test = new RegExp(svgRule.test.source.replace('svg|', ''));
+
+    config.module.rules.unshift({
+      test: /\.svg$/i,
+      type: 'asset',
+      resourceQuery: { not: [/c/] }, // exclude react component if *.svg?url
+    });
+
+    config.module.rules.unshift({
+      test: /\.svg$/i,
+      issuer: /\.(js|ts)(x)?$/,
+      resourceQuery: /c/, // *.svg?c
+      use: [{ loader: '@svgr/webpack', options: { icon: true } }],
+    });
+
     return config;
   },
   core: {

CONTRIBUTING.md

@@ -25,7 +25,7 @@ Each commit message should include a **type**, a **scope** and a **subject**:
  <type>(<scope>): <subject>
 ```
 
-Lines should not exceed 100 characters. This allows the message to be easier to read on github as well as in various git tools and produces a nice, neat commit log ie:
+Lines should not exceed 100 characters. This allows the message to be easier to read on GitHub as well as in various git tools and produces a nice, neat commit log ie:
 
 ```
  #271 feat(containers): add exposed ports in the containers view
@@ -63,7 +63,7 @@ The subject contains succinct description of the change:
 
 ## Contribution process
 
-Our contribution process is described below. Some of the steps can be visualized inside Github via specific `status/` labels, such as `status/1-functional-review` or `status/2-technical-review`.
+Our contribution process is described below. Some of the steps can be visualized inside GitHub via specific `status/` labels, such as `status/1-functional-review` or `status/2-technical-review`.
 
 ### Bug report
 
@@ -93,7 +93,7 @@ $ yarn start
 
 Portainer can now be accessed at <https://localhost:9443>.
 
-Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
+Find more detailed steps at <https://docs.portainer.io/contribute/build>.
 
 ### Build customisation
 
@@ -103,6 +103,10 @@ You can customise the following settings:
 - `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`).
 - `PORTAINER_FLAGS`: a list of flags to be used on the portainer commandline, in the form `--admin-password=<pwd hash> --feat fdo=false --feat open-amt` (default: `""`).
 
+## Testing your build
+
+The `--log-level=DEBUG` flag can be passed to the Portainer container in order to provide additional debug output which may be useful when troubleshooting your builds. Please note that this flag was originally intended for internal use and as such the format, functionality and output may change between releases without warning.
+
 ## Adding api docs
 
 When adding a new resource (or a route handler), we should add a new tag to api/http/handler/handler.go#L136 like this:

api/.golangci.yaml

@@ -0,0 +1,26 @@
+linters:
+  # Disable all linters.
+  disable-all: true
+  enable:
+    - depguard
+linters-settings:
+  depguard:
+    list-type: denylist
+    include-go-root: true
+    packages:
+      - github.com/sirupsen/logrus
+      - golang.org/x/exp
+    packages-with-error-message:
+      - github.com/sirupsen/logrus: 'logging is allowed only by github.com/rs/zerolog'
+    ignore-file-rules:
+      - "**/*_test.go"
+    # Create additional guards that follow the same configuration pattern.
+    # Results from all guards are aggregated together.
+    # additional-guards:
+    #   - list-type: allowlist
+    #     include-go-root: false
+    #     packages:
+    #       - github.com/sirupsen/logrus
+    #     # Specify rules by which the linter ignores certain files for consideration.
+    #     ignore-file-rules:
+    #       - "!**/*_test.go"

api/adminmonitor/admin_monitor.go

@@ -54,7 +54,8 @@ func (m *Monitor) Start() {
 		case <-time.After(m.timeout):
 			initialized, err := m.WasInitialized()
 			if err != nil {
-				log.Fatal().Err(err).Msg("")
+				log.Error().Err(err).Msg("AdminMonitor failed to determine if Portainer is Initialized")
+				return
 			}
 
 			if !initialized {

api/chisel/schedules.go

@@ -6,9 +6,13 @@ import (
 )
 
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
-func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *portainer.EdgeJob) {
+func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portainer.EdgeJob) {
+	if endpoint.Edge.AsyncMode {
+		return
+	}
+
 	service.mu.Lock()
-	tunnel := service.getTunnelDetails(endpointID)
+	tunnel := service.getTunnelDetails(endpoint.ID)
 
 	existingJobIndex := -1
 	for idx, existingJob := range tunnel.Jobs {
@@ -24,7 +28,7 @@ func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *por
 		tunnel.Jobs[existingJobIndex] = *edgeJob
 	}
 
-	cache.Del(endpointID)
+	cache.Del(endpoint.ID)
 
 	service.mu.Unlock()
 }

api/chisel/tunnel.go

@@ -2,6 +2,7 @@ package chisel
 
 import (
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"math/rand"
 	"strings"
@@ -66,6 +67,10 @@ func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portai
 
 // GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
 func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
+	if endpoint.Edge.AsyncMode {
+		return portainer.TunnelDetails{}, errors.New("cannot open tunnel on async endpoint")
+	}
+
 	tunnel := service.GetTunnelDetails(endpoint.ID)
 
 	if tunnel.Status == portainer.EdgeAgentActive {

api/cli/cli.go

@@ -36,7 +36,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
 		DemoEnvironment:           kingpin.Flag("demo", "Demo environment").Bool(),
 		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
-		FeatureFlags:              BoolPairs(kingpin.Flag("feat", "List of feature flags").Hidden()),
+		FeatureFlags:              kingpin.Flag("feat", "List of feature flags").Strings(),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
 		NoAnalytics:               kingpin.Flag("no-analytics", "Disable Analytics in app (deprecated)").Bool(),
 		TLS:                       kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),

api/cmd/portainer/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"context"
 	"crypto/sha256"
-	"fmt"
 	"math/rand"
 	"os"
 	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -47,6 +45,7 @@ import (
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks/deployments"
+	"github.com/portainer/portainer/pkg/featureflags"
 	"github.com/portainer/portainer/pkg/libhelm"
 
 	"github.com/gofrs/uuid"
@@ -318,45 +317,6 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 	return dataStore.SSLSettings().UpdateSettings(sslSettings)
 }
 
-// enableFeaturesFromFlags turns on or off feature flags
-// e.g.  portainer --feat open-amt --feat fdo=true ... (defaults to true)
-// note, settings are persisted to the DB. To turn off `--feat open-amt=false`
-func enableFeaturesFromFlags(dataStore dataservices.DataStore, flags *portainer.CLIFlags) error {
-	settings, err := dataStore.Settings().Settings()
-	if err != nil {
-		return err
-	}
-
-	if settings.FeatureFlagSettings == nil {
-		settings.FeatureFlagSettings = make(map[portainer.Feature]bool)
-	}
-
-	// loop through feature flags to check if they are supported
-	for _, feat := range *flags.FeatureFlags {
-		var correspondingFeature *portainer.Feature
-		for i, supportedFeat := range portainer.SupportedFeatureFlags {
-			if strings.EqualFold(feat.Name, string(supportedFeat)) {
-				correspondingFeature = &portainer.SupportedFeatureFlags[i]
-			}
-		}
-
-		if correspondingFeature == nil {
-			return fmt.Errorf("unknown feature flag '%s'", feat.Name)
-		}
-
-		featureState, err := strconv.ParseBool(feat.Value)
-		if err != nil {
-			return fmt.Errorf("feature flag's '%s' value should be true or false", feat.Name)
-		}
-
-		log.Info().Str("feature", string(*correspondingFeature)).Bool("state", featureState).Msg("")
-
-		settings.FeatureFlagSettings[*correspondingFeature] = featureState
-	}
-
-	return dataStore.Settings().UpdateSettings(settings)
-}
-
 func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	private, public, err := fileService.LoadKeyPair()
 	if err != nil {
@@ -547,6 +507,10 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
 func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
 
+	if flags.FeatureFlags != nil {
+		featureflags.Parse(*flags.FeatureFlags, portainer.SupportedFeatureFlags)
+	}
+
 	fileService := initFileService(*flags.Data)
 	encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
 	if encryptionKey == nil {
@@ -576,11 +540,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing JWT service")
 	}
 
-	err = enableFeaturesFromFlags(dataStore, flags)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed enabling feature flag")
-	}
-
 	ldapService := initLDAPService()
 
 	oauthService := initOAuthService()
@@ -730,21 +689,17 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing upgrade service")
 	}
 
-	// FIXME: In 2.16 we changed the way ingress controller permissions are
-	// stored. Instead of being stored as annotation on an ingress rule, we keep
-	// them in our database. However, in order to run the migration we need an
-	// admin kube client to run lookup the old ingress rules and compare them
-	// with the current existing ingress classes.
-	//
-	// Unfortunately, our migrations run as part of the database initialization
-	// and our kubeclients require an initialized database. So it is not
-	// possible to do this migration as part of our normal flow. We DO have a
-	// migration which toggles a boolean in kubernetes configuration that
-	// indicated that this "post init" migration should be run. If/when this is
-	// resolved we can remove this function.
-	err = kubernetesClientFactory.PostInitMigrateIngresses()
-	if err != nil {
-		log.Fatal().Err(err).Msg("failure during creation of new database")
+	// Our normal migrations run as part of the database initialization
+	// but some more complex migrations require access to a kubernetes or docker
+	// client. Therefore we run a separate migration process just before
+	// starting the server.
+	postInitMigrator := datastore.NewPostInitMigrator(
+		kubernetesClientFactory,
+		dockerClientFactory,
+		dataStore,
+	)
+	if err := postInitMigrator.PostInitMigrate(); err != nil {
+		log.Fatal().Err(err).Msg("failure during post init migrations")
 	}
 
 	return &http.Server{

api/cmd/portainer/main_test.go

@@ -1,111 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/cli"
-	"github.com/portainer/portainer/api/dataservices"
-	"github.com/portainer/portainer/api/datastore"
-
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/alecthomas/kingpin.v2"
-)
-
-type mockKingpinSetting string
-
-func (m mockKingpinSetting) SetValue(value kingpin.Value) {
-	value.Set(string(m))
-}
-
-func Test_enableFeaturesFromFlags(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
-
-	tests := []struct {
-		featureFlag string
-		isSupported bool
-	}{
-		{"test", false},
-	}
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("%s succeeds:%v", test.featureFlag, test.isSupported), func(t *testing.T) {
-			mockKingpinSetting := mockKingpinSetting(test.featureFlag)
-			flags := &portainer.CLIFlags{FeatureFlags: cli.BoolPairs(mockKingpinSetting)}
-			err := enableFeaturesFromFlags(store, flags)
-			if test.isSupported {
-				is.NoError(err)
-			} else {
-				is.Error(err)
-			}
-		})
-	}
-
-	t.Run("passes for all supported feature flags", func(t *testing.T) {
-		for _, flag := range portainer.SupportedFeatureFlags {
-			mockKingpinSetting := mockKingpinSetting(flag)
-			flags := &portainer.CLIFlags{FeatureFlags: cli.BoolPairs(mockKingpinSetting)}
-			err := enableFeaturesFromFlags(store, flags)
-			is.NoError(err)
-		}
-	})
-}
-
-const FeatTest portainer.Feature = "optional-test"
-
-func optionalFunc(dataStore dataservices.DataStore) string {
-
-	// TODO: this is a code smell - finding out if a feature flag is enabled should not require having access to the store, and asking for a settings obj.
-	//       ideally, the `if` should look more like:
-	//       if featureflags.FlagEnabled(FeatTest) {}
-	settings, err := dataStore.Settings().Settings()
-	if err != nil {
-		return err.Error()
-	}
-
-	if settings.FeatureFlagSettings[FeatTest] {
-		return "enabled"
-	}
-	return "disabled"
-}
-
-func Test_optionalFeature(t *testing.T) {
-	portainer.SupportedFeatureFlags = append(portainer.SupportedFeatureFlags, FeatTest)
-
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(t, true, true)
-	defer teardown()
-
-	// Enable the test feature
-	t.Run(fmt.Sprintf("%s succeeds:%v", FeatTest, true), func(t *testing.T) {
-		mockKingpinSetting := mockKingpinSetting(FeatTest)
-		flags := &portainer.CLIFlags{FeatureFlags: cli.BoolPairs(mockKingpinSetting)}
-		err := enableFeaturesFromFlags(store, flags)
-		is.NoError(err)
-		is.Equal("enabled", optionalFunc(store))
-	})
-
-	// Same store, so the feature flag should still be enabled
-	t.Run(fmt.Sprintf("%s succeeds:%v", FeatTest, true), func(t *testing.T) {
-		is.Equal("enabled", optionalFunc(store))
-	})
-
-	// disable the test feature
-	t.Run(fmt.Sprintf("%s succeeds:%v", FeatTest, true), func(t *testing.T) {
-		mockKingpinSetting := mockKingpinSetting(FeatTest + "=false")
-		flags := &portainer.CLIFlags{FeatureFlags: cli.BoolPairs(mockKingpinSetting)}
-		err := enableFeaturesFromFlags(store, flags)
-		is.NoError(err)
-		is.Equal("disabled", optionalFunc(store))
-	})
-
-	// Same store, so feature flag should still be disabled
-	t.Run(fmt.Sprintf("%s succeeds:%v", FeatTest, true), func(t *testing.T) {
-		is.Equal("disabled", optionalFunc(store))
-	})
-
-}

api/connection.go

@@ -4,10 +4,35 @@ import (
 	"io"
 )
 
+type ReadTransaction interface {
+	GetObject(bucketName string, key []byte, object interface{}) error
+	GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
+	GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
+	GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error
+}
+
+type Transaction interface {
+	ReadTransaction
+
+	SetServiceName(bucketName string) error
+	UpdateObject(bucketName string, key []byte, object interface{}) error
+	DeleteObject(bucketName string, key []byte) error
+	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
+	CreateObjectWithId(bucketName string, id int, obj interface{}) error
+	CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
+	DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error
+	GetNextIdentifier(bucketName string) int
+}
+
 type Connection interface {
+	Transaction
+
 	Open() error
 	Close() error
 
+	UpdateTx(fn func(Transaction) error) error
+	ViewTx(fn func(Transaction) error) error
+
 	// write the db contents to filename as json (the schema needs defining)
 	ExportRaw(filename string) error
 
@@ -21,20 +46,9 @@ type Connection interface {
 	NeedsEncryptionMigration() (bool, error)
 	SetEncrypted(encrypted bool)
 
-	SetServiceName(bucketName string) error
-	GetObject(bucketName string, key []byte, object interface{}) error
-	UpdateObject(bucketName string, key []byte, object interface{}) error
-	UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
-	DeleteObject(bucketName string, key []byte) error
-	DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error
-	GetNextIdentifier(bucketName string) int
-	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
-	CreateObjectWithId(bucketName string, id int, obj interface{}) error
-	CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
-	GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
-	GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
-	ConvertToKey(v int) []byte
-
 	BackupMetadata() (map[string]interface{}, error)
 	RestoreMetadata(s map[string]interface{}) error
+
+	UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
+	ConvertToKey(v int) []byte
 }

api/crypto/tls.go

@@ -6,8 +6,8 @@ import (
 	"os"
 )
 
-// CreateServerTLSConfiguration creates a basic tls.Config to be used by servers with recommended TLS settings
-func CreateServerTLSConfiguration() *tls.Config {
+// CreateTLSConfiguration creates a basic tls.Config with recommended TLS settings
+func CreateTLSConfiguration() *tls.Config {
 	return &tls.Config{
 		MinVersion: tls.VersionTLS12,
 		CipherSuites: []uint16{
@@ -27,7 +27,7 @@ func CreateServerTLSConfiguration() *tls.Config {
 // CreateTLSConfigurationFromBytes initializes a tls.Config using a CA certificate, a certificate and a key
 // loaded from memory.
 func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) {
-	config := &tls.Config{}
+	config := CreateTLSConfiguration()
 	config.InsecureSkipVerify = skipServerVerification
 
 	if !skipClientVerification {
@@ -50,7 +50,7 @@ func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerific
 // CreateTLSConfigurationFromDisk initializes a tls.Config using a CA certificate, a certificate and a key
 // loaded from disk.
 func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipServerVerification bool) (*tls.Config, error) {
-	config := &tls.Config{}
+	config := CreateTLSConfiguration()
 	config.InsecureSkipVerify = skipServerVerification
 
 	if certPath != "" && keyPath != "" {

api/database/boltdb/db.go

@@ -1,7 +1,6 @@
 package boltdb
 
 import (
-	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -10,6 +9,7 @@ import (
 	"path"
 	"time"
 
+	portainer "github.com/portainer/portainer/api"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
@@ -132,9 +132,11 @@ func (connection *DbConnection) Open() error {
 	if err != nil {
 		return err
 	}
+
 	db.MaxBatchSize = connection.MaxBatchSize
 	db.MaxBatchDelay = connection.MaxBatchDelay
 	connection.DB = db
+
 	return nil
 }
 
@@ -144,9 +146,30 @@ func (connection *DbConnection) Close() error {
 	if connection.DB != nil {
 		return connection.DB.Close()
 	}
+
 	return nil
 }
 
+func (connection *DbConnection) txFn(fn func(portainer.Transaction) error) func(*bolt.Tx) error {
+	return func(tx *bolt.Tx) error {
+		return fn(&DbTransaction{conn: connection, tx: tx})
+	}
+}
+
+// UpdateTx executes the given function inside a read-write transaction
+func (connection *DbConnection) UpdateTx(fn func(portainer.Transaction) error) error {
+	if connection.MaxBatchDelay > 0 && connection.MaxBatchSize > 1 {
+		return connection.Batch(connection.txFn(fn))
+	}
+
+	return connection.Update(connection.txFn(fn))
+}
+
+// ViewTx executes the given function inside a read-only transaction
+func (connection *DbConnection) ViewTx(fn func(portainer.Transaction) error) error {
+	return connection.View(connection.txFn(fn))
+}
+
 // BackupTo backs up db to a provided writer.
 // It does hot backup and doesn't block other database reads and writes
 func (connection *DbConnection) BackupTo(w io.Writer) error {
@@ -180,34 +203,16 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 
 // CreateBucket is a generic function used to create a bucket inside a database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
-		return err
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.SetServiceName(bucketName)
 	})
 }
 
 // GetObject is a generic function used to retrieve an unmarshalled object from a database.
 func (connection *DbConnection) GetObject(bucketName string, key []byte, object interface{}) error {
-	var data []byte
-
-	err := connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		value := bucket.Get(key)
-		if value == nil {
-			return dserrors.ErrObjectNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-
-		return nil
+	return connection.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetObject(bucketName, key, object)
 	})
-	if err != nil {
-		return err
-	}
-
-	return connection.UnmarshalObjectWithJsoniter(data, object)
 }
 
 func (connection *DbConnection) getEncryptionKey() []byte {
@@ -220,14 +225,8 @@ func (connection *DbConnection) getEncryptionKey() []byte {
 
 // UpdateObject is a generic function used to update an object inside a database.
 func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
-	data, err := connection.MarshalObject(object)
-	if err != nil {
-		return err
-	}
-
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-		return bucket.Put(key, data)
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.UpdateObject(bucketName, key, object)
 	})
 }
 
@@ -259,34 +258,16 @@ func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte,
 
 // DeleteObject is a generic function used to delete an object inside a database.
 func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-		return bucket.Delete(key)
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.DeleteObject(bucketName, key)
 	})
 }
 
 // DeleteAllObjects delete all objects where matching() returns (id, ok).
 // TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
 func (connection *DbConnection) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := connection.UnmarshalObject(v, &obj)
-			if err != nil {
-				return err
-			}
-
-			if id, ok := matching(obj); ok {
-				err := bucket.Delete(connection.ConvertToKey(id))
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		return nil
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.DeleteAllObjects(bucketName, obj, matching)
 	})
 }
 
@@ -294,13 +275,8 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, obj interfac
 func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 	var identifier int
 
-	connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-		id, err := bucket.NextSequence()
-		if err != nil {
-			return err
-		}
-		identifier = int(id)
+	_ = connection.UpdateTx(func(tx portainer.Transaction) error {
+		identifier = tx.GetNextIdentifier(bucketName)
 		return nil
 	})
 
@@ -309,108 +285,41 @@ func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 
 // CreateObject creates a new object in the bucket, using the next bucket sequence id
 func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		seqId, _ := bucket.NextSequence()
-		id, obj := fn(seqId)
-
-		data, err := connection.MarshalObject(obj)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(connection.ConvertToKey(int(id)), data)
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.CreateObject(bucketName, fn)
 	})
 }
 
 // CreateObjectWithId creates a new object in the bucket, using the specified id
 func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-		data, err := connection.MarshalObject(obj)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(connection.ConvertToKey(id), data)
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.CreateObjectWithId(bucketName, id, obj)
 	})
 }
 
 // CreateObjectWithStringId creates a new object in the bucket, using the specified id
 func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-		data, err := connection.MarshalObject(obj)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(id, data)
+	return connection.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.CreateObjectWithStringId(bucketName, id, obj)
 	})
 }
 
 func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
-	err := connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := connection.UnmarshalObject(v, obj)
-			if err != nil {
-				return err
-			}
-			obj, err = append(obj)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
+	return connection.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetAll(bucketName, obj, append)
 	})
-
-	return err
 }
 
 // TODO: decide which Unmarshal to use, and use one...
 func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
-	err := connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := connection.UnmarshalObjectWithJsoniter(v, obj)
-			if err != nil {
-				return err
-			}
-			obj, err = append(obj)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
+	return connection.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetAllWithJsoniter(bucketName, obj, append)
 	})
-	return err
 }
 
 func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
-	return connection.View(func(tx *bolt.Tx) error {
-		cursor := tx.Bucket([]byte(bucketName)).Cursor()
-
-		for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
-			err := connection.UnmarshalObjectWithJsoniter(v, obj)
-			if err != nil {
-				return err
-			}
-
-			obj, err = append(obj)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
+	return connection.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, append)
 	})
 }
 

api/database/boltdb/tx.go

@@ -0,0 +1,172 @@
+package boltdb
+
+import (
+	"bytes"
+
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+	bolt "go.etcd.io/bbolt"
+)
+
+type DbTransaction struct {
+	conn *DbConnection
+	tx   *bolt.Tx
+}
+
+func (tx *DbTransaction) SetServiceName(bucketName string) error {
+	_, err := tx.tx.CreateBucketIfNotExists([]byte(bucketName))
+	return err
+}
+
+func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interface{}) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	value := bucket.Get(key)
+	if value == nil {
+		return dserrors.ErrObjectNotFound
+	}
+
+	data := make([]byte, len(value))
+	copy(data, value)
+
+	return tx.conn.UnmarshalObjectWithJsoniter(data, object)
+}
+
+func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
+	data, err := tx.conn.MarshalObject(object)
+	if err != nil {
+		return err
+	}
+
+	bucket := tx.tx.Bucket([]byte(bucketName))
+	return bucket.Put(key, data)
+}
+
+func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+	return bucket.Delete(key)
+}
+
+func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	cursor := bucket.Cursor()
+	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObject(v, &obj)
+		if err != nil {
+			return err
+		}
+
+		if id, ok := matching(obj); ok {
+			err := bucket.Delete(tx.conn.ConvertToKey(id))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+	id, err := bucket.NextSequence()
+	if err != nil {
+		log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifer")
+
+		return 0
+	}
+
+	return int(id)
+}
+
+func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	seqId, _ := bucket.NextSequence()
+	id, obj := fn(seqId)
+
+	data, err := tx.conn.MarshalObject(obj)
+	if err != nil {
+		return err
+	}
+
+	return bucket.Put(tx.conn.ConvertToKey(int(id)), data)
+}
+
+func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+	data, err := tx.conn.MarshalObject(obj)
+	if err != nil {
+		return err
+	}
+
+	return bucket.Put(tx.conn.ConvertToKey(id), data)
+}
+
+func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+	data, err := tx.conn.MarshalObject(obj)
+	if err != nil {
+		return err
+	}
+
+	return bucket.Put(id, data)
+}
+
+func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	cursor := bucket.Cursor()
+	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObject(v, obj)
+		if err != nil {
+			return err
+		}
+
+		obj, err = append(obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	cursor := bucket.Cursor()
+	for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		if err != nil {
+			return err
+		}
+
+		obj, err = append(obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
+
+	for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
+		err := tx.conn.UnmarshalObjectWithJsoniter(v, obj)
+		if err != nil {
+			return err
+		}
+
+		obj, err = append(obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/database/boltdb/tx_test.go

@@ -0,0 +1,126 @@
+package boltdb
+
+import (
+	"errors"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+const testBucketName = "test-bucket"
+const testId = 1234
+
+type testStruct struct {
+	Key   string
+	Value string
+}
+
+func TestTxs(t *testing.T) {
+	conn := DbConnection{
+		Path: t.TempDir(),
+	}
+
+	err := conn.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	// Error propagation
+	err = conn.UpdateTx(func(tx portainer.Transaction) error {
+		return errors.New("this is an error")
+	})
+	if err == nil {
+		t.Fatal("an error was expected, got nil instead")
+	}
+
+	// Create an object
+	newObj := testStruct{
+		Key:   "key",
+		Value: "value",
+	}
+
+	err = conn.UpdateTx(func(tx portainer.Transaction) error {
+		err = tx.SetServiceName(testBucketName)
+		if err != nil {
+			return err
+		}
+
+		return tx.CreateObjectWithId(testBucketName, testId, newObj)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	obj := testStruct{}
+	err = conn.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if obj.Key != newObj.Key || obj.Value != newObj.Value {
+		t.Fatalf("expected %s:%s, got %s:%s instead", newObj.Key, newObj.Value, obj.Key, obj.Value)
+	}
+
+	// Update an object
+	updatedObj := testStruct{
+		Key:   "updated-key",
+		Value: "updated-value",
+	}
+
+	err = conn.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.UpdateObject(testBucketName, conn.ConvertToKey(testId), &updatedObj)
+	})
+
+	err = conn.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if obj.Key != updatedObj.Key || obj.Value != updatedObj.Value {
+		t.Fatalf("expected %s:%s, got %s:%s instead", updatedObj.Key, updatedObj.Value, obj.Key, obj.Value)
+	}
+
+	// Delete an object
+	err = conn.UpdateTx(func(tx portainer.Transaction) error {
+		return tx.DeleteObject(testBucketName, conn.ConvertToKey(testId))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = conn.ViewTx(func(tx portainer.Transaction) error {
+		return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
+	})
+	if err != dserrors.ErrObjectNotFound {
+		t.Fatal(err)
+	}
+
+	// Get next identifier
+	err = conn.UpdateTx(func(tx portainer.Transaction) error {
+		id1 := tx.GetNextIdentifier(testBucketName)
+		id2 := tx.GetNextIdentifier(testBucketName)
+
+		if id1+1 != id2 {
+			return errors.New("unexpected identifier sequence")
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Try to write in a read transaction
+	err = conn.ViewTx(func(tx portainer.Transaction) error {
+		return tx.CreateObjectWithId(testBucketName, testId, newObj)
+	})
+	if err == nil {
+		t.Fatal("an error was expected, got nil instead")
+	}
+}

api/dataservices/edgegroup/edgegroup.go

@@ -1,11 +1,7 @@
 package edgegroup
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/rs/zerolog/log"
 )
 
 // BucketName represents the name of the bucket where this service stores data.
@@ -32,47 +28,46 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-// EdgeGroups return an array containing all the Edge groups.
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
+// EdgeGroups return a slice containing all the Edge groups.
 func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
-	var groups = make([]portainer.EdgeGroup, 0)
+	var groups []portainer.EdgeGroup
+	var err error
 
-	err := service.connection.GetAllWithJsoniter(
-		BucketName,
-		&portainer.EdgeGroup{},
-		func(obj interface{}) (interface{}, error) {
-			group, ok := obj.(*portainer.EdgeGroup)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
-				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
-			}
-			groups = append(groups, *group)
-
-			return &portainer.EdgeGroup{}, nil
-		})
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		groups, err = service.Tx(tx).EdgeGroups()
+		return err
+	})
 
 	return groups, err
 }
 
 // EdgeGroup returns an Edge group by ID.
 func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
-	var group portainer.EdgeGroup
-	identifier := service.connection.ConvertToKey(int(ID))
+	var group *portainer.EdgeGroup
+	var err error
 
-	err := service.connection.GetObject(BucketName, identifier, &group)
-	if err != nil {
-		return nil, err
-	}
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		group, err = service.Tx(tx).EdgeGroup(ID)
+		return err
+	})
 
-	return &group, nil
+	return group, err
 }
 
-// Deprecated: Use UpdateEdgeGroupFunc instead.
+// UpdateEdgeGroup updates an edge group.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, group)
 }
 
-// UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
+// Deprecated: UpdateEdgeGroupFunc updates an edge group inside a transaction avoiding data races.
 func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
 	id := service.connection.ConvertToKey(int(ID))
 	edgeGroup := &portainer.EdgeGroup{}
@@ -84,17 +79,14 @@ func (service *Service) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc
 
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).DeleteEdgeGroup(ID)
+	})
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) Create(group *portainer.EdgeGroup) error {
-	return service.connection.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			group.ID = portainer.EdgeGroupID(id)
-			return int(group.ID), group
-		},
-	)
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Create(group)
+	})
 }

api/dataservices/edgegroup/tx.go

@@ -0,0 +1,80 @@
+package edgegroup
+
+import (
+	"errors"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EdgeGroups return a slice containing all the Edge groups.
+func (service ServiceTx) EdgeGroups() ([]portainer.EdgeGroup, error) {
+	var groups = make([]portainer.EdgeGroup, 0)
+
+	err := service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.EdgeGroup{},
+		func(obj interface{}) (interface{}, error) {
+			group, ok := obj.(*portainer.EdgeGroup)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
+				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
+			}
+			groups = append(groups, *group)
+
+			return &portainer.EdgeGroup{}, nil
+		})
+
+	return groups, err
+}
+
+// EdgeGroup returns an Edge group by ID.
+func (service ServiceTx) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
+	var group portainer.EdgeGroup
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &group)
+	if err != nil {
+		return nil, err
+	}
+
+	return &group, nil
+}
+
+// UpdateEdgeGroup updates an edge group.
+func (service ServiceTx) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, group)
+}
+
+// UpdateEdgeGroupFunc is a no-op inside a transaction.
+func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFunc func(edgeGroup *portainer.EdgeGroup)) error {
+	return errors.New("cannot be called inside a transaction")
+}
+
+// DeleteEdgeGroup deletes an Edge group.
+func (service ServiceTx) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			group.ID = portainer.EdgeGroupID(id)
+			return int(group.ID), group
+		},
+	)
+}

api/dataservices/edgejob/edgejob.go

@@ -8,10 +8,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "edgejobs"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "edgejobs"
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
@@ -34,6 +32,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // EdgeJobs returns a list of Edge jobs
 func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
@@ -80,12 +85,22 @@ func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJo
 	)
 }
 
-// UpdateEdgeJob updates an Edge job by ID
+// Deprecated: use UpdateEdgeJobFunc instead
 func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
 }
 
+// UpdateEdgeJobFunc updates an edge job inside a transaction avoiding data races.
+func (service *Service) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
+	id := service.connection.ConvertToKey(int(ID))
+	edgeJob := &portainer.EdgeJob{}
+
+	return service.connection.UpdateObjectFunc(BucketName, id, edgeJob, func() {
+		updateFunc(edgeJob)
+	})
+}
+
 // DeleteEdgeJob deletes an Edge job
 func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := service.connection.ConvertToKey(int(ID))

api/dataservices/edgejob/tx.go

@@ -0,0 +1,84 @@
+package edgejob
+
+import (
+	"errors"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EdgeJobs returns a list of Edge jobs
+func (service ServiceTx) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EdgeJob{},
+		func(obj interface{}) (interface{}, error) {
+			job, ok := obj.(*portainer.EdgeJob)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
+				return nil, fmt.Errorf("failed to convert to EdgeJob object: %s", obj)
+			}
+
+			edgeJobs = append(edgeJobs, *job)
+
+			return &portainer.EdgeJob{}, nil
+		})
+
+	return edgeJobs, err
+}
+
+// EdgeJob returns an Edge job by ID
+func (service ServiceTx) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
+}
+
+// Create creates a new EdgeJob
+func (service ServiceTx) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.tx.CreateObjectWithId(BucketName, int(edgeJob.ID), edgeJob)
+}
+
+// UpdateEdgeJob updates an edge job
+func (service ServiceTx) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, edgeJob)
+}
+
+// UpdateEdgeJobFunc is a no-op inside a transaction.
+func (service ServiceTx) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
+	return errors.New("cannot be called inside a transaction")
+}
+
+// DeleteEdgeJob deletes an Edge job
+func (service ServiceTx) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.tx.GetNextIdentifier(BucketName)
+}

api/dataservices/edgestack/edgestack.go

@@ -5,21 +5,19 @@ import (
 	"sync"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/edge/cache"
 
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "edge_stack"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "edge_stack"
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	connection portainer.Connection
-	idxVersion map[portainer.EdgeStackID]int
-	mu         sync.RWMutex
+	connection          portainer.Connection
+	idxVersion          map[portainer.EdgeStackID]int
+	mu                  sync.RWMutex
+	cacheInvalidationFn func(portainer.EdgeStackID)
 }
 
 func (service *Service) BucketName() string {
@@ -27,15 +25,20 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
+func NewService(connection portainer.Connection, cacheInvalidationFn func(portainer.EdgeStackID)) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	s := &Service{
-		connection: connection,
-		idxVersion: make(map[portainer.EdgeStackID]int),
+		connection:          connection,
+		idxVersion:          make(map[portainer.EdgeStackID]int),
+		cacheInvalidationFn: cacheInvalidationFn,
+	}
+
+	if s.cacheInvalidationFn == nil {
+		s.cacheInvalidationFn = func(portainer.EdgeStackID) {}
 	}
 
 	es, err := s.EdgeStacks()
@@ -50,6 +53,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	return s, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // EdgeStacks returns an array containing all edge stacks
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
@@ -109,12 +119,9 @@ func (service *Service) Create(id portainer.EdgeStackID, edgeStack *portainer.Ed
 
 	service.mu.Lock()
 	service.idxVersion[id] = edgeStack.Version
+	service.cacheInvalidationFn(id)
 	service.mu.Unlock()
 
-	for endpointID := range edgeStack.Status {
-		cache.Del(endpointID)
-	}
-
 	return nil
 }
 
@@ -123,37 +130,15 @@ func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *por
 	service.mu.Lock()
 	defer service.mu.Unlock()
 
-	prevEdgeStack, err := service.EdgeStack(ID)
-	if err != nil {
-		return err
-	}
-
 	identifier := service.connection.ConvertToKey(int(ID))
 
-	err = service.connection.UpdateObject(BucketName, identifier, edgeStack)
+	err := service.connection.UpdateObject(BucketName, identifier, edgeStack)
 	if err != nil {
 		return err
 	}
 
 	service.idxVersion[ID] = edgeStack.Version
-
-	// Invalidate cache for removed environments
-	for endpointID := range prevEdgeStack.Status {
-		if _, ok := edgeStack.Status[endpointID]; !ok {
-			cache.Del(endpointID)
-		}
-	}
-
-	// Invalidate cache when version changes and for added environments
-	for endpointID := range edgeStack.Status {
-		if prevEdgeStack.Version == edgeStack.Version {
-			if _, ok := prevEdgeStack.Status[endpointID]; ok {
-				continue
-			}
-		}
-
-		cache.Del(endpointID)
-	}
+	service.cacheInvalidationFn(ID)
 
 	return nil
 }
@@ -167,35 +152,10 @@ func (service *Service) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc
 	defer service.mu.Unlock()
 
 	return service.connection.UpdateObjectFunc(BucketName, id, edgeStack, func() {
-		prevEndpoints := make(map[portainer.EndpointID]struct{}, len(edgeStack.Status))
-		for endpointID := range edgeStack.Status {
-			if _, ok := edgeStack.Status[endpointID]; !ok {
-				prevEndpoints[endpointID] = struct{}{}
-			}
-		}
-
 		updateFunc(edgeStack)
 
-		prevVersion := service.idxVersion[ID]
 		service.idxVersion[ID] = edgeStack.Version
-
-		// Invalidate cache for removed environments
-		for endpointID := range prevEndpoints {
-			if _, ok := edgeStack.Status[endpointID]; !ok {
-				cache.Del(endpointID)
-			}
-		}
-
-		// Invalidate cache when version changes and for added environments
-		for endpointID := range edgeStack.Status {
-			if prevVersion == edgeStack.Version {
-				if _, ok := prevEndpoints[endpointID]; ok {
-					continue
-				}
-			}
-
-			cache.Del(endpointID)
-		}
+		service.cacheInvalidationFn(ID)
 	})
 }
 
@@ -204,23 +164,16 @@ func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	service.mu.Lock()
 	defer service.mu.Unlock()
 
-	edgeStack, err := service.EdgeStack(ID)
-	if err != nil {
-		return err
-	}
-
 	identifier := service.connection.ConvertToKey(int(ID))
 
-	err = service.connection.DeleteObject(BucketName, identifier)
+	err := service.connection.DeleteObject(BucketName, identifier)
 	if err != nil {
 		return err
 	}
 
 	delete(service.idxVersion, ID)
 
-	for endpointID := range edgeStack.Status {
-		cache.Del(endpointID)
-	}
+	service.cacheInvalidationFn(ID)
 
 	return nil
 }

api/dataservices/edgestack/tx.go

@@ -0,0 +1,131 @@
+package edgestack
+
+import (
+	"errors"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EdgeStacks returns an array containing all edge stacks
+func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
+	var stacks = make([]portainer.EdgeStack, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EdgeStack{},
+		func(obj interface{}) (interface{}, error) {
+			stack, ok := obj.(*portainer.EdgeStack)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
+				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
+			}
+
+			stacks = append(stacks, *stack)
+
+			return &portainer.EdgeStack{}, nil
+		})
+
+	return stacks, err
+}
+
+// EdgeStack returns an Edge stack by ID.
+func (service ServiceTx) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error) {
+	var stack portainer.EdgeStack
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
+
+	return &stack, nil
+}
+
+// EdgeStackVersion returns the version of the given edge stack ID directly from an in-memory index
+func (service ServiceTx) EdgeStackVersion(ID portainer.EdgeStackID) (int, bool) {
+	service.service.mu.RLock()
+	v, ok := service.service.idxVersion[ID]
+	service.service.mu.RUnlock()
+
+	return v, ok
+}
+
+// CreateEdgeStack saves an Edge stack object to db.
+func (service ServiceTx) Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
+	edgeStack.ID = id
+
+	err := service.tx.CreateObjectWithId(
+		BucketName,
+		int(edgeStack.ID),
+		edgeStack,
+	)
+	if err != nil {
+		return err
+	}
+
+	service.service.mu.Lock()
+	service.service.idxVersion[id] = edgeStack.Version
+	service.service.cacheInvalidationFn(id)
+	service.service.mu.Unlock()
+
+	return nil
+}
+
+// UpdateEdgeStack updates an Edge stack.
+func (service ServiceTx) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
+	service.service.mu.Lock()
+	defer service.service.mu.Unlock()
+
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.UpdateObject(BucketName, identifier, edgeStack)
+	if err != nil {
+		return err
+	}
+
+	service.service.idxVersion[ID] = edgeStack.Version
+	service.service.cacheInvalidationFn(ID)
+
+	return nil
+}
+
+// UpdateEdgeStackFunc is a no-op inside a transaction.
+func (service ServiceTx) UpdateEdgeStackFunc(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error {
+	return errors.New("cannot be called inside a transaction")
+}
+
+// DeleteEdgeStack deletes an Edge stack.
+func (service ServiceTx) DeleteEdgeStack(ID portainer.EdgeStackID) error {
+	service.service.mu.Lock()
+	defer service.service.mu.Unlock()
+
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.DeleteObject(BucketName, identifier)
+	if err != nil {
+		return err
+	}
+
+	delete(service.service.idxVersion, ID)
+
+	service.service.cacheInvalidationFn(ID)
+
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.tx.GetNextIdentifier(BucketName)
+}

api/dataservices/endpoint/endpoint.go

@@ -1,20 +1,14 @@
 package endpoint
 
 import (
-	"fmt"
 	"sync"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/edge/cache"
-
-	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "endpoints"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "endpoints"
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
@@ -56,84 +50,54 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	return s, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // Endpoint returns an environment(endpoint) by ID.
 func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
-	var endpoint portainer.Endpoint
-	identifier := service.connection.ConvertToKey(int(ID))
+	var endpoint *portainer.Endpoint
+	var err error
 
-	err := service.connection.GetObject(BucketName, identifier, &endpoint)
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		endpoint, err = service.Tx(tx).Endpoint(ID)
+		return err
+	})
 	if err != nil {
 		return nil, err
 	}
 
 	endpoint.LastCheckInDate, _ = service.Heartbeat(ID)
 
-	return &endpoint, nil
+	return endpoint, nil
 }
 
 // UpdateEndpoint updates an environment(endpoint).
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.UpdateObject(BucketName, identifier, endpoint)
-	if err != nil {
-		return err
-	}
-
-	service.mu.Lock()
-	if len(endpoint.EdgeID) > 0 {
-		service.idxEdgeID[endpoint.EdgeID] = ID
-	}
-	service.heartbeats.Store(ID, endpoint.LastCheckInDate)
-	service.mu.Unlock()
-
-	cache.Del(endpoint.ID)
-
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).UpdateEndpoint(ID, endpoint)
+	})
 }
 
 // DeleteEndpoint deletes an environment(endpoint).
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-
-	err := service.connection.DeleteObject(BucketName, identifier)
-	if err != nil {
-		return err
-	}
-
-	service.mu.Lock()
-	for edgeID, endpointID := range service.idxEdgeID {
-		if endpointID == ID {
-			delete(service.idxEdgeID, edgeID)
-			break
-		}
-	}
-	service.heartbeats.Delete(ID)
-	service.mu.Unlock()
-
-	cache.Del(ID)
-
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).DeleteEndpoint(ID)
+	})
 }
 
 // Endpoints return an array containing all the environments(endpoints).
 func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
+	var endpoints []portainer.Endpoint
+	var err error
 
-	err := service.connection.GetAllWithJsoniter(
-		BucketName,
-		&portainer.Endpoint{},
-		func(obj interface{}) (interface{}, error) {
-			endpoint, ok := obj.(*portainer.Endpoint)
-			if !ok {
-				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
-				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
-			}
-
-			endpoints = append(endpoints, *endpoint)
-
-			return &portainer.Endpoint{}, nil
-		})
+	err = service.connection.ViewTx(func(tx portainer.Transaction) error {
+		endpoints, err = service.Tx(tx).Endpoints()
+		return err
+	})
 
 	if err != nil {
 		return endpoints, err
@@ -170,22 +134,20 @@ func (service *Service) UpdateHeartbeat(endpointID portainer.EndpointID) {
 
 // CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
 func (service *Service) Create(endpoint *portainer.Endpoint) error {
-	err := service.connection.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
-	if err != nil {
-		return err
-	}
-
-	service.mu.Lock()
-	if len(endpoint.EdgeID) > 0 {
-		service.idxEdgeID[endpoint.EdgeID] = endpoint.ID
-	}
-	service.heartbeats.Store(endpoint.ID, endpoint.LastCheckInDate)
-	service.mu.Unlock()
-
-	return nil
+	return service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return service.Tx(tx).Create(endpoint)
+	})
 }
 
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
 func (service *Service) GetNextIdentifier() int {
-	return service.connection.GetNextIdentifier(BucketName)
+	var identifier int
+
+	service.connection.UpdateTx(func(tx portainer.Transaction) error {
+		identifier = service.Tx(tx).GetNextIdentifier()
+
+		return nil
+	})
+
+	return identifier
 }

api/dataservices/endpoint/tx.go

@@ -0,0 +1,137 @@
+package endpoint
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge/cache"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Endpoint returns an environment(endpoint) by ID.
+func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
+	var endpoint portainer.Endpoint
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpoint, nil
+}
+
+// UpdateEndpoint updates an environment(endpoint).
+func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.UpdateObject(BucketName, identifier, endpoint)
+	if err != nil {
+		return err
+	}
+
+	service.service.mu.Lock()
+	if len(endpoint.EdgeID) > 0 {
+		service.service.idxEdgeID[endpoint.EdgeID] = ID
+	}
+	service.service.heartbeats.Store(ID, endpoint.LastCheckInDate)
+	service.service.mu.Unlock()
+
+	cache.Del(endpoint.ID)
+
+	return nil
+}
+
+// DeleteEndpoint deletes an environment(endpoint).
+func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.DeleteObject(BucketName, identifier)
+	if err != nil {
+		return err
+	}
+
+	service.service.mu.Lock()
+	for edgeID, endpointID := range service.service.idxEdgeID {
+		if endpointID == ID {
+			delete(service.service.idxEdgeID, edgeID)
+			break
+		}
+	}
+	service.service.heartbeats.Delete(ID)
+	service.service.mu.Unlock()
+
+	cache.Del(ID)
+
+	return nil
+}
+
+// Endpoints return an array containing all the environments(endpoints).
+func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	err := service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Endpoint{},
+		func(obj interface{}) (interface{}, error) {
+			endpoint, ok := obj.(*portainer.Endpoint)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
+				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
+			}
+
+			endpoints = append(endpoints, *endpoint)
+
+			return &portainer.Endpoint{}, nil
+		})
+
+	return endpoints, err
+}
+
+func (service ServiceTx) EndpointIDByEdgeID(edgeID string) (portainer.EndpointID, bool) {
+	log.Error().Str("func", "EndpointIDByEdgeID").Msg("cannot be called inside a transaction")
+
+	return 0, false
+}
+
+func (service ServiceTx) Heartbeat(endpointID portainer.EndpointID) (int64, bool) {
+	log.Error().Str("func", "Heartbeat").Msg("cannot be called inside a transaction")
+
+	return 0, false
+}
+
+func (service ServiceTx) UpdateHeartbeat(endpointID portainer.EndpointID) {
+	log.Error().Str("func", "UpdateHeartbeat").Msg("cannot be called inside a transaction")
+}
+
+// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
+func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
+	err := service.tx.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
+	if err != nil {
+		return err
+	}
+
+	service.service.mu.Lock()
+	if len(endpoint.EdgeID) > 0 {
+		service.service.idxEdgeID[endpoint.EdgeID] = endpoint.ID
+	}
+	service.service.heartbeats.Store(endpoint.ID, endpoint.LastCheckInDate)
+	service.service.mu.Unlock()
+
+	return nil
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service ServiceTx) GetNextIdentifier() int {
+	return service.tx.GetNextIdentifier(BucketName)
+}

api/dataservices/endpointgroup/endpointgroup.go

@@ -34,6 +34,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // EndpointGroup returns an environment(endpoint) group by ID.
 func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
 	var endpointGroup portainer.EndpointGroup

api/dataservices/endpointgroup/tx.go

@@ -0,0 +1,76 @@
+package endpointgroup
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EndpointGroup returns an environment(endpoint) group by ID.
+func (service ServiceTx) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
+	var endpointGroup portainer.EndpointGroup
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &endpointGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointGroup, nil
+}
+
+// UpdateEndpointGroup updates an environment(endpoint) group.
+func (service ServiceTx) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, endpointGroup)
+}
+
+// DeleteEndpointGroup deletes an environment(endpoint) group.
+func (service ServiceTx) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+// EndpointGroups return an array containing all the environment(endpoint) groups.
+func (service ServiceTx) EndpointGroups() ([]portainer.EndpointGroup, error) {
+	var endpointGroups = make([]portainer.EndpointGroup, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EndpointGroup{},
+		func(obj interface{}) (interface{}, error) {
+			endpointGroup, ok := obj.(*portainer.EndpointGroup)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
+				return nil, fmt.Errorf("failed to convert to EndpointGroup object: %s", obj)
+			}
+
+			endpointGroups = append(endpointGroups, *endpointGroup)
+
+			return &portainer.EndpointGroup{}, nil
+		})
+
+	return endpointGroups, err
+}
+
+// CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
+func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			endpointGroup.ID = portainer.EndpointGroupID(id)
+			return int(endpointGroup.ID), endpointGroup
+		},
+	)
+}

api/dataservices/endpointrelation/endpointrelation.go

@@ -9,20 +9,23 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "endpoint_relations"
-)
+// BucketName represents the name of the bucket where this service stores data.
+const BucketName = "endpoint_relations"
 
 // Service represents a service for managing environment(endpoint) relation data.
 type Service struct {
-	connection portainer.Connection
+	connection    portainer.Connection
+	updateStackFn func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error
 }
 
 func (service *Service) BucketName() string {
 	return BucketName
 }
 
+func (service *Service) RegisterUpdateStackFunction(updateFunc func(ID portainer.EdgeStackID, updateFunc func(edgeStack *portainer.EdgeStack)) error) {
+	service.updateStackFn = updateFunc
+}
+
 // NewService creates a new instance of a service.
 func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
@@ -35,6 +38,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // EndpointRelations returns an array of all EndpointRelations
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
@@ -80,18 +90,97 @@ func (service *Service) Create(endpointRelation *portainer.EndpointRelation) err
 
 // UpdateEndpointRelation updates an Environment(Endpoint) relation object
 func (service *Service) UpdateEndpointRelation(endpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
+	previousRelationState, _ := service.EndpointRelation(endpointID)
+
 	identifier := service.connection.ConvertToKey(int(endpointID))
 	err := service.connection.UpdateObject(BucketName, identifier, endpointRelation)
 	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
-	return err
+	updatedRelationState, _ := service.EndpointRelation(endpointID)
+
+	service.updateEdgeStacksAfterRelationChange(previousRelationState, updatedRelationState)
+
+	return nil
 }
 
 // DeleteEndpointRelation deletes an Environment(Endpoint) relation object
 func (service *Service) DeleteEndpointRelation(endpointID portainer.EndpointID) error {
+	deletedRelation, _ := service.EndpointRelation(endpointID)
+
 	identifier := service.connection.ConvertToKey(int(endpointID))
 	err := service.connection.DeleteObject(BucketName, identifier)
 	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
 
-	return err
+	service.updateEdgeStacksAfterRelationChange(deletedRelation, nil)
+
+	return nil
+}
+
+func (service *Service) InvalidateEdgeCacheForEdgeStack(edgeStackID portainer.EdgeStackID) {
+	rels, err := service.EndpointRelations()
+	if err != nil {
+		log.Error().Err(err).Msg("cannot retrieve endpoint relations")
+		return
+	}
+
+	for _, rel := range rels {
+		for id := range rel.EdgeStacks {
+			if edgeStackID == id {
+				cache.Del(rel.EndpointID)
+			}
+		}
+	}
+}
+
+func (service *Service) updateEdgeStacksAfterRelationChange(previousRelationState *portainer.EndpointRelation, updatedRelationState *portainer.EndpointRelation) {
+	relations, _ := service.EndpointRelations()
+
+	stacksToUpdate := map[portainer.EdgeStackID]bool{}
+
+	if previousRelationState != nil {
+		for stackId, enabled := range previousRelationState.EdgeStacks {
+			// flag stack for update if stack is not in the updated relation state
+			// = stack has been removed for this relation
+			// or this relation has been deleted
+			if enabled && (updatedRelationState == nil || !updatedRelationState.EdgeStacks[stackId]) {
+				stacksToUpdate[stackId] = true
+			}
+		}
+	}
+
+	if updatedRelationState != nil {
+		for stackId, enabled := range updatedRelationState.EdgeStacks {
+			// flag stack for update if stack is not in the previous relation state
+			// = stack has been added for this relation
+			if enabled && (previousRelationState == nil || !previousRelationState.EdgeStacks[stackId]) {
+				stacksToUpdate[stackId] = true
+			}
+		}
+	}
+
+	// for each stack referenced by the updated relation
+	// list how many time this stack is referenced in all relations
+	// in order to update the stack deployments count
+	for refStackId, refStackEnabled := range stacksToUpdate {
+		if refStackEnabled {
+			numDeployments := 0
+			for _, r := range relations {
+				for sId, enabled := range r.EdgeStacks {
+					if enabled && sId == refStackId {
+						numDeployments += 1
+					}
+				}
+			}
+
+			service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
+				edgeStack.NumDeployments = numDeployments
+			})
+		}
+	}
 }

api/dataservices/endpointrelation/tx.go

@@ -0,0 +1,159 @@
+package endpointrelation
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge/cache"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// EndpointRelations returns an array of all EndpointRelations
+func (service ServiceTx) EndpointRelations() ([]portainer.EndpointRelation, error) {
+	var all = make([]portainer.EndpointRelation, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.EndpointRelation{},
+		func(obj interface{}) (interface{}, error) {
+			r, ok := obj.(*portainer.EndpointRelation)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
+				return nil, fmt.Errorf("failed to convert to EndpointRelation object: %s", obj)
+			}
+
+			all = append(all, *r)
+
+			return &portainer.EndpointRelation{}, nil
+		})
+
+	return all, err
+}
+
+// EndpointRelation returns an Environment(Endpoint) relation object by EndpointID
+func (service ServiceTx) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
+	var endpointRelation portainer.EndpointRelation
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+
+	err := service.tx.GetObject(BucketName, identifier, &endpointRelation)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointRelation, nil
+}
+
+// CreateEndpointRelation saves endpointRelation
+func (service ServiceTx) Create(endpointRelation *portainer.EndpointRelation) error {
+	err := service.tx.CreateObjectWithId(BucketName, int(endpointRelation.EndpointID), endpointRelation)
+	cache.Del(endpointRelation.EndpointID)
+
+	return err
+}
+
+// UpdateEndpointRelation updates an Environment(Endpoint) relation object
+func (service ServiceTx) UpdateEndpointRelation(endpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
+	previousRelationState, _ := service.EndpointRelation(endpointID)
+
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	err := service.tx.UpdateObject(BucketName, identifier, endpointRelation)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
+
+	updatedRelationState, _ := service.EndpointRelation(endpointID)
+
+	service.updateEdgeStacksAfterRelationChange(previousRelationState, updatedRelationState)
+
+	return nil
+}
+
+// DeleteEndpointRelation deletes an Environment(Endpoint) relation object
+func (service ServiceTx) DeleteEndpointRelation(endpointID portainer.EndpointID) error {
+	deletedRelation, _ := service.EndpointRelation(endpointID)
+
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	err := service.tx.DeleteObject(BucketName, identifier)
+	cache.Del(endpointID)
+	if err != nil {
+		return err
+	}
+
+	service.updateEdgeStacksAfterRelationChange(deletedRelation, nil)
+
+	return nil
+}
+
+func (service ServiceTx) InvalidateEdgeCacheForEdgeStack(edgeStackID portainer.EdgeStackID) {
+	rels, err := service.EndpointRelations()
+	if err != nil {
+		log.Error().Err(err).Msg("cannot retrieve endpoint relations")
+		return
+	}
+
+	for _, rel := range rels {
+		for id := range rel.EdgeStacks {
+			if edgeStackID == id {
+				cache.Del(rel.EndpointID)
+			}
+		}
+	}
+}
+
+func (service ServiceTx) updateEdgeStacksAfterRelationChange(previousRelationState *portainer.EndpointRelation, updatedRelationState *portainer.EndpointRelation) {
+	relations, _ := service.EndpointRelations()
+
+	stacksToUpdate := map[portainer.EdgeStackID]bool{}
+
+	if previousRelationState != nil {
+		for stackId, enabled := range previousRelationState.EdgeStacks {
+			// flag stack for update if stack is not in the updated relation state
+			// = stack has been removed for this relation
+			// or this relation has been deleted
+			if enabled && (updatedRelationState == nil || !updatedRelationState.EdgeStacks[stackId]) {
+				stacksToUpdate[stackId] = true
+			}
+		}
+	}
+
+	if updatedRelationState != nil {
+		for stackId, enabled := range updatedRelationState.EdgeStacks {
+			// flag stack for update if stack is not in the previous relation state
+			// = stack has been added for this relation
+			if enabled && (previousRelationState == nil || !previousRelationState.EdgeStacks[stackId]) {
+				stacksToUpdate[stackId] = true
+			}
+		}
+	}
+
+	// for each stack referenced by the updated relation
+	// list how many time this stack is referenced in all relations
+	// in order to update the stack deployments count
+	for refStackId, refStackEnabled := range stacksToUpdate {
+		if refStackEnabled {
+			numDeployments := 0
+			for _, r := range relations {
+				for sId, enabled := range r.EdgeStacks {
+					if enabled && sId == refStackId {
+						numDeployments += 1
+					}
+				}
+			}
+
+			service.service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
+				edgeStack.NumDeployments = numDeployments
+			})
+		}
+	}
+}

api/dataservices/interface.go

@@ -13,16 +13,7 @@ import (
 )
 
 type (
-	// DataStore defines the interface to manage the data
-	DataStore interface {
-		Open() (newStore bool, err error)
-		Init() error
-		Close() error
-		MigrateData() error
-		Rollback(force bool) error
-		CheckCurrentEdition() error
-		BackupTo(w io.Writer) error
-		Export(filename string) (err error)
+	DataStoreTx interface {
 		IsErrObjectNotFound(err error) bool
 		CustomTemplate() CustomTemplateService
 		EdgeGroup() EdgeGroupService
@@ -50,6 +41,22 @@ type (
 		Webhook() WebhookService
 	}
 
+	// DataStore defines the interface to manage the data
+	DataStore interface {
+		Open() (newStore bool, err error)
+		Init() error
+		Close() error
+		UpdateTx(func(DataStoreTx) error) error
+		ViewTx(func(DataStoreTx) error) error
+		MigrateData() error
+		Rollback(force bool) error
+		CheckCurrentEdition() error
+		BackupTo(w io.Writer) error
+		Export(filename string) (err error)
+
+		DataStoreTx
+	}
+
 	// CustomTemplateService represents a service to manage custom templates
 	CustomTemplateService interface {
 		GetNextIdentifier() int
@@ -78,6 +85,7 @@ type (
 		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
 		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error
 		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
 		BucketName() string
@@ -204,7 +212,6 @@ type (
 	SettingsService interface {
 		Settings() (*portainer.Settings, error)
 		UpdateSettings(settings *portainer.Settings) error
-		IsFeatureFlagEnabled(feature portainer.Feature) bool
 		BucketName() string
 	}
 

api/dataservices/settings/settings.go

@@ -47,17 +47,3 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
 	return service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
 }
-
-func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
-	settings, err := service.Settings()
-	if err != nil {
-		return false
-	}
-
-	featureFlagSetting, ok := settings.FeatureFlagSettings[feature]
-	if ok {
-		return featureFlagSetting
-	}
-
-	return false
-}

api/dataservices/snapshot/snapshot.go

@@ -31,6 +31,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
 	var snapshot portainer.Snapshot
 	identifier := service.connection.ConvertToKey(int(endpointID))

api/dataservices/snapshot/tx.go

@@ -0,0 +1,63 @@
+package snapshot
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+func (service ServiceTx) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot portainer.Snapshot
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+
+	err := service.tx.GetObject(BucketName, identifier, &snapshot)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service ServiceTx) Snapshots() ([]portainer.Snapshot, error) {
+	var snapshots = make([]portainer.Snapshot, 0)
+
+	err := service.tx.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Snapshot{},
+		func(obj interface{}) (interface{}, error) {
+			snapshot, ok := obj.(*portainer.Snapshot)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
+				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
+			}
+			snapshots = append(snapshots, *snapshot)
+			return &portainer.Snapshot{}, nil
+		})
+
+	return snapshots, err
+}
+
+func (service ServiceTx) UpdateSnapshot(snapshot *portainer.Snapshot) error {
+	identifier := service.service.connection.ConvertToKey(int(snapshot.EndpointID))
+	return service.tx.UpdateObject(BucketName, identifier, snapshot)
+}
+
+func (service ServiceTx) DeleteSnapshot(endpointID portainer.EndpointID) error {
+	identifier := service.service.connection.ConvertToKey(int(endpointID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}
+
+func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
+	return service.tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+}

api/dataservices/stack/tests/stack_test.go

@@ -68,13 +68,13 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 
 	if webhookID == "" {
 		if b.count%2 == 0 {
-			stack.AutoUpdate = &portainer.StackAutoUpdate{
+			stack.AutoUpdate = &portainer.AutoUpdateSettings{
 				Interval: "",
 				Webhook:  "",
 			}
 		} // else keep AutoUpdate nil
 	} else {
-		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
+		stack.AutoUpdate = &portainer.AutoUpdateSettings{Webhook: webhookID}
 	}
 
 	err := b.store.StackService.Create(&stack)
@@ -91,8 +91,8 @@ func Test_RefreshableStacks(t *testing.T) {
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}
-	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
-	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
+	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}
+	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
 
 	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
 		err := store.Stack().Create(stack)

api/dataservices/tag/tag.go

@@ -34,6 +34,13 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
+func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: service,
+		tx:      tx,
+	}
+}
+
 // Tags return an array containing all the tags.
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)

api/dataservices/tag/tx.go

@@ -0,0 +1,82 @@
+package tag
+
+import (
+	"errors"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) BucketName() string {
+	return BucketName
+}
+
+// Tags return an array containing all the tags.
+func (service ServiceTx) Tags() ([]portainer.Tag, error) {
+	var tags = make([]portainer.Tag, 0)
+
+	err := service.tx.GetAll(
+		BucketName,
+		&portainer.Tag{},
+		func(obj interface{}) (interface{}, error) {
+			tag, ok := obj.(*portainer.Tag)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
+				return nil, fmt.Errorf("failed to convert to Tag object: %s", obj)
+			}
+
+			tags = append(tags, *tag)
+
+			return &portainer.Tag{}, nil
+		})
+
+	return tags, err
+}
+
+// Tag returns a tag by ID.
+func (service ServiceTx) Tag(ID portainer.TagID) (*portainer.Tag, error) {
+	var tag portainer.Tag
+	identifier := service.service.connection.ConvertToKey(int(ID))
+
+	err := service.tx.GetObject(BucketName, identifier, &tag)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tag, nil
+}
+
+// CreateTag creates a new tag.
+func (service ServiceTx) Create(tag *portainer.Tag) error {
+	return service.tx.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			tag.ID = portainer.TagID(id)
+			return int(tag.ID), tag
+		},
+	)
+}
+
+// UpdateTag updates a tag
+func (service ServiceTx) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.UpdateObject(BucketName, identifier, tag)
+}
+
+// UpdateTagFunc is a no-op inside a transaction
+func (service ServiceTx) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
+	return errors.New("cannot be called inside a transaction")
+}
+
+// DeleteTag deletes a tag.
+func (service ServiceTx) DeleteTag(ID portainer.TagID) error {
+	identifier := service.service.connection.ConvertToKey(int(ID))
+	return service.tx.DeleteObject(BucketName, identifier)
+}

api/datastore/datastore.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
 
 	"github.com/rs/zerolog/log"
@@ -61,6 +62,24 @@ func (store *Store) Close() error {
 	return store.connection.Close()
 }
 
+func (store *Store) UpdateTx(fn func(dataservices.DataStoreTx) error) error {
+	return store.connection.UpdateTx(func(tx portainer.Transaction) error {
+		return fn(&StoreTx{
+			store: store,
+			tx:    tx,
+		})
+	})
+}
+
+func (store *Store) ViewTx(fn func(dataservices.DataStoreTx) error) error {
+	return store.connection.ViewTx(func(tx portainer.Transaction) error {
+		return fn(&StoreTx{
+			store: store,
+			tx:    tx,
+		})
+	})
+}
+
 // BackupTo backs up db to a provided writer.
 // It does hot backup and doesn't block other database reads and writes
 func (store *Store) BackupTo(w io.Writer) error {

api/datastore/migrate_data.go

@@ -82,6 +82,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		DockerhubService:        store.DockerHubService,
 		AuthorizationService:    authorization.NewService(store),
 		EdgeStackService:        store.EdgeStackService,
+		EdgeJobService:          store.EdgeJobService,
 	}
 }
 

api/datastore/migrate_post_init.go

@@ -0,0 +1,116 @@
+package datastore
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/docker"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/rs/zerolog/log"
+)
+
+type PostInitMigrator struct {
+	kubeFactory   *cli.ClientFactory
+	dockerFactory *docker.ClientFactory
+	dataStore     dataservices.DataStore
+}
+
+func NewPostInitMigrator(
+	kubeFactory *cli.ClientFactory,
+	dockerFactory *docker.ClientFactory,
+	dataStore dataservices.DataStore,
+) *PostInitMigrator {
+	return &PostInitMigrator{
+		kubeFactory:   kubeFactory,
+		dockerFactory: dockerFactory,
+		dataStore:     dataStore,
+	}
+}
+
+func (migrator *PostInitMigrator) PostInitMigrate() error {
+	if err := migrator.PostInitMigrateIngresses(); err != nil {
+		return err
+	}
+
+	migrator.PostInitMigrateGPUs()
+
+	return nil
+}
+
+func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
+	endpoints, err := migrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		return err
+	}
+	for i := range endpoints {
+		// Early exit if we do not need to migrate!
+		if endpoints[i].PostInitMigrations.MigrateIngresses == false {
+			return nil
+		}
+
+		err := migrator.kubeFactory.MigrateEndpointIngresses(&endpoints[i])
+		if err != nil {
+			log.Debug().Err(err).Msg("failure migrating endpoint ingresses")
+		}
+	}
+
+	return nil
+}
+
+// PostInitMigrateGPUs will check all docker endpoints for containers with GPUs and set EnableGPUManagement to true if any are found
+// If there's an error getting the containers, we'll log it and move on
+func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
+	environments, err := migrator.dataStore.Endpoint().Endpoints()
+	if err != nil {
+		log.Err(err).Msg("failure getting endpoints")
+		return
+	}
+	for i := range environments {
+		if environments[i].Type == portainer.DockerEnvironment {
+			// // Early exit if we do not need to migrate!
+			if environments[i].PostInitMigrations.MigrateGPUs == false {
+				return
+			}
+
+			// set the MigrateGPUs flag to false so we don't run this again
+			environments[i].PostInitMigrations.MigrateGPUs = false
+			migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
+
+			// create a docker client
+			dockerClient, err := migrator.dockerFactory.CreateClient(&environments[i], "", nil)
+			if err != nil {
+				log.Err(err).Msg("failure creating docker client for environment: " + environments[i].Name)
+				return
+			}
+			defer dockerClient.Close()
+
+			// get all containers
+			containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+			if err != nil {
+				log.Err(err).Msg("failed to list containers")
+				return
+			}
+
+			// check for a gpu on each container. If even one GPU is found, set EnableGPUManagement to true for the whole endpoint
+		containersLoop:
+			for _, container := range containers {
+				// https://www.sobyte.net/post/2022-10/go-docker/ has nice documentation on the docker client with GPUs
+				containerDetails, err := dockerClient.ContainerInspect(context.Background(), container.ID)
+				if err != nil {
+					log.Err(err).Msg("failed to inspect container")
+					return
+				}
+				deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
+				for _, deviceRequest := range deviceRequests {
+					if deviceRequest.Driver == "nvidia" {
+						environments[i].EnableGPUManagement = true
+						migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
+						break containersLoop
+					}
+				}
+			}
+		}
+	}
+}

api/datastore/migrator/migrate_dbversion90.go

@@ -0,0 +1,92 @@
+package migrator
+
+import (
+	"github.com/rs/zerolog/log"
+
+	portainer "github.com/portainer/portainer/api"
+	portainerDsErrors "github.com/portainer/portainer/api/dataservices/errors"
+)
+
+func (m *Migrator) migrateDBVersionToDB90() error {
+	if err := m.updateUserThemeForDB90(); err != nil {
+		return err
+	}
+
+	if err := m.updateEnableGpuManagementFeatures(); err != nil {
+		return err
+	}
+
+	return m.updateEdgeStackStatusForDB90()
+}
+
+func (m *Migrator) updateEdgeStackStatusForDB90() error {
+	log.Info().Msg("clean up deleted endpoints from edge jobs")
+
+	edgeJobs, err := m.edgeJobService.EdgeJobs()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeJob := range edgeJobs {
+		for endpointId := range edgeJob.Endpoints {
+			_, err := m.endpointService.Endpoint(endpointId)
+			if err == portainerDsErrors.ErrObjectNotFound {
+				delete(edgeJob.Endpoints, endpointId)
+
+				err = m.edgeJobService.UpdateEdgeJob(edgeJob.ID, &edgeJob)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateUserThemeForDB90() error {
+	log.Info().Msg("updating existing user theme settings")
+
+	users, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for i := range users {
+		user := &users[i]
+		if user.UserTheme != "" {
+			user.ThemeSettings.Color = user.UserTheme
+		}
+
+		if err := m.userService.UpdateUser(user.ID, user); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEnableGpuManagementFeatures() error {
+	// get all environments
+	environments, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, environment := range environments {
+		if environment.Type == portainer.DockerEnvironment {
+			// set the PostInitMigrations.MigrateGPUs to true on this environment to run the migration only on the 2.18 upgrade
+			environment.PostInitMigrations.MigrateGPUs = true
+			// if there's one or more gpu, set the EnableGpuManagement setting to true
+			gpuList := environment.Gpus
+			if len(gpuList) > 0 {
+				environment.EnableGPUManagement = true
+			}
+			// update the environment
+			if err := m.endpointService.UpdateEndpoint(environment.ID, &environment); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -3,6 +3,7 @@ package migrator
 import (
 	"errors"
 
+	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"github.com/portainer/portainer/api/dataservices/edgestack"
 
 	"github.com/Masterminds/semver"
@@ -56,6 +57,7 @@ type (
 		authorizationService    *authorization.Service
 		dockerhubService        *dockerhub.Service
 		edgeStackService        *edgestack.Service
+		edgeJobService          *edgejob.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -81,6 +83,7 @@ type (
 		AuthorizationService    *authorization.Service
 		DockerhubService        *dockerhub.Service
 		EdgeStackService        *edgestack.Service
+		EdgeJobService          *edgejob.Service
 	}
 )
 
@@ -108,6 +111,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		authorizationService:    parameters.AuthorizationService,
 		dockerhubService:        parameters.DockerhubService,
 		edgeStackService:        parameters.EdgeStackService,
+		edgeJobService:          parameters.EdgeJobService,
 	}
 
 	migrator.initMigrations()
@@ -205,6 +209,7 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.16", m.migrateDBVersionToDB70)
 	m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
 	m.addMigrations("2.17", m.migrateDBVersionToDB80)
+	m.addMigrations("2.18", m.migrateDBVersionToDB90)
 
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.

api/datastore/services.go

@@ -93,11 +93,18 @@ func (store *Store) initServices() error {
 	}
 	store.DockerHubService = dockerhubService
 
-	edgeStackService, err := edgestack.NewService(store.connection)
+	endpointRelationService, err := endpointrelation.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	edgeStackService, err := edgestack.NewService(store.connection, endpointRelationService.InvalidateEdgeCacheForEdgeStack)
 	if err != nil {
 		return err
 	}
 	store.EdgeStackService = edgeStackService
+	endpointRelationService.RegisterUpdateStackFunction(edgeStackService.UpdateEdgeStackFunc)
 
 	edgeGroupService, err := edgegroup.NewService(store.connection)
 	if err != nil {
@@ -123,12 +130,6 @@ func (store *Store) initServices() error {
 	}
 	store.EndpointService = endpointService
 
-	endpointRelationService, err := endpointrelation.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointRelationService = endpointRelationService
-
 	extensionService, err := extension.NewService(store.connection)
 	if err != nil {
 		return err

api/datastore/services_tx.go

@@ -0,0 +1,67 @@
+package datastore
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+type StoreTx struct {
+	store *Store
+	tx    portainer.Transaction
+}
+
+func (tx *StoreTx) IsErrObjectNotFound(err error) bool {
+	return tx.store.IsErrObjectNotFound(err)
+}
+
+func (tx *StoreTx) CustomTemplate() dataservices.CustomTemplateService { return nil }
+
+func (tx *StoreTx) EdgeGroup() dataservices.EdgeGroupService {
+	return tx.store.EdgeGroupService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) EdgeJob() dataservices.EdgeJobService {
+	return tx.store.EdgeJobService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) EdgeStack() dataservices.EdgeStackService {
+	return tx.store.EdgeStackService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) Endpoint() dataservices.EndpointService {
+	return tx.store.EndpointService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) EndpointGroup() dataservices.EndpointGroupService {
+	return tx.store.EndpointGroupService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) EndpointRelation() dataservices.EndpointRelationService {
+	return tx.store.EndpointRelationService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) FDOProfile() dataservices.FDOProfileService                 { return nil }
+func (tx *StoreTx) HelmUserRepository() dataservices.HelmUserRepositoryService { return nil }
+func (tx *StoreTx) Registry() dataservices.RegistryService                     { return nil }
+func (tx *StoreTx) ResourceControl() dataservices.ResourceControlService       { return nil }
+func (tx *StoreTx) Role() dataservices.RoleService                             { return nil }
+func (tx *StoreTx) APIKeyRepository() dataservices.APIKeyRepository            { return nil }
+func (tx *StoreTx) Settings() dataservices.SettingsService                     { return nil }
+
+func (tx *StoreTx) Snapshot() dataservices.SnapshotService {
+	return tx.store.SnapshotService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) SSLSettings() dataservices.SSLSettingsService { return nil }
+func (tx *StoreTx) Stack() dataservices.StackService             { return nil }
+
+func (tx *StoreTx) Tag() dataservices.TagService {
+	return tx.store.TagService.Tx(tx.tx)
+}
+
+func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService { return nil }
+func (tx *StoreTx) Team() dataservices.TeamService                     { return nil }
+func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService     { return nil }
+func (tx *StoreTx) User() dataservices.UserService                     { return nil }
+func (tx *StoreTx) Version() dataservices.VersionService               { return nil }
+func (tx *StoreTx) Webhook() dataservices.WebhookService               { return nil }

api/datastore/test_data/output_24_to_latest.json

@@ -46,6 +46,7 @@
       },
       "EdgeCheckinInterval": 0,
       "EdgeKey": "",
+      "EnableGPUManagement": false,
       "Gpus": [],
       "GroupId": 1,
       "Id": 1,
@@ -71,6 +72,7 @@
       "LastCheckInDate": 0,
       "Name": "local",
       "PostInitMigrations": {
+        "MigrateGPUs": true,
         "MigrateIngresses": true
       },
       "PublicURL": "",
@@ -902,6 +904,10 @@
         "PortainerUserRevokeToken": true
       },
       "Role": 1,
+      "ThemeSettings": {
+        "color": "",
+        "subtleUpgradeButton": false
+      },
       "TokenIssueAt": 0,
       "UserTheme": "",
       "Username": "admin"
@@ -929,12 +935,16 @@
         "PortainerUserRevokeToken": true
       },
       "Role": 1,
+      "ThemeSettings": {
+        "color": "",
+        "subtleUpgradeButton": false
+      },
       "TokenIssueAt": 0,
       "UserTheme": "",
       "Username": "prabhat"
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.18.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/docker/client.go

@@ -59,6 +59,19 @@ func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	)
 }
 
+func CreateClientFromEnv() (*client.Client, error) {
+	return client.NewClientWithOpts(
+		client.FromEnv,
+		client.WithVersion(dockerClientVersion),
+	)
+}
+
+func CreateSimpleClient() (*client.Client, error) {
+	return client.NewClientWithOpts(
+		client.WithVersion(dockerClientVersion),
+	)
+}
+
 func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*client.Client, error) {
 	httpCli, err := httpClient(endpoint, timeout)
 	if err != nil {

api/docker/snapshot.go

@@ -167,22 +167,29 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 			// snapshot GPUs
 			response, err := cli.ContainerInspect(context.Background(), container.ID)
 			if err != nil {
-				return err
-			}
-
-			var gpuOptions *_container.DeviceRequest = nil
-			for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
-				if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
-					gpuOptions = &deviceRequest
+				// Inspect a container will fail when the container runs on a different
+				// Swarm node, so it is better to log the error instead of return error
+				// when the Swarm mode is enabled
+				if !snapshot.Swarm {
+					return err
+				} else {
+					log.Info().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
 				}
-			}
-
-			if gpuOptions != nil {
-				if gpuOptions.Count == -1 {
-					gpuUseAll = true
+			} else {
+				var gpuOptions *_container.DeviceRequest = nil
+				for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
+					if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
+						gpuOptions = &deviceRequest
+					}
 				}
-				for _, id := range gpuOptions.DeviceIDs {
-					gpuUseSet[id] = struct{}{}
+
+				if gpuOptions != nil {
+					if gpuOptions.Count == -1 {
+						gpuUseAll = true
+					}
+					for _, id := range gpuOptions.DeviceIDs {
+						gpuUseSet[id] = struct{}{}
+					}
 				}
 			}
 		}
@@ -213,7 +220,9 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 	snapshot.HealthyContainerCount = healthyContainers
 	snapshot.UnhealthyContainerCount = unhealthyContainers
 	snapshot.StackCount += len(stacks)
-	snapshot.SnapshotRaw.Containers = containers
+	for _, container := range containers {
+		snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
+	}
 	return nil
 }
 

api/exec/compose_stack.go

@@ -66,7 +66,7 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 	return errors.Wrap(err, "failed to deploy a stack")
 }
 
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+// Down stops and removes containers, networks, images, and volumes
 func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := manager.fetchEndpointProxy(endpoint)
 	if err != nil {
@@ -81,14 +81,12 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack, false)
-
-	err = manager.deployer.Remove(ctx, filePaths, libstack.Options{
+	err = manager.deployer.Remove(ctx, stack.Name, nil, libstack.Options{
 		WorkingDir:  stack.ProjectPath,
 		EnvFilePath: envFilePath,
 		Host:        url,
-		ProjectName: stack.Name,
 	})
+
 	return errors.Wrap(err, "failed to remove a stack")
 }
 

api/git/azure.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,11 +11,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/portainer/portainer/api/archive"
+	"github.com/portainer/portainer/api/crypto"
+	gittypes "github.com/portainer/portainer/api/git/types"
+
 	"github.com/go-git/go-git/v5/plumbing/transport/client"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
-	gittypes "github.com/portainer/portainer/api/git/types"
 )
 
 const (
@@ -63,9 +64,11 @@ func NewAzureClient() *azureClient {
 }
 
 func newHttpClientForAzure() *http.Client {
+	tlsConfig := crypto.CreateTLSConfiguration()
+
 	httpsCli := &http.Client{
 		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			TLSClientConfig: tlsConfig,
 			Proxy:           http.ProxyFromEnvironment,
 		},
 		Timeout: 300 * time.Second,

api/git/backup.go

@@ -0,0 +1,62 @@
+package git
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	ErrInvalidGitCredential = errors.New("Invalid git credential")
+)
+
+type CloneOptions struct {
+	ProjectPath   string
+	URL           string
+	ReferenceName string
+	Username      string
+	Password      string
+}
+
+func CloneWithBackup(gitService portainer.GitService, fileService portainer.FileService, options CloneOptions) (clean func(), err error) {
+	backupProjectPath := fmt.Sprintf("%s-old", options.ProjectPath)
+	cleanUp := false
+	cleanFn := func() {
+		if !cleanUp {
+			return
+		}
+
+		err = fileService.RemoveDirectory(backupProjectPath)
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to remove git repository directory")
+		}
+	}
+
+	err = filesystem.MoveDirectory(options.ProjectPath, backupProjectPath)
+	if err != nil {
+		return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
+	}
+
+	cleanUp = true
+
+	err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password)
+	if err != nil {
+		cleanUp = false
+		restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath)
+		if restoreError != nil {
+			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
+		}
+
+		if err == gittypes.ErrAuthenticationFailure {
+			return cleanFn, errors.WithMessage(err, ErrInvalidGitCredential.Error())
+		}
+
+		return cleanFn, errors.WithMessage(err, "Unable to clone git repository")
+	}
+
+	return cleanFn, nil
+}

api/git/credentials.go

@@ -0,0 +1,13 @@
+package git
+
+import (
+	gittypes "github.com/portainer/portainer/api/git/types"
+)
+
+func GetCredentials(auth *gittypes.GitAuthentication) (string, string, error) {
+	if auth == nil {
+		return "", "", nil
+	}
+
+	return auth.Username, auth.Password, nil
+}

api/git/update/update.go

@@ -0,0 +1,94 @@
+package update
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/git"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/rs/zerolog/log"
+)
+
+// UpdateGitObject updates a git object based on its config
+func UpdateGitObject(gitService portainer.GitService, dataStore dataservices.DataStore, objId string, gitConfig *gittypes.RepoConfig, autoUpdateConfig *portainer.AutoUpdateSettings, projectPath string) (bool, string, error) {
+	if gitConfig == nil {
+		return false, "", nil
+	}
+
+	log.Debug().
+		Str("url", gitConfig.URL).
+		Str("ref", gitConfig.ReferenceName).
+		Str("object", objId).
+		Msg("the object has a git config, try to poll from git repository")
+
+	username, password, err := git.GetCredentials(gitConfig.Authentication)
+	if err != nil {
+		return false, "", errors.WithMessagef(err, "failed to get credentials for %v", objId)
+	}
+
+	newHash, err := gitService.LatestCommitID(gitConfig.URL, gitConfig.ReferenceName, username, password)
+	if err != nil {
+		return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId)
+	}
+
+	hashChanged := !strings.EqualFold(newHash, string(gitConfig.ConfigHash))
+	forceUpdate := autoUpdateConfig != nil && autoUpdateConfig.ForceUpdate
+	if !hashChanged && !forceUpdate {
+		log.Debug().
+			Str("hash", newHash).
+			Str("url", gitConfig.URL).
+			Str("ref", gitConfig.ReferenceName).
+			Str("object", objId).
+			Msg("git repo is up to date")
+
+		return false, newHash, nil
+	}
+
+	cloneParams := &cloneRepositoryParameters{
+		url:   gitConfig.URL,
+		ref:   gitConfig.ReferenceName,
+		toDir: projectPath,
+	}
+	if gitConfig.Authentication != nil {
+		cloneParams.auth = &gitAuth{
+			username: username,
+			password: password,
+		}
+	}
+
+	if err := cloneGitRepository(gitService, cloneParams); err != nil {
+		return false, "", errors.WithMessagef(err, "failed to do a fresh clone of %v", objId)
+	}
+
+	log.Debug().
+		Str("hash", newHash).
+		Str("url", gitConfig.URL).
+		Str("ref", gitConfig.ReferenceName).
+		Str("object", objId).
+		Msg("git repo cloned updated")
+
+	return true, newHash, nil
+}
+
+type cloneRepositoryParameters struct {
+	url   string
+	ref   string
+	toDir string
+	auth  *gitAuth
+}
+
+type gitAuth struct {
+	username string
+	password string
+}
+
+func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error {
+	if cloneParams.auth != nil {
+		return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password)
+	}
+
+	return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "")
+}

api/git/update/validate.go

@@ -0,0 +1,31 @@
+package update
+
+import (
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	portainer "github.com/portainer/portainer/api"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+)
+
+func ValidateAutoUpdateSettings(autoUpdate *portainer.AutoUpdateSettings) error {
+	if autoUpdate == nil {
+		return nil
+	}
+
+	if autoUpdate.Webhook == "" && autoUpdate.Interval == "" {
+		return httperrors.NewInvalidPayloadError("Webhook or Interval must be provided")
+	}
+
+	if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) {
+		return httperrors.NewInvalidPayloadError("invalid Webhook format")
+	}
+
+	if autoUpdate.Interval != "" {
+		if _, err := time.ParseDuration(autoUpdate.Interval); err != nil {
+			return httperrors.NewInvalidPayloadError("invalid Interval format")
+		}
+	}
+
+	return nil
+}

api/git/update/validate_test.go

@@ -1,4 +1,4 @@
-package stackutils
+package update
 
 import (
 	"testing"
@@ -7,25 +7,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_ValidateStackAutoUpdate(t *testing.T) {
+func Test_ValidateAutoUpdate(t *testing.T) {
 	tests := []struct {
 		name    string
-		value   *portainer.StackAutoUpdate
+		value   *portainer.AutoUpdateSettings
 		wantErr bool
 	}{
 		{
 			name:    "webhook is not a valid UUID",
-			value:   &portainer.StackAutoUpdate{Webhook: "fake-webhook"},
+			value:   &portainer.AutoUpdateSettings{Webhook: "fake-webhook"},
 			wantErr: true,
 		},
 		{
 			name:    "incorrect interval value",
-			value:   &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"},
+			value:   &portainer.AutoUpdateSettings{Interval: "1dd2hh3mm"},
 			wantErr: true,
 		},
 		{
 			name: "valid auto update",
-			value: &portainer.StackAutoUpdate{
+			value: &portainer.AutoUpdateSettings{
 				Webhook:  "8dce8c2f-9ca1-482b-ad20-271e86536ada",
 				Interval: "5h30m40s10ms",
 			},
@@ -35,7 +35,7 @@ func Test_ValidateStackAutoUpdate(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := ValidateStackAutoUpdate(tt.value)
+			err := ValidateAutoUpdateSettings(tt.value)
 			assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err)
 		})
 	}

api/git/validate.go

@@ -0,0 +1,25 @@
+package git
+
+import (
+	"github.com/asaskevich/govalidator"
+
+	gittypes "github.com/portainer/portainer/api/git/types"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+)
+
+func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
+	if govalidator.IsNull(repoConfig.URL) || !govalidator.IsURL(repoConfig.URL) {
+		return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
+	}
+
+	return ValidateRepoAuthentication(repoConfig.Authentication)
+
+}
+
+func ValidateRepoAuthentication(auth *gittypes.GitAuthentication) error {
+	if auth != nil && govalidator.IsNull(auth.Password) {
+		return httperrors.NewInvalidPayloadError("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+
+	return nil
+}

api/go.mod

@@ -1,6 +1,6 @@
 module github.com/portainer/portainer/api
 
-go 1.18
+go 1.19
 
 require (
 	github.com/Masterminds/semver v1.5.0
@@ -23,7 +23,7 @@ require (
 	github.com/go-playground/validator/v10 v10.10.1
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.2.0
-	github.com/google/go-cmp v0.5.8
+	github.com/google/go-cmp v0.5.9
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
@@ -36,30 +36,29 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27
+	github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7
 	github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a
-	github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9
+	github.com/portainer/libhttp v0.0.0-20230206214615-dabd58de9f44
+	github.com/portainer/portainer/pkg/featureflags v0.0.0-20230209201943-d73622ed9cd4
 	github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724
 	github.com/portainer/portainer/third_party/digest v0.0.0-20221201002639-8fd0efa34f73
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/rs/zerolog v1.28.0
+	github.com/rs/zerolog v1.29.0
 	github.com/stretchr/testify v1.8.1
 	github.com/viney-shih/go-lock v1.1.1
 	go.etcd.io/bbolt v1.3.6
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.25.3
-	k8s.io/apimachinery v0.25.3
-	k8s.io/client-go v0.25.3
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/client-go v0.26.1
 	software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78
 )
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
 	github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 // indirect
@@ -72,7 +71,7 @@ require (
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
@@ -82,7 +81,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.1.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
@@ -102,8 +101,8 @@ require (
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
@@ -124,21 +123,21 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/net v0.2.0 // indirect
-	golang.org/x/sys v0.2.0 // indirect
-	golang.org/x/term v0.2.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.3.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
-	k8s.io/klog/v2 v2.70.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

api/go.sum

@@ -41,10 +41,6 @@ github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/VictoriaMetrics/fastcache v1.12.0 h1:vnVi/y9yKDcD9akmc4NqAoqgQhJrOwUF+j9LTgn4QDE=
 github.com/VictoriaMetrics/fastcache v1.12.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
@@ -111,8 +107,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
-github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
-github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
+github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -151,15 +147,14 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
 github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
-github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
+github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
@@ -221,8 +216,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -306,10 +301,13 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
@@ -328,8 +326,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
-github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
+github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
+github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
@@ -344,14 +342,20 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20221122145319-915b021aea84 h1:d1P8i0pCPvAfxH6nSLUFm6NYoi8tMrIpafaZXSV8Lac=
-github.com/portainer/docker-compose-wrapper v0.0.0-20221122145319-915b021aea84/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
 github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27 h1:PceCpp86SDYb3lZHT4KpuBCkmcJMW5x1qrdFNEfAdUo=
 github.com/portainer/docker-compose-wrapper v0.0.0-20221215210951-2c30d1b17a27/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230209071700-ee11af9c546a h1:4VGM1OH15fqm5rgki0eLF6vND/NxHfoPt3CA6/YdA0k=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230209071700-ee11af9c546a/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230209082344-8a5b52de366f h1:z/lmLhZMMSIwg70Ap1rPluXNe1vQXH9gfK9K/ols4JA=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230209082344-8a5b52de366f/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7 h1:/i985KPNw0KvVtLhTEPUa86aJMtun5ZPOyFCJzdY+dY=
+github.com/portainer/docker-compose-wrapper v0.0.0-20230301083819-3dbc6abf1ce7/go.mod h1:03UmPLyjiPUexGJuW20mQXvmsoSpeErvMlItJGtq/Ww=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a h1:B0z3skIMT+OwVNJPQhKp52X+9OWW6A9n5UWig3lHBJk=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
-github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9 h1:L7o0L+1qq+LzKjzgRB6bDIh5ZrZ5A1oSS+WgWzDgJIo=
-github.com/portainer/libhttp v0.0.0-20221121135534-76f46e09c9a9/go.mod h1:H49JLiywwLt2rrJVroafEWy8fIs0i7mThAThK40sbb8=
+github.com/portainer/libhttp v0.0.0-20230206214615-dabd58de9f44 h1:4LYprPd3TsYjHk7CaTmCov1ceG6VKJsL40fJIWiRxpw=
+github.com/portainer/libhttp v0.0.0-20230206214615-dabd58de9f44/go.mod h1:H49JLiywwLt2rrJVroafEWy8fIs0i7mThAThK40sbb8=
+github.com/portainer/portainer/pkg/featureflags v0.0.0-20230209201943-d73622ed9cd4 h1:gnXwaF0GnFUIlynRq994WFOtqOULTKZks4aSWuonlhA=
+github.com/portainer/portainer/pkg/featureflags v0.0.0-20230209201943-d73622ed9cd4/go.mod h1:T37rFZMg+PhRhT9n/z9cLSj9khJSdwHj3/Ac5PZQgKI=
 github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724 h1:FZrRVMpxXdUV+p5VSCAy9Uz7RzAeEJr2ytlctvMrsHY=
 github.com/portainer/portainer/pkg/libhelm v0.0.0-20221201012749-4fee35924724/go.mod h1:WUdwNVH9GMffP4qf4U2ea2qCYfti2V7S+IhGpO8Sxv0=
 github.com/portainer/portainer/third_party/digest v0.0.0-20221201002639-8fd0efa34f73 h1:7bPOnwucE0nor0so1BQJxQKCL5t+vCWO4nAz/S0lci0=
@@ -364,15 +368,14 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
-github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -471,7 +474,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -490,15 +492,16 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
-golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -551,13 +554,16 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
-golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -565,8 +571,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -697,8 +703,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -733,19 +739,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
-k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
-k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
-k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
-k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=
-k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
-k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
+k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
+k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
+k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
+k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
+k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
+k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
+k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
@@ -753,7 +758,7 @@ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 h1:SqYE5+A2qvRhErbsXFfUEUmpWEKxxRSMgGLkvRAFOV4=
 software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78/go.mod h1:B7Wf0Ya4DHF9Yw+qfZuJijQYkWicqDa+79Ytmmq3Kjg=

api/hostmanagement/openamt/openamt.go

@@ -2,7 +2,6 @@ package openamt
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -11,6 +10,8 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+
 	"golang.org/x/sync/errgroup"
 )
 
@@ -32,11 +33,14 @@ type Service struct {
 
 // NewService initializes a new service.
 func NewService() *Service {
+	tlsConfig := crypto.CreateTLSConfiguration()
+	tlsConfig.InsecureSkipVerify = true
+
 	return &Service{
 		httpsClient: &http.Client{
 			Timeout: httpClientTimeout,
 			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				TLSClientConfig: tlsConfig,
 			},
 		},
 	}

api/http/errors/invalidpayload.go

@@ -0,0 +1,13 @@
+package errors
+
+type InvalidPayloadError struct {
+	msg string
+}
+
+func (e *InvalidPayloadError) Error() string {
+	return e.msg
+}
+
+func NewInvalidPayloadError(msg string) *InvalidPayloadError {
+	return &InvalidPayloadError{msg: msg}
+}

api/http/handler/backup/handler.go

@@ -61,10 +61,12 @@ func adminAccess(next http.Handler) http.Handler {
 		securityContext, err := security.RetrieveRestrictedRequestContext(r)
 		if err != nil {
 			httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user info from request context", err)
+			return
 		}
 
 		if !securityContext.IsAdmin {
 			httperror.WriteError(w, http.StatusUnauthorized, "User is not authorized to perform the action", nil)
+			return
 		}
 
 		next.ServeHTTP(w, r)

api/http/handler/customtemplates/customtemplate_delete.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/rs/zerolog/log"
 )
 
 // @id CustomTemplateDelete
@@ -61,7 +62,7 @@ func (handler *Handler) customTemplateDelete(w http.ResponseWriter, r *http.Requ
 
 	err = handler.FileService.RemoveDirectory(customTemplate.ProjectPath)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove custom template files from disk", err)
+		log.Warn().Err(err).Msg("Unable to remove custom template files from disk")
 	}
 
 	if resourceControl != nil {

api/http/handler/edgegroups/edgegroup_create.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 
 	"github.com/asaskevich/govalidator"
@@ -54,45 +55,58 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 		return httperror.BadRequest("Invalid request payload", err)
 	}
 
-	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve Edge groups from the database", err)
-	}
+	var edgeGroup *portainer.EdgeGroup
 
-	for _, edgeGroup := range edgeGroups {
-		if edgeGroup.Name == payload.Name {
-			return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
+	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		edgeGroups, err := tx.EdgeGroup().EdgeGroups()
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve Edge groups from the database", err)
 		}
-	}
 
-	edgeGroup := &portainer.EdgeGroup{
-		Name:         payload.Name,
-		Dynamic:      payload.Dynamic,
-		TagIDs:       []portainer.TagID{},
-		Endpoints:    []portainer.EndpointID{},
-		PartialMatch: payload.PartialMatch,
-	}
-
-	if edgeGroup.Dynamic {
-		edgeGroup.TagIDs = payload.TagIDs
-	} else {
-		endpointIDs := []portainer.EndpointID{}
-		for _, endpointID := range payload.Endpoints {
-			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
-			if err != nil {
-				return httperror.InternalServerError("Unable to retrieve environment from the database", err)
-			}
-
-			if endpointutils.IsEdgeEndpoint(endpoint) {
-				endpointIDs = append(endpointIDs, endpoint.ID)
+		for _, edgeGroup := range edgeGroups {
+			if edgeGroup.Name == payload.Name {
+				return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
 			}
 		}
-		edgeGroup.Endpoints = endpointIDs
-	}
 
-	err = handler.DataStore.EdgeGroup().Create(edgeGroup)
+		edgeGroup = &portainer.EdgeGroup{
+			Name:         payload.Name,
+			Dynamic:      payload.Dynamic,
+			TagIDs:       []portainer.TagID{},
+			Endpoints:    []portainer.EndpointID{},
+			PartialMatch: payload.PartialMatch,
+		}
+
+		if edgeGroup.Dynamic {
+			edgeGroup.TagIDs = payload.TagIDs
+		} else {
+			endpointIDs := []portainer.EndpointID{}
+			for _, endpointID := range payload.Endpoints {
+				endpoint, err := tx.Endpoint().Endpoint(endpointID)
+				if err != nil {
+					return httperror.InternalServerError("Unable to retrieve environment from the database", err)
+				}
+
+				if endpointutils.IsEdgeEndpoint(endpoint) {
+					endpointIDs = append(endpointIDs, endpoint.ID)
+				}
+			}
+			edgeGroup.Endpoints = endpointIDs
+		}
+
+		err = tx.EdgeGroup().Create(edgeGroup)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist the Edge group inside the database", err)
+		}
+
+		return nil
+	})
 	if err != nil {
-		return httperror.InternalServerError("Unable to persist the Edge group inside the database", err)
+		if httpErr, ok := err.(*httperror.HandlerError); ok {
+			return httpErr
+		}
+
+		return httperror.InternalServerError("Unexpected error", err)
 	}
 
 	return response.JSON(w, edgeGroup)

api/http/handler/edgegroups/edgegroup_update.go

@@ -153,7 +153,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 
-		err = handler.updateEndpointEdgeJobs(edgeGroup.ID, endpointID, edgeJobs, operation)
+		err = handler.updateEndpointEdgeJobs(edgeGroup.ID, endpoint, edgeJobs, operation)
 		if err != nil {
 			return httperror.InternalServerError("Unable to persist Environment Edge Jobs changes inside the database", err)
 		}
@@ -200,7 +200,7 @@ func (handler *Handler) updateEndpointStacks(endpointID portainer.EndpointID) er
 	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 }
 
-func (handler *Handler) updateEndpointEdgeJobs(edgeGroupID portainer.EdgeGroupID, endpointID portainer.EndpointID, edgeJobs []portainer.EdgeJob, operation string) error {
+func (handler *Handler) updateEndpointEdgeJobs(edgeGroupID portainer.EdgeGroupID, endpoint *portainer.Endpoint, edgeJobs []portainer.EdgeJob, operation string) error {
 	for _, edgeJob := range edgeJobs {
 		if !slices.Contains(edgeJob.EdgeGroups, edgeGroupID) {
 			continue
@@ -208,9 +208,9 @@ func (handler *Handler) updateEndpointEdgeJobs(edgeGroupID portainer.EdgeGroupID
 
 		switch operation {
 		case "add":
-			handler.ReverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
+			handler.ReverseTunnelService.AddEdgeJob(endpoint, &edgeJob)
 		case "remove":
-			handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
+			handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpoint.ID, edgeJob.ID)
 		}
 	}
 

api/http/handler/edgejobs/edgejob_create.go

@@ -274,7 +274,12 @@ func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []
 	}
 
 	for endpointID := range endpointsMap {
-		handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
+		endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return err
+		}
+
+		handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
 	}
 
 	return handler.DataStore.EdgeJob().Create(edgeJob.ID, edgeJob)

api/http/handler/edgejobs/edgejob_delete.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/maps"
+	"github.com/rs/zerolog/log"
 )
 
 // @id EdgeJobDelete
@@ -40,7 +41,7 @@ func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *h
 	edgeJobFolder := handler.FileService.GetEdgeJobFolder(strconv.Itoa(edgeJobID))
 	err = handler.FileService.RemoveDirectory(edgeJobFolder)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove the files associated to the Edge job on the filesystem", err)
+		log.Warn().Err(err).Msg("Unable to remove the files associated to the Edge job on the filesystem")
 	}
 
 	handler.ReverseTunnelService.RemoveEdgeJob(edgeJob.ID)

api/http/handler/edgejobs/edgejob_tasklogs_clear.go

@@ -44,22 +44,33 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
 		return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
 	}
 
+	err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
+	if err != nil {
+		return httperror.InternalServerError("Unable to clear log file from disk", err)
+	}
+
 	endpointID := portainer.EndpointID(taskID)
 	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
 	if err != nil {
 		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
 	}
 
-	if slices.Contains(endpointsFromGroups, endpointID) {
-		edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
-			CollectLogs: false,
-			LogsStatus:  portainer.EdgeJobLogsStatusIdle,
+	err = handler.DataStore.EdgeJob().UpdateEdgeJobFunc(edgeJob.ID, func(j *portainer.EdgeJob) {
+		if slices.Contains(endpointsFromGroups, endpointID) {
+			j.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+				CollectLogs: false,
+				LogsStatus:  portainer.EdgeJobLogsStatusIdle,
+			}
+		} else {
+			meta := j.Endpoints[endpointID]
+			meta.CollectLogs = false
+			meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
+			j.Endpoints[endpointID] = meta
 		}
-	} else {
-		meta := edgeJob.Endpoints[endpointID]
-		meta.CollectLogs = false
-		meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
-		edgeJob.Endpoints[endpointID] = meta
+	})
+
+	if err != nil {
+		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
 	}
 
 	err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
@@ -67,12 +78,12 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
 		return httperror.InternalServerError("Unable to clear log file from disk", err)
 	}
 
-	handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
-
-	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 	if err != nil {
-		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
+		return httperror.NotFound("Unable to retrieve environment from the database", err)
 	}
 
+	handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
+
 	return response.Empty(w)
 }

api/http/handler/edgejobs/edgejob_tasklogs_collect.go

@@ -7,6 +7,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/slices"
 )
@@ -36,46 +37,58 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
 		return httperror.BadRequest("Invalid Task identifier route variable", err)
 	}
 
-	edgeJob, err := handler.DataStore.EdgeJob().EdgeJob(portainer.EdgeJobID(edgeJobID))
-	if handler.DataStore.IsErrObjectNotFound(err) {
-		return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
-	} else if err != nil {
-		return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
-	}
-
-	endpointID := portainer.EndpointID(taskID)
-	endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, handler.DataStore)
-	if err != nil {
-		return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
-	}
-
-	if slices.Contains(endpointsFromGroups, endpointID) {
-		edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
-			CollectLogs: true,
-			LogsStatus:  portainer.EdgeJobLogsStatusPending,
+	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		edgeJob, err := tx.EdgeJob().EdgeJob(portainer.EdgeJobID(edgeJobID))
+		if handler.DataStore.IsErrObjectNotFound(err) {
+			return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
+		} else if err != nil {
+			return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
 		}
-	} else {
-		meta := edgeJob.Endpoints[endpointID]
-		meta.CollectLogs = true
-		meta.LogsStatus = portainer.EdgeJobLogsStatusPending
-		edgeJob.Endpoints[endpointID] = meta
-	}
 
-	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+		endpointID := portainer.EndpointID(taskID)
+		endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
+		if err != nil {
+			return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
+		}
+
+		if slices.Contains(endpointsFromGroups, endpointID) {
+			edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
+				CollectLogs: true,
+				LogsStatus:  portainer.EdgeJobLogsStatusPending,
+			}
+		} else {
+			meta := edgeJob.Endpoints[endpointID]
+			meta.CollectLogs = true
+			meta.LogsStatus = portainer.EdgeJobLogsStatusPending
+			edgeJob.Endpoints[endpointID] = meta
+		}
+
+		err = tx.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
+		}
+
+		endpoint, err := tx.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return httperror.InternalServerError("Unable to retrieve environment from the database", err)
+		}
+
+		if endpoint.Edge.AsyncMode {
+			return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
+		}
+
+		handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
+
+		return nil
+	})
+
 	if err != nil {
-		return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
-	}
+		if httpErr, ok := err.(*httperror.HandlerError); ok {
+			return httpErr
+		}
 
-	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve environment from the database", err)
+		return httperror.InternalServerError("Unexpected error", err)
 	}
 
-	if endpoint.Edge.AsyncMode {
-		return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
-	}
-
-	handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
-
 	return response.Empty(w)
 }

api/http/handler/edgejobs/edgejob_update.go

@@ -212,7 +212,12 @@ func (handler *Handler) updateEdgeSchedule(edgeJob *portainer.EdgeJob, payload *
 	maps.Copy(endpointsFromGroupsToAddMap, edgeJob.Endpoints)
 
 	for endpointID := range endpointsFromGroupsToAddMap {
-		handler.ReverseTunnelService.AddEdgeJob(endpointID, edgeJob)
+		endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
+		if err != nil {
+			return err
+		}
+
+		handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
 	}
 
 	for endpointID := range endpointsToRemove {

api/http/handler/edgestacks/edgestack_update.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/rs/zerolog/log"
 )
 
 type updateEdgeStackPayload struct {
@@ -135,7 +136,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		// deployment type was changed - need to delete the old file
 		err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 		if err != nil {
-			return httperror.InternalServerError("Unable to clear old files", err)
+			log.Warn().Err(err).Msg("Unable to clear old files")
 		}
 
 		stack.EntryPoint = ""

api/http/handler/endpointedge/endpoint_edgejob_logs.go

@@ -74,7 +74,7 @@ func (handler *Handler) endpointEdgeJobsLogs(w http.ResponseWriter, r *http.Requ
 
 	err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
 
-	handler.ReverseTunnelService.AddEdgeJob(endpoint.ID, edgeJob)
+	handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
 
 	if err != nil {
 		return httperror.InternalServerError("Unable to persist edge job changes to the database", err)

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -416,7 +416,7 @@ func TestEdgeJobsResponse(t *testing.T) {
 		Version:        57,
 	}
 
-	handler.ReverseTunnelService.AddEdgeJob(endpoint.ID, &edgeJob)
+	handler.ReverseTunnelService.AddEdgeJob(&endpoint, &edgeJob)
 
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpoint.ID), nil)
 	if err != nil {

api/http/handler/endpoints/endpoint_create.go

@@ -164,9 +164,6 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 	checkinInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "CheckinInterval", true)
 	payload.EdgeCheckinInterval = checkinInterval
 
-	isEdgeDevice, _ := request.RetrieveBooleanMultiPartFormValue(r, "IsEdgeDevice", true)
-	payload.IsEdgeDevice = isEdgeDevice
-
 	return nil
 }
 
@@ -196,7 +193,6 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 // @param TagIDs formData []int false "List of tag identifiers to which this environment(endpoint) is associated"
 // @param EdgeCheckinInterval formData int false "The check in interval for edge agent (in seconds)"
 // @param EdgeTunnelServerAddress formData string true "URL or IP address that will be used to establish a reverse tunnel"
-// @param IsEdgeDevice formData bool false "Is Edge Device"
 // @param Gpus formData array false "List of GPUs"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"

api/http/handler/endpoints/endpoint_delete.go

@@ -137,8 +137,10 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 	for idx := range edgeJobs {
 		edgeJob := &edgeJobs[idx]
 		if _, ok := edgeJob.Endpoints[endpoint.ID]; ok {
-			delete(edgeJob.Endpoints, endpoint.ID)
-			err = handler.DataStore.EdgeJob().UpdateEdgeJob(edgeJob.ID, edgeJob)
+			err = handler.DataStore.EdgeJob().UpdateEdgeJobFunc(edgeJob.ID, func(j *portainer.EdgeJob) {
+				delete(j.Endpoints, endpoint.ID)
+			})
+
 			if err != nil {
 				return httperror.InternalServerError("Unable to update edge job", err)
 			}

api/http/handler/endpoints/endpoint_list.go

@@ -42,8 +42,8 @@ const (
 // @param endpointIds query []int false "will return only these environments(endpoints)"
 // @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned"
 // @param agentVersions query []string false "will return only environments with on of these agent versions"
-// @param edgeDevice query bool false "if exists true show only edge devices, false show only regular edge endpoints. if missing, will show both types (relevant only for edge endpoints)"
-// @param edgeDeviceUntrusted query bool false "if true, show only untrusted endpoints, if false show only trusted (relevant only for edge devices, and if edgeDevice is true)"
+// @param edgeAsync query bool false "if exists true show only edge async agents, false show only standard edge agents. if missing, will show both types (relevant only for edge agents)"
+// @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
 // @param name query string false "will return only environments(endpoints) with this name"
 // @success 200 {array} portainer.Endpoint "Endpoints"
 // @failure 500 "Server error"

api/http/handler/endpoints/endpoint_list_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/testhelpers"
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -104,59 +103,58 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
 	}
 }
 
-func Test_endpointList_edgeDeviceFilter(t *testing.T) {
+func Test_endpointList_edgeFilter(t *testing.T) {
 
-	trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularEndpoint := portainer.Endpoint{ID: 5, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.DockerEnvironment}
+	trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
 
 	handler, teardown := setup(t, []portainer.Endpoint{
-		trustedEdgeDevice,
-		untrustedEdgeDevice,
-		regularUntrustedEdgeEndpoint,
-		regularTrustedEdgeEndpoint,
+		trustedEdgeAsync,
+		untrustedEdgeAsync,
+		regularUntrustedEdgeStandard,
+		regularTrustedEdgeStandard,
 		regularEndpoint,
 	})
 
 	defer teardown()
 
-	type endpointListEdgeDeviceTest struct {
+	type endpointListEdgeTest struct {
 		endpointListTest
-		edgeDevice          *bool
+		edgeAsync           *bool
 		edgeDeviceUntrusted bool
 	}
 
-	tests := []endpointListEdgeDeviceTest{
+	tests := []endpointListEdgeTest{
 		{
 			endpointListTest: endpointListTest{
-				"should show all endpoints except of the untrusted devices",
-				[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID, regularEndpoint.ID},
+				"should show all endpoints expect of the untrusted devices",
+				[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID, regularEndpoint.ID},
 			},
-			edgeDevice: nil,
 		},
 		{
 			endpointListTest: endpointListTest{
-				"should show only trusted edge devices and regular endpoints",
-				[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
+				"should show only trusted edge async agents and regular endpoints",
+				[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
 			},
-			edgeDevice: BoolAddr(true),
+			edgeAsync: BoolAddr(true),
 		},
 		{
 			endpointListTest: endpointListTest{
 				"should show only untrusted edge devices and regular endpoints",
-				[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
+				[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
 			},
-			edgeDevice:          BoolAddr(true),
+			edgeAsync:           BoolAddr(true),
 			edgeDeviceUntrusted: true,
 		},
 		{
 			endpointListTest: endpointListTest{
 				"should show no edge devices",
-				[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+				[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
 			},
-			edgeDevice: BoolAddr(false),
+			edgeAsync: BoolAddr(false),
 		},
 	}
 
@@ -165,8 +163,8 @@ func Test_endpointList_edgeDeviceFilter(t *testing.T) {
 			is := assert.New(t)
 
 			query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted)
-			if test.edgeDevice != nil {
-				query += fmt.Sprintf("edgeDevice=%v&", *test.edgeDevice)
+			if test.edgeAsync != nil {
+				query += fmt.Sprintf("edgeAsync=%v&", *test.edgeAsync)
 			}
 
 			req := buildEndpointListRequest(query)
@@ -198,7 +196,7 @@ func setup(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, tear
 	err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
-	bouncer := helper.NewTestRequestBouncer()
+	bouncer := testhelpers.NewTestRequestBouncer()
 	handler = NewHandler(bouncer, nil)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()

api/http/handler/endpoints/endpoint_settings_update.go

@@ -28,6 +28,10 @@ type endpointSettingsUpdatePayload struct {
 	AllowSysctlSettingForRegularUsers *bool `json:"allowSysctlSettingForRegularUsers" example:"true"`
 	// Whether host management features are enabled
 	EnableHostManagementFeatures *bool `json:"enableHostManagementFeatures" example:"true"`
+
+	EnableGPUManagement *bool `json:"enableGPUManagement" example:"false"`
+
+	Gpus []portainer.Pair `json:"gpus"`
 }
 
 func (payload *endpointSettingsUpdatePayload) Validate(r *http.Request) error {
@@ -107,6 +111,14 @@ func (handler *Handler) endpointSettingsUpdate(w http.ResponseWriter, r *http.Re
 		securitySettings.EnableHostManagementFeatures = *payload.EnableHostManagementFeatures
 	}
 
+	if payload.EnableGPUManagement != nil {
+		endpoint.EnableGPUManagement = *payload.EnableGPUManagement
+	}
+
+	if payload.Gpus != nil {
+		endpoint.Gpus = payload.Gpus
+	}
+
 	endpoint.SecuritySettings = securitySettings
 
 	err = handler.DataStore.Endpoint().UpdateEndpoint(portainer.EndpointID(endpointID), endpoint)

api/http/handler/endpoints/filter.go

@@ -15,14 +15,15 @@ import (
 )
 
 type EnvironmentsQuery struct {
-	search              string
-	types               []portainer.EndpointType
-	tagIds              []portainer.TagID
-	endpointIds         []portainer.EndpointID
-	tagsPartialMatch    bool
-	groupIds            []portainer.EndpointGroupID
-	status              []portainer.EndpointStatus
-	edgeDevice          *bool
+	search           string
+	types            []portainer.EndpointType
+	tagIds           []portainer.TagID
+	endpointIds      []portainer.EndpointID
+	tagsPartialMatch bool
+	groupIds         []portainer.EndpointGroupID
+	status           []portainer.EndpointStatus
+	// if edgeAsync not nil, will filter edge endpoints based on this value
+	edgeAsync           *bool
 	edgeDeviceUntrusted bool
 	excludeSnapshots    bool
 	name                string
@@ -66,11 +67,10 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 
 	name, _ := request.RetrieveQueryParameter(r, "name", true)
 
-	edgeDeviceParam, _ := request.RetrieveQueryParameter(r, "edgeDevice", true)
-
-	var edgeDevice *bool
-	if edgeDeviceParam != "" {
-		edgeDevice = BoolAddr(edgeDeviceParam == "true")
+	var edgeAsync *bool
+	edgeAsyncParam, _ := request.RetrieveQueryParameter(r, "edgeAsync", true)
+	if edgeAsyncParam != "" {
+		edgeAsync = BoolAddr(edgeAsyncParam == "true")
 	}
 
 	edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true)
@@ -85,7 +85,7 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 		tagsPartialMatch:    tagsPartialMatch,
 		groupIds:            groupIDs,
 		status:              status,
-		edgeDevice:          edgeDevice,
+		edgeAsync:           edgeAsync,
 		edgeDeviceUntrusted: edgeDeviceUntrusted,
 		excludeSnapshots:    excludeSnapshots,
 		name:                name,
@@ -108,15 +108,26 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End
 		filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
 	}
 
-	if query.edgeDevice != nil {
-		filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, *query.edgeDevice, query.edgeDeviceUntrusted)
-	} else {
-		// If the edgeDevice parameter is not set, we need to filter out the untrusted edge devices
+	// filter async edge environments
+	if query.edgeAsync != nil {
 		filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
-			return !endpoint.IsEdgeDevice || endpoint.UserTrusted
+			if !endpointutils.IsEdgeEndpoint(&endpoint) {
+				return true
+			}
+
+			return endpoint.Edge.AsyncMode == *query.edgeAsync
 		})
 	}
 
+	// filter edge environments by trusted/untrusted
+	filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
+		if !endpointutils.IsEdgeEndpoint(&endpoint) {
+			return true
+		}
+
+		return endpoint.UserTrusted == !query.edgeDeviceUntrusted
+	})
+
 	if len(query.status) > 0 {
 		filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings)
 	}
@@ -274,30 +285,6 @@ func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []port
 	return endpoints[:n]
 }
 
-func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDevice bool, untrusted bool) []portainer.Endpoint {
-	n := 0
-	for _, endpoint := range endpoints {
-		if shouldReturnEdgeDevice(endpoint, edgeDevice, untrusted) {
-			endpoints[n] = endpoint
-			n++
-		}
-	}
-
-	return endpoints[:n]
-}
-
-func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceParam bool, untrustedParam bool) bool {
-	if !endpointutils.IsEdgeEndpoint(&endpoint) {
-		return true
-	}
-
-	if !edgeDeviceParam {
-		return !endpoint.IsEdgeDevice
-	}
-
-	return endpoint.IsEdgeDevice && endpoint.UserTrusted == !untrustedParam
-}
-
 func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
 	tags := make([]string, 0, len(tagIDs))
 

api/http/handler/endpoints/filter_test.go

@@ -6,7 +6,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/internal/testhelpers"
-	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -74,19 +73,19 @@ func Test_Filter_AgentVersion(t *testing.T) {
 	runTests(tests, t, handler, endpoints)
 }
 
-func Test_Filter_edgeDeviceFilter(t *testing.T) {
+func Test_Filter_edgeFilter(t *testing.T) {
 
-	trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
-	regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
+	regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
 	regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
 
 	endpoints := []portainer.Endpoint{
-		trustedEdgeDevice,
-		untrustedEdgeDevice,
-		regularUntrustedEdgeEndpoint,
-		regularTrustedEdgeEndpoint,
+		trustedEdgeAsync,
+		untrustedEdgeAsync,
+		regularUntrustedEdgeStandard,
+		regularTrustedEdgeStandard,
 		regularEndpoint,
 	}
 
@@ -96,32 +95,32 @@ func Test_Filter_edgeDeviceFilter(t *testing.T) {
 
 	tests := []filterTest{
 		{
-			"should show all edge endpoints except of the untrusted devices",
-			[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+			"should show all edge endpoints except of the untrusted edge",
+			[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID},
 			EnvironmentsQuery{
 				types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment},
 			},
 		},
 		{
 			"should show only trusted edge devices and other regular endpoints",
-			[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
+			[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
 			EnvironmentsQuery{
-				edgeDevice: BoolAddr(true),
+				edgeAsync: BoolAddr(true),
 			},
 		},
 		{
 			"should show only untrusted edge devices and other regular endpoints",
-			[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
+			[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
 			EnvironmentsQuery{
-				edgeDevice:          BoolAddr(true),
+				edgeAsync:           BoolAddr(true),
 				edgeDeviceUntrusted: true,
 			},
 		},
 		{
 			"should show no edge devices",
-			[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
+			[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
 			EnvironmentsQuery{
-				edgeDevice: BoolAddr(false),
+				edgeAsync: BoolAddr(false),
 			},
 		},
 	}
@@ -168,7 +167,7 @@ func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) (handler *Han
 	err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
-	bouncer := helper.NewTestRequestBouncer()
+	bouncer := testhelpers.NewTestRequestBouncer()
 	handler = NewHandler(bouncer, nil)
 	handler.DataStore = store
 	handler.ComposeStackManager = testhelpers.NewComposeStackManager()

api/http/handler/handler.go

@@ -82,7 +82,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.17.0
+// @version 2.18.0
 // @description.markdown api-description.md
 // @termsOfService
 

api/http/handler/kubernetes/services.go

@@ -37,7 +37,15 @@ func (handler *Handler) getKubernetesServices(w http.ResponseWriter, r *http.Req
 		)
 	}
 
-	services, err := cli.GetServices(namespace)
+	lookup, err := request.RetrieveBooleanQueryParameter(r, "lookupapplications", true)
+	if err != nil {
+		return httperror.BadRequest(
+			"Invalid lookupapplications query parameter",
+			err,
+		)
+	}
+
+	services, err := cli.GetServices(namespace, lookup)
 	if err != nil {
 		return httperror.InternalServerError(
 			"Unable to retrieve services",

api/http/handler/settings/settings_public.go

@@ -7,6 +7,7 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/pkg/featureflags"
 )
 
 type publicSettingsResponse struct {
@@ -21,7 +22,7 @@ type publicSettingsResponse struct {
 	// Whether edge compute features are enabled
 	EnableEdgeComputeFeatures bool `json:"EnableEdgeComputeFeatures" example:"true"`
 	// Supported feature flags
-	Features map[portainer.Feature]bool `json:"Features"`
+	Features map[featureflags.Feature]bool `json:"Features"`
 	// The URL used for oauth login
 	OAuthLoginURI string `json:"OAuthLoginURI" example:"https://gitlab.com/oauth"`
 	// The URL used for oauth logout
@@ -39,8 +40,6 @@ type publicSettingsResponse struct {
 	IsAMTEnabled bool
 
 	Edge struct {
-		// Whether the device has been started in edge async mode
-		AsyncMode bool
 		// The ping interval for edge agent - used in edge async mode [seconds]
 		PingInterval int `json:"PingInterval" example:"60"`
 		// The snapshot interval for edge agent - used in edge async mode [seconds]
@@ -80,12 +79,11 @@ func generatePublicSettings(appSettings *portainer.Settings) *publicSettingsResp
 		ShowKomposeBuildOption:    appSettings.ShowKomposeBuildOption,
 		EnableTelemetry:           appSettings.EnableTelemetry,
 		KubeconfigExpiry:          appSettings.KubeconfigExpiry,
-		Features:                  appSettings.FeatureFlagSettings,
+		Features:                  featureflags.FeatureFlags(),
 		IsFDOEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.FDOConfiguration.Enabled,
 		IsAMTEnabled:              appSettings.EnableEdgeComputeFeatures && appSettings.OpenAMTConfiguration.Enabled,
 	}
 
-	publicSettings.Edge.AsyncMode = appSettings.Edge.AsyncMode
 	publicSettings.Edge.PingInterval = appSettings.Edge.PingInterval
 	publicSettings.Edge.SnapshotInterval = appSettings.Edge.SnapshotInterval
 	publicSettings.Edge.CommandInterval = appSettings.Edge.CommandInterval

api/http/handler/stacks/create_compose_stack.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
@@ -156,14 +157,14 @@ type composeStackFromGitRepositoryPayload struct {
 	// Applicable when deploying with multiple stack files
 	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
 	// Optional auto update configuration
-	AutoUpdate *portainer.StackAutoUpdate
+	AutoUpdate *portainer.AutoUpdateSettings
 	// A list of environment(endpoint) variables used during stack deployment
 	Env []portainer.Pair
 	// Whether the stack is from a app template
 	FromAppTemplate bool `example:"false"`
 }
 
-func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
+func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		Name: name,
 		RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{
@@ -191,7 +192,7 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil

api/http/handler/stacks/create_kubernetes_stack.go

@@ -11,6 +11,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/git/update"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
@@ -44,10 +45,10 @@ type kubernetesGitDeploymentPayload struct {
 	RepositoryPassword       string
 	ManifestFile             string
 	AdditionalFiles          []string
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
 }
 
-func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate) stackbuilders.StackPayload {
+func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		StackName: name,
 		RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{
@@ -101,7 +102,7 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.ManifestFile) {
 		return errors.New("Invalid manifest file in repository")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	if govalidator.IsNull(payload.StackName) {

api/http/handler/stacks/create_swarm_stack.go

@@ -10,6 +10,7 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/stackbuilders"
 	"github.com/portainer/portainer/api/stacks/stackutils"
@@ -115,7 +116,7 @@ type swarmStackFromGitRepositoryPayload struct {
 	// Applicable when deploying with multiple stack files
 	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
 	// Optional auto update configuration
-	AutoUpdate *portainer.StackAutoUpdate
+	AutoUpdate *portainer.AutoUpdateSettings
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -131,13 +132,13 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil
 }
 
-func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
+func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		Name:    name,
 		SwarmID: swarmID,

api/http/handler/stacks/stack_delete.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackutils"
+	"github.com/rs/zerolog/log"
 )
 
 // @id StackDelete
@@ -136,7 +137,7 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 
 	err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 	if err != nil {
-		return httperror.InternalServerError("Unable to remove stack files from disk", err)
+		log.Warn().Err(err).Msg("Unable to remove stack files from disk")
 	}
 
 	return response.Empty(w)

api/http/handler/stacks/stack_update_git.go

@@ -10,6 +10,7 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/git/update"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/stacks/deployments"
@@ -17,7 +18,7 @@ import (
 )
 
 type stackGitUpdatePayload struct {
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
 	Env                      []portainer.Pair
 	Prune                    bool
 	RepositoryReferenceName  string
@@ -27,7 +28,7 @@ type stackGitUpdatePayload struct {
 }
 
 func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -11,13 +11,12 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
 	"github.com/portainer/portainer/api/stacks/stackutils"
-
-	"github.com/rs/zerolog/log"
 )
 
 type stackGitRedployPayload struct {
@@ -154,22 +153,12 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
 		repositoryUsername = payload.RepositoryUsername
 	}
 
-	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	clean, err := git.CloneWithBackup(handler.GitService, handler.FileService, git.CloneOptions{ProjectPath: stack.ProjectPath, URL: stack.GitConfig.URL, ReferenceName: stack.GitConfig.ReferenceName, Username: repositoryUsername, Password: repositoryPassword})
 	if err != nil {
-		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
-		if restoreError != nil {
-			log.Warn().Err(restoreError).Msg("failed restoring backup folder")
-		}
-
-		return httperror.InternalServerError("Unable to clone git repository", err)
+		return httperror.InternalServerError("Unable to clone git repository directory", err)
 	}
 
-	defer func() {
-		err = handler.FileService.RemoveDirectory(backupProjectPath)
-		if err != nil {
-			log.Warn().Err(err).Msg("unable to remove git repository directory")
-		}
-	}()
+	defer clean()
 
 	httpErr := handler.deployStack(r, stack, payload.PullImage, endpoint)
 	if httpErr != nil {

api/http/handler/stacks/update_kubernetes_stack.go

@@ -11,10 +11,10 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
 	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
 	k "github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/stacks/deployments"
-	"github.com/portainer/portainer/api/stacks/stackutils"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
@@ -30,7 +30,7 @@ type kubernetesGitStackUpdatePayload struct {
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
-	AutoUpdate               *portainer.StackAutoUpdate
+	AutoUpdate               *portainer.AutoUpdateSettings
 }
 
 func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error {
@@ -41,7 +41,7 @@ func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error
 }
 
 func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error {
-	if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
+	if err := update.ValidateAutoUpdateSettings(payload.AutoUpdate); err != nil {
 		return err
 	}
 	return nil

api/http/handler/stacks/webhook_invoke.go

@@ -9,7 +9,6 @@ import (
 	"github.com/portainer/portainer/api/stacks/deployments"
 
 	"github.com/gofrs/uuid"
-	"github.com/rs/zerolog/log"
 )
 
 // @id WebhookInvoke
@@ -43,8 +42,6 @@ func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *h
 			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: "Autoupdate for the stack isn't available", Err: err}
 		}
 
-		log.Error().Err(err).Msg("failed to update the stack")
-
 		return httperror.InternalServerError("Failed to update the stack", err)
 	}
 

api/http/handler/stacks/webhook_invoke_test.go

@@ -18,7 +18,8 @@ func TestHandler_webhookInvoke(t *testing.T) {
 
 	webhookID := newGuidString(t)
 	store.StackService.Create(&portainer.Stack{
-		AutoUpdate: &portainer.StackAutoUpdate{
+		ID: 1,
+		AutoUpdate: &portainer.AutoUpdateSettings{
 			Webhook: webhookID,
 		},
 	})

api/http/handler/users/user_update.go

@@ -15,10 +15,18 @@ import (
 	"github.com/asaskevich/govalidator"
 )
 
+type themePayload struct {
+	// Color represents the color theme of the UI
+	Color *string `json:"color" example:"dark" enums:"dark,light,highcontrast,auto"`
+	// SubtleUpgradeButton indicates if the upgrade banner should be displayed in a subtle way
+	SubtleUpgradeButton *bool `json:"subtleUpgradeButton" example:"false"`
+}
+
 type userUpdatePayload struct {
-	Username  string `validate:"required" example:"bob"`
-	Password  string `validate:"required" example:"cg9Wgky3"`
-	UserTheme string `example:"dark"`
+	Username string `validate:"required" example:"bob"`
+	Password string `validate:"required" example:"cg9Wgky3"`
+	Theme    *themePayload
+
 	// User role (1 for administrator account and 2 for regular account)
 	Role int `validate:"required" enums:"1,2" example:"2"`
 }
@@ -108,8 +116,14 @@ func (handler *Handler) userUpdate(w http.ResponseWriter, r *http.Request) *http
 		user.TokenIssueAt = time.Now().Unix()
 	}
 
-	if payload.UserTheme != "" {
-		user.UserTheme = payload.UserTheme
+	if payload.Theme != nil {
+		if payload.Theme.Color != nil {
+			user.ThemeSettings.Color = *payload.Theme.Color
+		}
+
+		if payload.Theme.SubtleUpgradeButton != nil {
+			user.ThemeSettings.SubtleUpgradeButton = *payload.Theme.SubtleUpgradeButton
+		}
 	}
 
 	if payload.Role != 0 {
@@ -125,5 +139,8 @@ func (handler *Handler) userUpdate(w http.ResponseWriter, r *http.Request) *http
 	// remove all of the users persisted API keys
 	handler.apiKeyService.InvalidateUserKeyCache(user.ID)
 
+	// hide the password field in the response payload
+	user.Password = ""
+
 	return response.JSON(w, user)
 }

api/http/handler/websocket/proxy.go

@@ -1,14 +1,15 @@
 package websocket
 
 import (
-	"crypto/tls"
 	"fmt"
 	"net/http"
 	"net/url"
 
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+
 	"github.com/gorilla/websocket"
 	"github.com/koding/websocketproxy"
-	portainer "github.com/portainer/portainer/api"
 )
 
 func (handler *Handler) proxyEdgeAgentWebsocketRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error {
@@ -62,10 +63,12 @@ func (handler *Handler) proxyAgentWebsocketRequest(w http.ResponseWriter, r *htt
 
 	if params.endpoint.TLSConfig.TLS || params.endpoint.TLSConfig.TLSSkipVerify {
 		agentURL.Scheme = "wss"
+
+		tlsConfig := crypto.CreateTLSConfiguration()
+		tlsConfig.InsecureSkipVerify = params.endpoint.TLSConfig.TLSSkipVerify
+
 		proxy.Dialer = &websocket.Dialer{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: params.endpoint.TLSConfig.TLSSkipVerify,
-			},
+			TLSClientConfig: tlsConfig,
 		}
 	}
 

api/http/middlewares/featureflag.go

@@ -5,14 +5,14 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/pkg/featureflags"
 )
 
-func FeatureFlag(settingsService dataservices.SettingsService, feature portainer.Feature) mux.MiddlewareFunc {
+func FeatureFlag(settingsService dataservices.SettingsService, feature featureflags.Feature) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {
-			enabled := settingsService.IsFeatureFlagEnabled(feature)
+			enabled := featureflags.IsEnabled(feature)
 
 			if !enabled {
 				httperror.WriteError(rw, http.StatusForbidden, "This feature is not enabled", nil)

api/http/models/kubernetes/application.go

@@ -0,0 +1,11 @@
+package kubernetes
+
+type (
+	K8sApplication struct {
+		UID       string            `json:",omitempty"`
+		Name      string            `json:""`
+		Namespace string            `json:",omitempty"`
+		Kind      string            `json:",omitempty"`
+		Labels    map[string]string `json:",omitempty"`
+	}
+)

api/http/models/kubernetes/ingress.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"errors"
 	"net/http"
+	"time"
 )
 
 type (
@@ -18,15 +19,17 @@ type (
 	K8sIngressControllers []K8sIngressController
 
 	K8sIngressInfo struct {
-		Name        string            `json:"Name"`
-		UID         string            `json:"UID"`
-		Type        string            `json:"Type"`
-		Namespace   string            `json:"Namespace"`
-		ClassName   string            `json:"ClassName"`
-		Annotations map[string]string `json:"Annotations"`
-		Hosts       []string          `json:"Hosts"`
-		Paths       []K8sIngressPath  `json:"Paths"`
-		TLS         []K8sIngressTLS   `json:"TLS"`
+		Name         string            `json:"Name"`
+		UID          string            `json:"UID"`
+		Type         string            `json:"Type"`
+		Namespace    string            `json:"Namespace"`
+		ClassName    string            `json:"ClassName"`
+		Annotations  map[string]string `json:"Annotations"`
+		Hosts        []string          `json:"Hosts"`
+		Paths        []K8sIngressPath  `json:"Paths"`
+		TLS          []K8sIngressTLS   `json:"TLS"`
+		Labels       map[string]string `json:"Labels,omitempty"`
+		CreationDate time.Time         `json:"CreationDate"`
 	}
 
 	K8sIngressTLS struct {

api/http/models/kubernetes/services.go

@@ -7,17 +7,23 @@ import (
 
 type (
 	K8sServiceInfo struct {
-		Name                          string              `json:"Name"`
-		UID                           string              `json:"UID"`
-		Type                          string              `json:"Type"`
-		Namespace                     string              `json:"Namespace"`
-		Annotations                   map[string]string   `json:"Annotations"`
-		CreationTimestamp             string              `json:"CreationTimestamp"`
-		Labels                        map[string]string   `json:"Labels"`
-		AllocateLoadBalancerNodePorts *bool               `json:"AllocateLoadBalancerNodePorts,omitempty"`
-		Ports                         []K8sServicePort    `json:"Ports"`
-		Selector                      map[string]string   `json:"Selector"`
-		IngressStatus                 []K8sServiceIngress `json:"IngressStatus"`
+		Name                          string
+		UID                           string
+		Type                          string
+		Namespace                     string
+		Annotations                   map[string]string
+		CreationTimestamp             string
+		Labels                        map[string]string
+		AllocateLoadBalancerNodePorts *bool `json:",omitempty"`
+		Ports                         []K8sServicePort
+		Selector                      map[string]string
+		IngressStatus                 []K8sServiceIngress `json:",omitempty"`
+
+		// serviceList screen
+		Applications []K8sApplication `json:",omitempty"`
+		ClusterIPs   []string         `json:",omitempty"`
+		ExternalName string           `json:",omitempty"`
+		ExternalIPs  []string         `json:",omitempty"`
 	}
 
 	K8sServicePort struct {
@@ -25,7 +31,7 @@ type (
 		NodePort   int    `json:"NodePort"`
 		Port       int    `json:"Port"`
 		Protocol   string `json:"Protocol"`
-		TargetPort int    `json:"TargetPort"`
+		TargetPort string `json:"TargetPort"`
 	}
 
 	K8sServiceIngress struct {

api/http/server.go

@@ -346,7 +346,7 @@ func (server *Server) Start() error {
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), // Disable HTTP/2
 	}
 
-	httpsServer.TLSConfig = crypto.CreateServerTLSConfiguration()
+	httpsServer.TLSConfig = crypto.CreateTLSConfiguration()
 	httpsServer.TLSConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
 		return server.SSLService.GetRawCertificate(), nil
 	}

api/internal/edge/edgegroup.go

@@ -45,7 +45,7 @@ func EdgeGroupSet(edgeGroupIDs []portainer.EdgeGroupID) map[portainer.EdgeGroupI
 	return set
 }
 
-func GetEndpointsFromEdgeGroups(edgeGroupIDs []portainer.EdgeGroupID, datastore dataservices.DataStore) ([]portainer.EndpointID, error) {
+func GetEndpointsFromEdgeGroups(edgeGroupIDs []portainer.EdgeGroupID, datastore dataservices.DataStoreTx) ([]portainer.EndpointID, error) {
 	endpoints, err := datastore.Endpoint().Endpoints()
 	if err != nil {
 		return nil, err

api/internal/edge/edgejob.go

@@ -14,7 +14,12 @@ func LoadEdgeJobs(dataStore dataservices.DataStore, reverseTunnelService portain
 
 	for _, edgeJob := range edgeJobs {
 		for endpointID := range edgeJob.Endpoints {
-			reverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
+			endpoint, err := dataStore.Endpoint().Endpoint(endpointID)
+			if err != nil {
+				return err
+			}
+
+			reverseTunnelService.AddEdgeJob(endpoint, &edgeJob)
 		}
 	}
 

api/internal/testhelpers/datastore.go

@@ -36,10 +36,13 @@ type testDatastore struct {
 	webhook                 dataservices.WebhookService
 }
 
-func (d *testDatastore) BackupTo(io.Writer) error                           { return nil }
-func (d *testDatastore) Open() (bool, error)                                { return false, nil }
-func (d *testDatastore) Init() error                                        { return nil }
-func (d *testDatastore) Close() error                                       { return nil }
+func (d *testDatastore) BackupTo(io.Writer) error                            { return nil }
+func (d *testDatastore) Open() (bool, error)                                 { return false, nil }
+func (d *testDatastore) Init() error                                         { return nil }
+func (d *testDatastore) Close() error                                        { return nil }
+func (d *testDatastore) UpdateTx(func(dataservices.DataStoreTx) error) error { return nil }
+func (d *testDatastore) ViewTx(func(dataservices.DataStoreTx) error) error   { return nil }
+
 func (d *testDatastore) CheckCurrentEdition() error                         { return nil }
 func (d *testDatastore) MigrateData() error                                 { return nil }
 func (d *testDatastore) Rollback(force bool) error                          { return nil }
@@ -115,9 +118,6 @@ func (s *stubSettingsService) UpdateSettings(settings *portainer.Settings) error
 	s.settings = settings
 	return nil
 }
-func (s *stubSettingsService) IsFeatureFlagEnabled(feature portainer.Feature) bool {
-	return false
-}
 func WithSettingsService(settings *portainer.Settings) datastoreOption {
 	return func(d *testDatastore) {
 		d.settings = &stubSettingsService{
@@ -163,6 +163,9 @@ func (s *stubEdgeJobService) Create(ID portainer.EdgeJobID, edgeJob *portainer.E
 func (s *stubEdgeJobService) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	return nil
 }
+func (s *stubEdgeJobService) UpdateEdgeJobFunc(ID portainer.EdgeJobID, updateFunc func(edgeJob *portainer.EdgeJob)) error {
+	return nil
+}
 func (s *stubEdgeJobService) DeleteEdgeJob(ID portainer.EdgeJobID) error { return nil }
 func (s *stubEdgeJobService) GetNextIdentifier() int                     { return 0 }