EE-4286 fix(docker): update tls certs for docker env

* fix(image): hide button issues [EE-4166]

README.md

@@ -20,7 +20,7 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
 
 ## Getting started
 
-- [Deploy Portainer](https://docs.portainer.io/v/ce-2.9/start/install)
+- [Deploy Portainer](https://docs.portainer.io/start/install)
 - [Documentation](https://documentation.portainer.io)
 - [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
 

api/adminmonitor/admin_monitor.go

@@ -25,7 +25,7 @@ type Monitor struct {
 	adminInitDisabled bool
 }
 
-// New creates a monitor that when started will wait for the timeout duration and then sends the timeout signal to disable the application
+// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
 func New(timeout time.Duration, datastore dataservices.DataStore, shutdownCtx context.Context) *Monitor {
 	return &Monitor{
 		timeout:           timeout,
@@ -105,12 +105,10 @@ func (m *Monitor) WasInstanceDisabled() bool {
 // Otherwise, it will pass through the request to next
 func (m *Monitor) WithRedirect(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if m.WasInstanceDisabled() {
-			if strings.HasPrefix(r.RequestURI, "/api") && r.RequestURI != "/api/status" && r.RequestURI != "/api/settings/public" {
-				w.Header().Set("redirect-reason", RedirectReasonAdminInitTimeout)
-				httperror.WriteError(w, http.StatusSeeOther, "Administrator initialization timeout", nil)
-				return
-			}
+		if m.WasInstanceDisabled() && strings.HasPrefix(r.RequestURI, "/api") && r.RequestURI != "/api/status" && r.RequestURI != "/api/settings/public" {
+			w.Header().Set("redirect-reason", RedirectReasonAdminInitTimeout)
+			httperror.WriteError(w, http.StatusSeeOther, "Administrator initialization timeout", nil)
+			return
 		}
 
 		next.ServeHTTP(w, r)

api/apikey/service.go

@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/pkg/errors"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/pkg/errors"
 )
 
 const portainerAPIKeyPrefix = "ptr_"

api/apikey/service_test.go

@@ -2,6 +2,7 @@ package apikey
 
 import (
 	"crypto/sha256"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -170,11 +171,9 @@ func Test_UpdateAPIKey(t *testing.T) {
 		_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
 		is.NoError(err)
 
-		log.Debug().Msgf("%+v", apiKey)
-		log.Debug().Msgf("%+v", apiKeyGot)
+		log.Debug().Str("wanted", fmt.Sprintf("%+v", apiKey)).Str("got", fmt.Sprintf("%+v", apiKeyGot)).Msg("")
 
 		is.Equal(apiKey.LastUsed, apiKeyGot.LastUsed)
-
 	})
 
 	t.Run("Successfully updates api-key in cache upon api-key update", func(t *testing.T) {

api/archive/targz_test.go

@@ -2,7 +2,6 @@ package archive
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -28,10 +27,10 @@ func listFiles(dir string) []string {
 func Test_shouldCreateArhive(t *testing.T) {
 	tmpdir := t.TempDir()
 	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
 
 	gzPath, err := TarGzDir(tmpdir)
 	assert.Nil(t, err)
@@ -48,7 +47,7 @@ func Test_shouldCreateArhive(t *testing.T) {
 	wasExtracted := func(p string) {
 		fullpath := path.Join(extractionDir, p)
 		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
+		copyContent, _ := os.ReadFile(fullpath)
 		assert.Equal(t, content, copyContent)
 	}
 
@@ -60,10 +59,10 @@ func Test_shouldCreateArhive(t *testing.T) {
 func Test_shouldCreateArhiveXXXXX(t *testing.T) {
 	tmpdir := t.TempDir()
 	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
 
 	gzPath, err := TarGzDir(tmpdir)
 	assert.Nil(t, err)
@@ -80,7 +79,7 @@ func Test_shouldCreateArhiveXXXXX(t *testing.T) {
 	wasExtracted := func(p string) {
 		fullpath := path.Join(extractionDir, p)
 		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
+		copyContent, _ := os.ReadFile(fullpath)
 		assert.Equal(t, content, copyContent)
 	}
 

api/archive/zip.go

@@ -4,12 +4,12 @@ import (
 	"archive/zip"
 	"bytes"
 	"fmt"
-	"github.com/pkg/errors"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/pkg/errors"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -36,7 +36,7 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 	}
 	defer f.Close()
 
-	data, err := ioutil.ReadAll(f)
+	data, err := io.ReadAll(f)
 	if err != nil {
 		return err
 	}

api/archive/zip_test.go

@@ -1,9 +1,10 @@
 package archive
 
 import (
-	"github.com/stretchr/testify/assert"
 	"path/filepath"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestUnzipFile(t *testing.T) {

api/chisel/service.go

@@ -246,9 +246,8 @@ func (service *Service) checkTunnels() {
 			err := service.snapshotEnvironment(endpointID, tunnel.Port)
 			if err != nil {
 				log.Error().
-					Int("endpoint_id", int(endpointID)).Err(
-
-					err).
+					Int("endpoint_id", int(endpointID)).
+					Err(err).
 					Msg("unable to snapshot Edge environment")
 			}
 		}

api/cli/cli.go

@@ -62,6 +62,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()

api/cli/pairlistbool.go

@@ -1,10 +1,10 @@
 package cli
 
 import (
-	portainer "github.com/portainer/portainer/api"
-
 	"strings"
 
+	portainer "github.com/portainer/portainer/api"
+
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 

api/cmd/portainer/import.go

@@ -1,28 +0,0 @@
-package main
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/datastore"
-
-	"github.com/rs/zerolog/log"
-)
-
-func importFromJson(fileService portainer.FileService, store *datastore.Store) {
-	// EXPERIMENTAL - if used with an incomplete json file, it will fail, as we don't have a way to default the model values
-	importFile := "/data/import.json"
-	if exists, _ := fileService.FileExists(importFile); exists {
-		if err := store.Import(importFile); err != nil {
-			log.Error().Str("filename", importFile).Err(err).Msg("import failed")
-			// TODO: should really rollback on failure, but then we have nothing.
-		} else {
-			log.Info().Str("filename", importFile).Msg("successfully imported the file to a new portainer database")
-		}
-
-		// TODO: this is bad - its to ensure that any defaults that were broken in import, or migrations get set back to what we want
-		// I also suspect that everything from "Init to Init" is potentially a migration
-		err := store.Init()
-		if err != nil {
-			log.Fatal().Err(err).Msg("failed initializing data store")
-		}
-	}
-}

api/cmd/portainer/log.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"fmt"
 	stdlog "log"
+	"os"
 
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -31,3 +33,23 @@ func setLoggingLevel(level string) {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 }
+
+func setLoggingMode(mode string) {
+	switch mode {
+	case "PRETTY":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			NoColor:       true,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage})
+	case "JSON":
+		log.Logger = log.Output(os.Stderr)
+	}
+}
+
+func formatMessage(i interface{}) string {
+	if i == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s |", i)
+}

api/cmd/portainer/main.go

@@ -41,7 +41,7 @@ import (
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
-	"github.com/portainer/portainer/api/stacks"
+	"github.com/portainer/portainer/api/stacks/deployments"
 
 	"github.com/rs/zerolog/log"
 )
@@ -186,6 +186,10 @@ func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
 }
 
 func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore) (dataservices.JWTService, error) {
+	if userSessionTimeout == "" {
+		userSessionTimeout = portainer.DefaultUserSessionTimeout
+	}
+
 	jwtService, err := jwt.NewService(userSessionTimeout, dataStore)
 	if err != nil {
 		return nil, err
@@ -235,11 +239,17 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService,
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout)
 }
 
-func initSnapshotService(snapshotIntervalFromFlag string, dataStore dataservices.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+func initSnapshotService(
+	snapshotIntervalFromFlag string,
+	dataStore dataservices.DataStore,
+	dockerClientFactory *docker.ClientFactory,
+	kubernetesClientFactory *kubecli.ClientFactory,
+	shutdownCtx context.Context,
+) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
@@ -306,12 +316,7 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 		sslSettings.HTTPEnabled = true
 	}
 
-	err = dataStore.SSLSettings().UpdateSettings(sslSettings)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return dataStore.SSLSettings().UpdateSettings(sslSettings)
 }
 
 // enableFeaturesFromFlags turns on or off feature flags
@@ -580,6 +585,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	ldapService := initLDAPService()
 
 	oauthService := initOAuthService()
+
 	gitService := initGitService(shutdownCtx)
 
 	openAMTService := openamt.NewService()
@@ -606,7 +612,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
+	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
@@ -704,15 +710,15 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed starting tunnel server")
 	}
 
+	scheduler := scheduler.NewScheduler(shutdownCtx)
+	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
+	deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
+
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
 	if err != nil {
 		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
-	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
-	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
-
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
@@ -726,8 +732,8 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ComposeStackManager:         composeStackManager,
 		KubernetesDeployer:          kubernetesDeployer,
 		HelmPackageManager:          helmPackageManager,
-		CryptoService:               cryptoService,
 		APIKeyService:               apiKeyService,
+		CryptoService:               cryptoService,
 		JWTService:                  jwtService,
 		FileService:                 fileService,
 		LDAPService:                 ldapService,
@@ -752,10 +758,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 func main() {
 	configureLogger()
+	setLoggingMode("PRETTY")
 
 	flags := initCLI()
 
 	setLoggingLevel(*flags.LogLevel)
+	setLoggingMode(*flags.LogMode)
 
 	for {
 		server := buildServer(flags)

api/cmd/portainer/main_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/alecthomas/kingpin.v2"
 )

api/connection.go

@@ -24,13 +24,13 @@ type Connection interface {
 	SetServiceName(bucketName string) error
 	GetObject(bucketName string, key []byte, object interface{}) error
 	UpdateObject(bucketName string, key []byte, object interface{}) error
+	UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
 	DeleteObject(bucketName string, key []byte) error
 	DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error
 	GetNextIdentifier(bucketName string) int
 	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
 	CreateObjectWithId(bucketName string, id int, obj interface{}) error
 	CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
-	CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error
 	GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
 	GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
 	ConvertToKey(v int) []byte

api/crypto/aes.go

@@ -13,7 +13,7 @@ import (
 // Person with better knowledge is welcomed to improve it.
 // sourced from https://golang.org/src/crypto/cipher/example_test.go
 
-var emptySalt []byte = make([]byte, 0, 0)
+var emptySalt []byte = make([]byte, 0)
 
 // AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
 // passphrase is used to generate an encryption key.

api/crypto/aes_test.go

@@ -2,7 +2,6 @@ package crypto
 
 import (
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -20,7 +19,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 	)
 
 	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
+	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
@@ -30,7 +29,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
 
@@ -45,7 +44,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 
 	io.Copy(decryptedFileWriter, decryptedReader)
 
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
 	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
 }
 
@@ -59,7 +58,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 	)
 
 	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
+	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
@@ -69,7 +68,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte(""))
 	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
 
@@ -84,7 +83,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 
 	io.Copy(decryptedFileWriter, decryptedReader)
 
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
 	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
 }
 
@@ -98,7 +97,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	)
 
 	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
+	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
@@ -108,7 +107,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
 
@@ -123,6 +122,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 
 	io.Copy(decryptedFileWriter, decryptedReader)
 
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
 	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
 }

api/crypto/tls.go

@@ -3,7 +3,7 @@ package crypto
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
+	"os"
 )
 
 // CreateServerTLSConfiguration creates a basic tls.Config to be used by servers with recommended TLS settings
@@ -63,7 +63,7 @@ func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipSe
 	}
 
 	if !skipServerVerification && caCertPath != "" {
-		caCert, err := ioutil.ReadFile(caCertPath)
+		caCert, err := os.ReadFile(caCertPath)
 		if err != nil {
 			return nil, err
 		}

api/database/boltdb/db.go

@@ -1,11 +1,11 @@
 package boltdb
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"time"
@@ -162,11 +162,11 @@ func (connection *DbConnection) ExportRaw(filename string) error {
 		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
-	b, err := connection.ExportJson(databasePath, true)
+	b, err := connection.ExportJSON(databasePath, true)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(filename, b, 0600)
+	return os.WriteFile(filename, b, 0600)
 }
 
 // ConvertToKey returns an 8-byte big endian representation of v.
@@ -178,7 +178,7 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 	return b
 }
 
-// CreateBucket is a generic function used to create a bucket inside a database database.
+// CreateBucket is a generic function used to create a bucket inside a database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
 	return connection.Batch(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
@@ -186,7 +186,7 @@ func (connection *DbConnection) SetServiceName(bucketName string) error {
 	})
 }
 
-// GetObject is a generic function used to retrieve an unmarshalled object from a database database.
+// GetObject is a generic function used to retrieve an unmarshalled object from a database.
 func (connection *DbConnection) GetObject(bucketName string, key []byte, object interface{}) error {
 	var data []byte
 
@@ -218,7 +218,7 @@ func (connection *DbConnection) getEncryptionKey() []byte {
 	return connection.EncryptionKey
 }
 
-// UpdateObject is a generic function used to update an object inside a database database.
+// UpdateObject is a generic function used to update an object inside a database.
 func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
 	data, err := connection.MarshalObject(object)
 	if err != nil {
@@ -231,7 +231,33 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje
 	})
 }
 
-// DeleteObject is a generic function used to delete an object inside a database database.
+// UpdateObjectFunc is a generic function used to update an object safely without race conditions.
+func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+
+		data := bucket.Get(key)
+		if data == nil {
+			return dserrors.ErrObjectNotFound
+		}
+
+		err := connection.UnmarshalObjectWithJsoniter(data, object)
+		if err != nil {
+			return err
+		}
+
+		updateFn()
+
+		data, err = connection.MarshalObject(object)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(key, data)
+	})
+}
+
+// DeleteObject is a generic function used to delete an object inside a database.
 func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
 	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
@@ -325,27 +351,6 @@ func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id [
 	})
 }
 
-// CreateObjectWithSetSequence creates a new object in the bucket, using the specified id, and sets the bucket sequence
-// avoid this :)
-func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error {
-	return connection.Batch(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(bucketName))
-
-		// We manually manage sequences for schedules
-		err := bucket.SetSequence(uint64(id))
-		if err != nil {
-			return err
-		}
-
-		data, err := connection.MarshalObject(obj)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(connection.ConvertToKey(id), data)
-	})
-}
-
 func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
 	err := connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
@@ -390,6 +395,27 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
 	return err
 }
 
+func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
+	return connection.View(func(tx *bolt.Tx) error {
+		cursor := tx.Bucket([]byte(bucketName)).Cursor()
+
+		for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
+			err := connection.UnmarshalObjectWithJsoniter(v, obj)
+			if err != nil {
+				return err
+			}
+
+			obj, err = append(obj)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// BackupMetadata will return a copy of the boltdb sequence numbers for all buckets.
 func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error) {
 	buckets := map[string]interface{}{}
 
@@ -408,6 +434,7 @@ func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error)
 	return buckets, err
 }
 
+// RestoreMetadata will restore the boltdb sequence numbers for all buckets.
 func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error {
 	var err error
 

api/database/boltdb/export.go

@@ -31,7 +31,7 @@ func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
 // using this function.
 // inspired by github.com/konoui/boltdb-exporter (which has no license)
 // but very much simplified, based on how we use boltdb
-func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, error) {
+func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, error) {
 	log.Debug().Str("databasePath", databasePath).Msg("exportJson")
 
 	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})

api/database/database.go

@@ -16,5 +16,6 @@ func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection
 			EncryptionKey: encryptionKey,
 		}, nil
 	}
-	return nil, fmt.Errorf("unknown storage database: %s", storeType)
+
+	return nil, fmt.Errorf("Unknown storage database: %s", storeType)
 }

api/database/models/ingress.go

@@ -12,6 +12,7 @@ type (
 		Type         string `json:"Type"`
 		Availability bool   `json:"Availability"`
 		New          bool   `json:"New"`
+		Used         bool   `json:"Used"`
 	}
 
 	K8sIngressControllers []K8sIngressController
@@ -58,9 +59,6 @@ func (r K8sIngressInfo) Validate(request *http.Request) error {
 	if r.Namespace == "" {
 		return errors.New("missing ingress Namespace from the request payload")
 	}
-	if r.ClassName == "" {
-		return errors.New("missing ingress ClassName from the request payload")
-	}
 	return nil
 }
 

api/database/models/namespaces.go

@@ -2,11 +2,11 @@ package models
 
 import "net/http"
 
-type K8sNamespaceInfo struct {
+type K8sNamespaceDetails struct {
 	Name        string            `json:"Name"`
 	Annotations map[string]string `json:"Annotations"`
 }
 
-func (r *K8sNamespaceInfo) Validate(request *http.Request) error {
+func (r *K8sNamespaceDetails) Validate(request *http.Request) error {
 	return nil
 }

api/dataservices/edgeupdateschedule/edgeupdateschedule.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/edgetypes"
-	"github.com/sirupsen/logrus"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -84,8 +85,8 @@ func (service *Service) List() ([]edgetypes.UpdateSchedule, error) {
 		func(obj interface{}) (interface{}, error) {
 			item, ok := obj.(*edgetypes.UpdateSchedule)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeUpdateSchedule object")
-				return nil, fmt.Errorf("failed to convert to EdgeUpdateSchedule object: %s", obj)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeUpdateSchedule object")
+				return nil, fmt.Errorf("Failed to convert to EdgeUpdateSchedule object: %s", obj)
 			}
 			list = append(list, *item)
 			return &edgetypes.UpdateSchedule{}, nil

api/dataservices/endpoint/endpoint.go

@@ -83,7 +83,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 
 // CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
 func (service *Service) Create(endpoint *portainer.Endpoint) error {
-	return service.connection.CreateObjectWithSetSequence(BucketName, int(endpoint.ID), endpoint)
+	return service.connection.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
 }
 
 // GetNextIdentifier returns the next identifier for an environment(endpoint).

api/dataservices/interface.go

@@ -24,7 +24,6 @@ type (
 		BackupTo(w io.Writer) error
 		Export(filename string) (err error)
 		IsErrObjectNotFound(err error) bool
-
 		CustomTemplate() CustomTemplateService
 		EdgeGroup() EdgeGroupService
 		EdgeJob() EdgeJobService
@@ -252,6 +251,7 @@ type (
 		Tag(ID portainer.TagID) (*portainer.Tag, error)
 		Create(tag *portainer.Tag) error
 		UpdateTag(ID portainer.TagID, tag *portainer.Tag) error
+		UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error
 		DeleteTag(ID portainer.TagID) error
 		BucketName() string
 	}

api/dataservices/schedule/schedule.go

@@ -108,7 +108,7 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 
 // Create assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	return service.connection.CreateObjectWithSetSequence(BucketName, int(schedule.ID), schedule)
+	return service.connection.CreateObjectWithId(BucketName, int(schedule.ID), schedule)
 }
 
 // GetNextIdentifier returns the next identifier for a schedule.

api/dataservices/snapshot/snapshot.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -51,7 +52,7 @@ func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
 		func(obj interface{}) (interface{}, error) {
 			snapshot, ok := obj.(*portainer.Snapshot)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Snapshot object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Snapshot object")
 				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
 			}
 			snapshots = append(snapshots, *snapshot)

api/dataservices/stack/stack.go

@@ -134,7 +134,7 @@ func (service *Service) GetNextIdentifier() int {
 
 // CreateStack creates a new stack.
 func (service *Service) Create(stack *portainer.Stack) error {
-	return service.connection.CreateObjectWithSetSequence(BucketName, int(stack.ID), stack)
+	return service.connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
 }
 
 // UpdateStack updates a stack.

api/dataservices/tag/tag.go

@@ -80,12 +80,24 @@ func (service *Service) Create(tag *portainer.Tag) error {
 	)
 }
 
-// UpdateTag updates a tag.
+// Deprecated: Use UpdateTagFunc instead.
 func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, tag)
 }
 
+// UpdateTagFunc updates a tag inside a transaction avoiding data races.
+func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error {
+	id := service.connection.ConvertToKey(int(ID))
+	tag := &portainer.Tag{}
+
+	service.connection.UpdateObjectFunc(BucketName, id, tag, func() {
+		updateFunc(tag)
+	})
+
+	return nil
+}
+
 // DeleteTag deletes a tag.
 func (service *Service) DeleteTag(ID portainer.TagID) error {
 	identifier := service.connection.ConvertToKey(int(ID))

api/datastore/datastore.go

@@ -156,6 +156,7 @@ func (store *Store) encryptDB() error {
 	if err != nil {
 		// Remove the new encrypted file that we failed to import
 		os.Remove(store.connection.GetDatabaseFilePath())
+
 		log.Fatal().Err(portainerErrors.ErrDBImportFailed).Msg("")
 	}
 

api/datastore/init.go

@@ -36,6 +36,7 @@ func (store *Store) checkOrCreateInstanceID() error {
 		instanceID := uid.String()
 		return store.VersionService.StoreInstanceID(instanceID)
 	}
+
 	return err
 }
 
@@ -88,7 +89,6 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 
 func (store *Store) checkOrCreateDefaultSSLSettings() error {
 	_, err := store.SSLSettings().Settings()
-
 	if store.IsErrObjectNotFound(err) {
 		defaultSSLSettings := &portainer.SSLSettings{
 			HTTPEnabled: true,
@@ -96,6 +96,7 @@ func (store *Store) checkOrCreateDefaultSSLSettings() error {
 
 		return store.SSLSettings().UpdateSettings(defaultSSLSettings)
 	}
+
 	return err
 }
 

api/datastore/migrate_data_test.go

@@ -259,7 +259,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
 		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
-	gotJSON, err := con.ExportJson(databasePath, false)
+	gotJSON, err := con.ExportJSON(databasePath, false)
 	if err != nil {
 		t.Logf(
 			"failed re-exporting database %s to JSON: %v",
@@ -275,7 +275,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
 
 	// Compare the result we got with the one we wanted.
 	if diff := cmp.Diff(wantJSON, gotJSON); diff != "" {
-		gotPath := filepath.Join(t.TempDir(), "portainer-migrator-test-fail.json")
+		gotPath := filepath.Join(os.TempDir(), "portainer-migrator-test-fail.json")
 		os.WriteFile(
 			gotPath,
 			gotJSON,

api/datastore/migrator/migrate_ce.go

@@ -1,13 +1,12 @@
 package migrator
 
 import (
-	"errors"
 	"reflect"
 	"runtime"
 
 	portainer "github.com/portainer/portainer/api"
 
-	werrors "github.com/pkg/errors"
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
 
@@ -17,7 +16,7 @@ type migration struct {
 }
 
 func migrationError(err error, context string) error {
-	return werrors.Wrap(err, "failed in "+context)
+	return errors.Wrap(err, "failed in "+context)
 }
 
 func newMigration(dbversion int, migrate func() error) migration {

api/datastore/migrator/migrate_dbversion19.go

@@ -15,6 +15,7 @@ func (m *Migrator) updateUsersToDBVersion20() error {
 }
 
 func (m *Migrator) updateSettingsToDBVersion20() error {
+	log.Info().Msg("updating settings")
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion22.go

@@ -22,6 +22,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 			return err
 		}
 	}
+
 	return nil
 }
 

api/datastore/migrator/migrate_dbversion26.go

@@ -3,7 +3,7 @@ package migrator
 import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/portainer/portainer/api/internal/stackutils"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 
 	"github.com/rs/zerolog/log"
 )

api/datastore/migrator/migrate_dbversion29.go

@@ -19,5 +19,6 @@ func (m *Migrator) MigrateSettingsToDB30() error {
 
 	legacySettings.OAuthSettings.SSO = false
 	legacySettings.OAuthSettings.LogoutURI = ""
+
 	return m.settingsService.UpdateSettings(legacySettings)
 }

api/datastore/migrator/migrate_dbversion32.go

@@ -13,6 +13,7 @@ func (m *Migrator) migrateDBVersionToDB33() error {
 }
 
 func (m *Migrator) migrateSettingsToDB33() error {
+	log.Info().Msg("setting default kubctl shell")
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion34_test.go

@@ -1,94 +0,0 @@
-package migrator
-
-const (
-	db35TestFile = "portainer-mig-35.db"
-	username     = "portainer"
-	password     = "password"
-)
-
-// TODO: this is exactly the kind of reaching into the internals of the store we should not do
-// func setupDB35Test(t *testing.T) *Migrator {
-// 	is := assert.New(t)
-// 	dbConn, err := bolt.Open(path.Join(t.TempDir(), db35TestFile), 0600, &bolt.Options{Timeout: 1 * time.Second})
-// 	is.NoError(err, "failed to init testing DB connection")
-
-// 	// Create an old style dockerhub authenticated account
-// 	dockerhubService, err := dockerhub.NewService(&database.DbConnection{DB: dbConn})
-// 	is.NoError(err, "failed to init testing registry service")
-// 	err = dockerhubService.UpdateDockerHub(&portainer.DockerHub{true, username, password})
-// 	is.NoError(err, "failed to create dockerhub account")
-
-// 	registryService, err := registry.NewService(&database.DbConnection{DB: dbConn})
-// 	is.NoError(err, "failed to init testing registry service")
-
-// 	endpointService, err := endpoint.NewService(&database.DbConnection{DB: dbConn})
-// 	is.NoError(err, "failed to init endpoint service")
-
-// 	m := &Migrator{
-// 		db:               dbConn,
-// 		dockerhubService: dockerhubService,
-// 		registryService:  registryService,
-// 		endpointService:  endpointService,
-// 	}
-
-// 	return m
-// }
-
-// // TestUpdateDockerhubToDB32 tests a normal upgrade
-// func TestUpdateDockerhubToDB32(t *testing.T) {
-// 	is := assert.New(t)
-// 	m := setupDB35Test(t)
-// 	defer m.db.Close()
-// 	defer os.Remove(db35TestFile)
-
-// 	if err := m.updateDockerhubToDB32(); err != nil {
-// 		t.Errorf("failed to update settings: %v", err)
-// 	}
-
-// 	// Verify we have a single registry were created
-// 	registries, err := m.registryService.Registries()
-// 	is.NoError(err, "failed to read registries from the RegistryService")
-// 	is.Equal(len(registries), 1, "only one migrated registry expected")
-// }
-
-// // TestUpdateDockerhubToDB32_with_duplicate_migrations tests an upgrade where in earlier versions a broken migration
-// // created a large number of duplicate "dockerhub migrated" registry entries.
-// func TestUpdateDockerhubToDB32_with_duplicate_migrations(t *testing.T) {
-// 	is := assert.New(t)
-// 	m := setupDB35Test(t)
-// 	defer m.db.Close()
-// 	defer os.Remove(db35TestFile)
-
-// 	// Create lots of duplicate entries...
-// 	registry := &portainer.Registry{
-// 		Type:             portainer.DockerHubRegistry,
-// 		Name:             "Dockerhub (authenticated - migrated)",
-// 		URL:              "docker.io",
-// 		Authentication:   true,
-// 		Username:         "portainer",
-// 		Password:         "password",
-// 		RegistryAccesses: portainer.RegistryAccesses{},
-// 	}
-
-// 	for i := 1; i < 150; i++ {
-// 		err := m.registryService.CreateRegistry(registry)
-// 		assert.NoError(t, err, "create registry failed")
-// 	}
-
-// 	// Verify they were created
-// 	registries, err := m.registryService.Registries()
-// 	is.NoError(err, "failed to read registries from the RegistryService")
-// 	is.Condition(func() bool {
-// 		return len(registries) > 1
-// 	}, "expected multiple duplicate registry entries")
-
-// 	// Now run the migrator
-// 	if err := m.updateDockerhubToDB32(); err != nil {
-// 		t.Errorf("failed to update settings: %v", err)
-// 	}
-
-// 	// Verify we have a single registry were created
-// 	registries, err = m.registryService.Registries()
-// 	is.NoError(err, "failed to read registries from the RegistryService")
-// 	is.Equal(len(registries), 1, "only one migrated registry expected")
-// }

api/datastore/migrator/migrate_dbversion35.go

@@ -14,6 +14,8 @@ func (m *Migrator) migrateDBVersionToDB36() error {
 }
 
 func (m *Migrator) migrateUsersToDB36() error {
+	log.Info().Msg("updating user authorizations")
+
 	users, err := m.userService.Users()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion70.go

@@ -2,11 +2,16 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
 	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) migrateDBVersionToDB70() error {
-	// foreach endpoint
+	log.Info().Msg("- add IngressAvailabilityPerNamespace field")
+	if err := m.updateIngressFieldsForEnvDB70(); err != nil {
+		return err
+	}
+
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
@@ -45,3 +50,21 @@ func (m *Migrator) migrateDBVersionToDB70() error {
 
 	return nil
 }
+
+func (m *Migrator) updateIngressFieldsForEnvDB70() error {
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		endpoint.Kubernetes.Configuration.IngressAvailabilityPerNamespace = true
+		endpoint.Kubernetes.Configuration.AllowNoneIngressClass = false
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/datastore/services.go

@@ -3,7 +3,7 @@ package datastore
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
@@ -518,7 +518,7 @@ func (store *Store) Export(filename string) (err error) {
 
 	if snapshot, err := store.Snapshot().Snapshots(); err != nil {
 		if !store.IsErrObjectNotFound(err) {
-			log.Err(err).Msg("Exporting Snapshots")
+			log.Error().Err(err).Msg("exporting Snapshots")
 		}
 	} else {
 		backup.Snapshot = snapshot
@@ -607,13 +607,13 @@ func (store *Store) Export(filename string) (err error) {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(filename, b, 0600)
+	return os.WriteFile(filename, b, 0600)
 }
 
 func (store *Store) Import(filename string) (err error) {
 	backup := storeExport{}
 
-	s, err := ioutil.ReadFile(filename)
+	s, err := os.ReadFile(filename)
 	if err != nil {
 		return err
 	}
@@ -707,7 +707,7 @@ func (store *Store) Import(filename string) (err error) {
 
 	for _, user := range backup.User {
 		if err := store.User().UpdateUser(user.ID, &user); err != nil {
-			log.Debug().Str("user", fmt.Sprintf("%+v", user)).Err(err).Msg("user: failed to Update Database")
+			log.Debug().Str("user", fmt.Sprintf("%+v", user)).Err(err).Msg("failed to update the user in the database")
 		}
 	}
 

api/datastore/test_data/output_24_to_latest.json

@@ -52,7 +52,9 @@
       "IsEdgeDevice": false,
       "Kubernetes": {
         "Configuration": {
+          "AllowNoneIngressClass": false,
           "EnableResourceOverCommit": false,
+          "IngressAvailabilityPerNamespace": true,
           "IngressClasses": null,
           "ResourceOverCommitPercentage": 0,
           "RestrictDefaultNamespace": false,
@@ -926,7 +928,7 @@
   ],
   "version": {
     "DB_UPDATING": "false",
-    "DB_VERSION": "70",
+    "DB_VERSION": "80",
     "INSTANCE_ID": "null"
   }
 }

api/docker/labels.go

@@ -0,0 +1,6 @@
+package docker
+
+const (
+	ComposeStackNameLabel = "com.docker.compose.project"
+	SwarmStackNameLabel   = "com.docker.stack.namespace"
+)

api/docker/snapshot.go

@@ -159,7 +159,7 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 	gpuUseSet := make(map[string]struct{})
 	gpuUseAll := false
 	for _, container := range containers {
-		if container.State == "exited" {
+		if container.State == "exited" || container.State == "stopped" {
 			stoppedContainers++
 		} else if container.State == "running" {
 			runningContainers++
@@ -194,7 +194,7 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 		}
 
 		for k, v := range container.Labels {
-			if k == "com.docker.compose.project" {
+			if k == ComposeStackNameLabel {
 				stacks[v] = struct{}{}
 			}
 		}

api/exec/compose_stack.go

@@ -8,15 +8,14 @@ import (
 	"path"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	libstack "github.com/portainer/docker-compose-wrapper"
 	"github.com/portainer/docker-compose-wrapper/compose"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
-	"github.com/portainer/portainer/api/internal/stackutils"
+	"github.com/portainer/portainer/api/stacks/stackutils"
+
+	"github.com/pkg/errors"
 )
 
 // ComposeStackManager is a wrapper for docker-compose binary
@@ -59,7 +58,7 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack)
+	filePaths := stackutils.GetStackFilePaths(stack, false)
 	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile, forceRereate)
 	return errors.Wrap(err, "failed to deploy a stack")
 }
@@ -79,7 +78,7 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack)
+	filePaths := stackutils.GetStackFilePaths(stack, false)
 
 	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
 	return errors.Wrap(err, "failed to remove a stack")
@@ -101,7 +100,7 @@ func (manager *ComposeStackManager) Pull(ctx context.Context, stack *portainer.S
 		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePaths := stackutils.GetStackFilePaths(stack)
+	filePaths := stackutils.GetStackFilePaths(stack, false)
 	err = manager.deployer.Pull(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
 	return errors.Wrap(err, "failed to pull images of the stack")
 }

api/exec/compose_stack_test.go

@@ -1,7 +1,7 @@
 package exec
 
 import (
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"testing"
@@ -55,7 +55,7 @@ func Test_createEnvFile(t *testing.T) {
 				assert.Equal(t, "stack.env", result)
 
 				f, _ := os.Open(path.Join(dir, "stack.env"))
-				content, _ := ioutil.ReadAll(f)
+				content, _ := io.ReadAll(f)
 
 				assert.Equal(t, tt.expected, string(content))
 			} else {
@@ -80,7 +80,7 @@ func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) {
 	assert.NoError(t, err)
 	assert.FileExists(t, path.Join(dir, "stack.env"))
 	f, _ := os.Open(path.Join(dir, "stack.env"))
-	content, _ := ioutil.ReadAll(f)
+	content, _ := io.ReadAll(f)
 
 	assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content)
 }

api/exec/exectest/kubernetes_mocks.go

@@ -6,6 +6,7 @@ import (
 
 type kubernetesMockDeployer struct{}
 
+// NewKubernetesDeployer creates a mock kubernetes deployer
 func NewKubernetesDeployer() portainer.KubernetesDeployer {
 	return &kubernetesMockDeployer{}
 }

api/exec/kubernetes_deploy.go

@@ -9,14 +9,14 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/kubernetes/cli"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/pkg/errors"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
@@ -73,6 +73,7 @@ func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *
 	if token == "" {
 		return "", fmt.Errorf("can not get a valid user service account token")
 	}
+
 	return token, nil
 }
 

api/exec/swarm_stack.go

@@ -14,7 +14,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/registryutils"
-	"github.com/portainer/portainer/api/internal/stackutils"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -90,7 +90,7 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, pullImage bool, endpoint *portainer.Endpoint) error {
-	filePaths := stackutils.GetStackFilePaths(stack)
+	filePaths := stackutils.GetStackFilePaths(stack, false)
 	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
 	if err != nil {
 		return err
@@ -203,12 +203,7 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string
 	headersObject["X-PortainerAgent-Signature"] = signature
 	headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey()
 
-	err = manager.fileService.WriteJSONToFile(configFilePath, config)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return manager.fileService.WriteJSONToFile(configFilePath, config)
 }
 
 func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]interface{}, error) {

api/filesystem/copy_test.go

@@ -1,7 +1,6 @@
 package filesystem
 
 import (
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -19,12 +18,12 @@ func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
 func Test_copyFile_shouldMakeAbackup(t *testing.T) {
 	tmpdir := t.TempDir()
 	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
 
 	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
 	assert.NoError(t, err)
 
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	copyContent, _ := os.ReadFile(path.Join(tmpdir, "copy"))
 	assert.Equal(t, content, copyContent)
 }
 
@@ -59,13 +58,13 @@ func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
 func Test_CopyPath_shouldCopyFile(t *testing.T) {
 	tmpdir := t.TempDir()
 	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+	os.WriteFile(path.Join(tmpdir, "file"), content, 0600)
 
 	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
 	err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
 	assert.NoError(t, err)
 
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	copyContent, err := os.ReadFile(path.Join(tmpdir, "backup", "file"))
 	assert.NoError(t, err)
 	assert.Equal(t, content, copyContent)
 }

api/filesystem/filesystem.go

@@ -163,7 +163,7 @@ func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExi
 	}
 
 	if !exists {
-		return errors.New(fmt.Sprintf("File (%s) doesn't exist", fromFilePath))
+		return fmt.Errorf("File (%s) doesn't exist", fromFilePath)
 	}
 
 	finput, err := os.Open(fromFilePath)

api/filesystem/write_test.go

@@ -1,7 +1,7 @@
 package filesystem
 
 import (
-	"io/ioutil"
+	"os"
 	"path"
 	"testing"
 
@@ -16,7 +16,7 @@ func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) {
 	err := WriteToFile(tmpFilePath, content)
 	assert.NoError(t, err)
 
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	fileContent, _ := os.ReadFile(tmpFilePath)
 	assert.Equal(t, content, fileContent)
 }
 
@@ -31,7 +31,7 @@ func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) {
 	err = WriteToFile(tmpFilePath, content)
 	assert.NoError(t, err)
 
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	fileContent, _ := os.ReadFile(tmpFilePath)
 	assert.Equal(t, content, fileContent)
 }
 
@@ -43,6 +43,6 @@ func Test_WriteFile_CanWriteANestedPath(t *testing.T) {
 	err := WriteToFile(tmpFilePath, content)
 	assert.NoError(t, err)
 
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	fileContent, _ := os.ReadFile(tmpFilePath)
 	assert.Equal(t, content, fileContent)
 }

api/git/azure.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -17,6 +16,7 @@ import (
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
+	gittypes "github.com/portainer/portainer/api/git/types"
 )
 
 const (
@@ -99,7 +99,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to build download url")
 	}
-	zipFile, err := ioutil.TempFile("", "azure-git-repo-*.zip")
+	zipFile, err := os.CreateTemp("", "azure-git-repo-*.zip")
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to create temp file")
 	}
@@ -483,9 +483,9 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
 
 func checkAzureStatusCode(err error, code int) error {
 	if code == http.StatusNotFound {
-		return ErrIncorrectRepositoryURL
+		return gittypes.ErrIncorrectRepositoryURL
 	} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
-		return ErrAuthenticationFailure
+		return gittypes.ErrAuthenticationFailure
 	}
 	return err
 }

api/git/azure_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	_ "github.com/joho/godotenv/autoload"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -145,7 +146,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -161,7 +162,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -240,7 +241,7 @@ func TestService_ListFiles_Azure(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrIncorrectRepositoryURL,
+				err:        gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}

api/git/azure_test.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"testing"
 
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -466,7 +467,7 @@ func Test_listRefs_azure(t *testing.T) {
 				password:      "test-token",
 			},
 			expect: expectResult{
-				err: ErrAuthenticationFailure,
+				err: gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -477,7 +478,7 @@ func Test_listRefs_azure(t *testing.T) {
 				password:      "",
 			},
 			expect: expectResult{
-				err: ErrAuthenticationFailure,
+				err: gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -488,7 +489,7 @@ func Test_listRefs_azure(t *testing.T) {
 				password:      accessToken,
 			},
 			expect: expectResult{
-				err: ErrIncorrectRepositoryURL,
+				err: gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}
@@ -540,7 +541,7 @@ func Test_listFiles_azure(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -555,7 +556,7 @@ func Test_listFiles_azure(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -599,7 +600,7 @@ func Test_listFiles_azure(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrIncorrectRepositoryURL,
+				err:        gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}

api/git/git.go

@@ -6,14 +6,14 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
+	"github.com/pkg/errors"
+	gittypes "github.com/portainer/portainer/api/git/types"
 )
 
 type gitClient struct {
@@ -41,7 +41,7 @@ func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) e
 
 	if err != nil {
 		if err.Error() == "authentication required" {
-			return ErrAuthenticationFailure
+			return gittypes.ErrAuthenticationFailure
 		}
 		return errors.Wrap(err, "failed to clone git repository")
 	}
@@ -66,7 +66,7 @@ func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string
 	refs, err := remote.List(listOptions)
 	if err != nil {
 		if err.Error() == "authentication required" {
-			return "", ErrAuthenticationFailure
+			return "", gittypes.ErrAuthenticationFailure
 		}
 		return "", errors.Wrap(err, "failed to list repository refs")
 	}
@@ -172,9 +172,9 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
 func checkGitError(err error) error {
 	errMsg := err.Error()
 	if errMsg == "repository not found" {
-		return ErrIncorrectRepositoryURL
+		return gittypes.ErrIncorrectRepositoryURL
 	} else if errMsg == "authentication required" {
-		return ErrAuthenticationFailure
+		return gittypes.ErrAuthenticationFailure
 	}
 	return err
 }

api/git/git_integration_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -99,7 +100,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -115,7 +116,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -194,7 +195,7 @@ func TestService_ListFiles_GitHub(t *testing.T) {
 			extensions: []string{},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrIncorrectRepositoryURL,
+				err:        gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}
@@ -273,7 +274,8 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) {
 
 func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) {
 	timeout := 10 * time.Millisecond
-	deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout))
+	deadlineCtx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout))
+	defer cancel()
 
 	service := NewService(deadlineCtx)
 	assert.False(t, service.timerHasStopped(), "timer should not be stopped")
@@ -296,9 +298,9 @@ func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
 	assert.GreaterOrEqual(t, len(refs), 1)
 	assert.Equal(t, 1, service.repoRefCache.Len())
 
-	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
 	assert.Error(t, err)
-	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoRefCache.Len())
 }
 
 func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
@@ -324,10 +326,13 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
 	assert.GreaterOrEqual(t, len(files), 1)
 	assert.Equal(t, 2, service.repoFileCache.Len())
 
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false)
+	assert.Error(t, err)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
 	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
 	assert.Error(t, err)
-	assert.Equal(t, 0, service.repoRefCache.Len())
-
+	assert.Equal(t, 1, service.repoRefCache.Len())
 	// The relevant file caches should be removed too
 	assert.Equal(t, 0, service.repoFileCache.Len())
 }

api/git/git_test.go

@@ -6,11 +6,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/portainer/portainer/api/archive"
-
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -148,7 +148,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
 				password:      "test-token",
 			},
 			expect: expectResult{
-				err: ErrAuthenticationFailure,
+				err: gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -159,7 +159,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
 				password:      "",
 			},
 			expect: expectResult{
-				err: ErrAuthenticationFailure,
+				err: gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -170,7 +170,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
 				password:      accessToken,
 			},
 			expect: expectResult{
-				err: ErrIncorrectRepositoryURL,
+				err: gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}
@@ -222,7 +222,7 @@ func Test_listFilesPrivateRepository(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -237,7 +237,7 @@ func Test_listFilesPrivateRepository(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrAuthenticationFailure,
+				err:        gittypes.ErrAuthenticationFailure,
 			},
 		},
 		{
@@ -281,7 +281,7 @@ func Test_listFilesPrivateRepository(t *testing.T) {
 			},
 			expect: expectResult{
 				shouldFail: true,
-				err:        ErrIncorrectRepositoryURL,
+				err:        gittypes.ErrIncorrectRepositoryURL,
 			},
 		},
 	}

api/git/service.go

@@ -2,19 +2,15 @@ package git
 
 import (
 	"context"
-	"errors"
-	"log"
 	"strings"
 	"sync"
 	"time"
 
 	lru "github.com/hashicorp/golang-lru"
+	"github.com/rs/zerolog/log"
 )
 
 var (
-	ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
-	ErrAuthenticationFailure  = errors.New("Authentication failed, please ensure that the git credentials are correct.")
-
 	REPOSITORY_CACHE_SIZE = 4
 	REPOSITORY_CACHE_TTL  = 5 * time.Minute
 )
@@ -78,12 +74,12 @@ func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Ser
 		var err error
 		service.repoRefCache, err = lru.New(cacheSize)
 		if err != nil {
-			log.Printf("[DEBUG] [git] [message: failed to create ref cache: %v\n", err)
+			log.Debug().Err(err).Msg("failed to create ref cache")
 		}
 
 		service.repoFileCache, err = lru.New(cacheSize)
 		if err != nil {
-			log.Printf("[DEBUG] [git] [message: failed to create file cache: %v\n", err)
+			log.Debug().Err(err).Msg("failed to create file cache")
 		}
 
 		if cacheTTL > 0 {
@@ -167,9 +163,10 @@ func (service *Service) LatestCommitID(repositoryURL, referenceName, username, p
 
 // ListRefs will list target repository's references without cloning the repository
 func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	refCacheKey := generateCacheKey(repositoryURL, password)
 	if service.cacheEnabled && hardRefresh {
 		// Should remove the cache explicitly, so that the following normal list can show the correct result
-		service.repoRefCache.Remove(repositoryURL)
+		service.repoRefCache.Remove(refCacheKey)
 		// Remove file caches pointed to the same repository
 		for _, fileCacheKey := range service.repoFileCache.Keys() {
 			key, ok := fileCacheKey.(string)
@@ -183,7 +180,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 
 	if service.repoRefCache != nil {
 		// Lookup the refs cache first
-		cache, ok := service.repoRefCache.Get(repositoryURL)
+		cache, ok := service.repoRefCache.Get(refCacheKey)
 		if ok {
 			refs, success := cache.([]string)
 			if success {
@@ -215,7 +212,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
 	}
 
 	if service.cacheEnabled && service.repoRefCache != nil {
-		service.repoRefCache.Add(options.repositoryUrl, refs)
+		service.repoRefCache.Add(refCacheKey, refs)
 	}
 	return refs, nil
 }

api/git/types/types.go

@@ -1,5 +1,12 @@
 package gittypes
 
+import "errors"
+
+var (
+	ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
+	ErrAuthenticationFailure  = errors.New("Authentication failed, please ensure that the git credentials are correct.")
+)
+
 // RepoConfig represents a configuration for a repo
 type RepoConfig struct {
 	// The repo url

api/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.1.8
 	github.com/go-playground/validator/v10 v10.10.1
 	github.com/gofrs/uuid v4.0.0+incompatible
-	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/golang-jwt/jwt/v4 v4.2.0
 	github.com/google/go-cmp v0.5.8
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.7.3
@@ -31,15 +31,15 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021
 	github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a
-	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
+	github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9
 	github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a
 	github.com/rkl-/digest v0.0.0-20180419075440-8316caa4a777
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.28.0
-	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/viney-shih/go-lock v1.1.1
 	go.etcd.io/bbolt v1.3.6
@@ -48,7 +48,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.22.5
 	k8s.io/apimachinery v0.22.5
 	k8s.io/client-go v0.22.5
@@ -105,6 +105,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect
 	github.com/x448/float16 v0.8.4 // indirect

api/go.sum

@@ -173,8 +173,8 @@ github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPh
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
+github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -352,6 +352,8 @@ github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrB
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw=
 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -363,8 +365,10 @@ github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021 h
 github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a h1:B0z3skIMT+OwVNJPQhKp52X+9OWW6A9n5UWig3lHBJk=
 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
-github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=
-github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
+github.com/portainer/libhelm v0.0.0-20221017001602-b619e0ddf8fc h1:Ms8ZDGvSydX7aoCfu06Rk8II2N4tlX3cn7whtGC5fDA=
+github.com/portainer/libhelm v0.0.0-20221017001602-b619e0ddf8fc/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
+github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9 h1:PMw+45hocpYsMQ7WLIlzsf4854BhDM8C631kr0DVJtc=
+github.com/portainer/libhelm v0.0.0-20221018213433-5ad83b50dbc9/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
 github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a h1:BJ5V4EDNhg3ImYbmXnGS8vrMhq6rzsEneIXyJh0g4dc=
 github.com/portainer/libhttp v0.0.0-20220916153711-5d61e12f4b0a/go.mod h1:ckuHnoLA5kLuE5WkvPBXmrw63LUMdSH4aX71QRi9y10=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -747,6 +751,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=

api/hostmanagement/openamt/authorization.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
@@ -33,7 +33,7 @@ func (service *Service) Authorization(configuration portainer.OpenAMTConfigurati
 	if err != nil {
 		return "", err
 	}
-	responseBody, readErr := ioutil.ReadAll(response.Body)
+	responseBody, readErr := io.ReadAll(response.Body)
 	if readErr != nil {
 		return "", readErr
 	}

api/hostmanagement/openamt/deviceFeatures.go

@@ -21,9 +21,6 @@ func (service *Service) enableDeviceFeatures(configuration portainer.OpenAMTConf
 	jsonValue, _ := json.Marshal(payload)
 
 	_, err := service.executeSaveRequest(http.MethodPost, url, configuration.MPSToken, jsonValue)
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }

api/hostmanagement/openamt/openamt.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"time"
 
@@ -83,11 +83,8 @@ func (service *Service) Configure(configuration portainer.OpenAMTConfiguration)
 	}
 
 	_, err = service.createOrUpdateDomain(configuration)
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }
 
 func (service *Service) executeSaveRequest(method string, url string, token string, payload []byte) ([]byte, error) {
@@ -102,7 +99,7 @@ func (service *Service) executeSaveRequest(method string, url string, token stri
 	if err != nil {
 		return nil, err
 	}
-	responseBody, readErr := ioutil.ReadAll(response.Body)
+	responseBody, readErr := io.ReadAll(response.Body)
 	if readErr != nil {
 		return nil, readErr
 	}
@@ -131,7 +128,7 @@ func (service *Service) executeGetRequest(url string, token string) ([]byte, err
 	if err != nil {
 		return nil, err
 	}
-	responseBody, readErr := ioutil.ReadAll(response.Body)
+	responseBody, readErr := io.ReadAll(response.Body)
 	if readErr != nil {
 		return nil, readErr
 	}
@@ -229,12 +226,7 @@ func (service *Service) ExecuteDeviceAction(configuration portainer.OpenAMTConfi
 	}
 	configuration.MPSToken = token
 
-	err = service.executeDeviceAction(configuration, deviceGUID, int(parsedAction))
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return service.executeDeviceAction(configuration, deviceGUID, int(parsedAction))
 }
 
 func (service *Service) EnableDeviceFeatures(configuration portainer.OpenAMTConfiguration, deviceGUID string, features portainer.OpenAMTDeviceEnabledFeatures) (string, error) {

api/http/client/client.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -96,7 +96,7 @@ func Get(url string, timeout int) ([]byte, error) {
 		return nil, errInvalidResponseStatus
 	}
 
-	body, err := ioutil.ReadAll(response.Body)
+	body, err := io.ReadAll(response.Body)
 	if err != nil {
 		return nil, err
 	}

api/http/handler/auth/authenticate.go

@@ -1,7 +1,6 @@
 package auth
 
 import (
-	"errors"
 	"net/http"
 	"strings"
 
@@ -13,6 +12,7 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 
 	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
 

api/http/handler/auth/authenticate_oauth.go

@@ -22,6 +22,7 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Code) {
 		return errors.New("Invalid OAuth authorization code")
 	}
+
 	return nil
 }
 

api/http/handler/auth/handler.go

@@ -3,13 +3,14 @@ package auth
 import (
 	"net/http"
 
-	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/http/security"
+
+	"github.com/gorilla/mux"
 )
 
 // Handler is the HTTP handler used to handle authentication operations.

api/http/handler/backup/backup_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -38,7 +37,7 @@ func listFiles(dir string) []string {
 
 func contains(t *testing.T, list []string, path string) {
 	assert.Contains(t, list, path)
-	copyContent, _ := ioutil.ReadFile(path)
+	copyContent, _ := os.ReadFile(path)
 	assert.Equal(t, "content\n", string(copyContent))
 }
 
@@ -56,8 +55,9 @@ func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T)
 	body, _ := io.ReadAll(response.Body)
 
 	tmpdir := t.TempDir()
+
 	archivePath := filepath.Join(tmpdir, "archive.tar.gz")
-	err := ioutil.WriteFile(archivePath, body, 0600)
+	err := os.WriteFile(archivePath, body, 0600)
 	if err != nil {
 		t.Fatal("Failed to save downloaded .tar.gz archive: ", err)
 	}
@@ -91,6 +91,7 @@ func Test_backupHandlerWithPassword_shouldCreateEncryptedATarballArchive(t *test
 	body, _ := io.ReadAll(response.Body)
 
 	tmpdir := t.TempDir()
+
 	dr, err := crypto.AesDecrypt(bytes.NewReader(body), []byte("secret"))
 	if err != nil {
 		t.Fatal("Failed to decrypt archive")

api/http/handler/backup/handler.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/adminmonitor"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/demo"
@@ -71,11 +70,3 @@ func adminAccess(next http.Handler) http.Handler {
 		next.ServeHTTP(w, r)
 	})
 }
-
-func systemWasInitialized(dataStore dataservices.DataStore) (bool, error) {
-	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
-	if err != nil {
-		return false, err
-	}
-	return len(users) > 0, nil
-}

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,20 +3,21 @@ package customtemplates
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"regexp"
 	"strconv"
 
+	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
-
-	"github.com/asaskevich/govalidator"
 	"github.com/rs/zerolog/log"
 )
 
@@ -141,12 +142,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
-	err := validateVariablesDefinitions(payload.Variables)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return validateVariablesDefinitions(payload.Variables)
 }
 
 func isValidNote(note string) bool {
@@ -250,12 +246,7 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
-	err := validateVariablesDefinitions(payload.Variables)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return validateVariablesDefinitions(payload.Variables)
 }
 
 func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (*portainer.CustomTemplate, error) {
@@ -290,6 +281,9 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 
 	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
 	if err != nil {
+		if err == gittypes.ErrAuthenticationFailure {
+			return nil, fmt.Errorf("invalid git credential")
+		}
 		return nil, err
 	}
 
@@ -386,16 +380,13 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 	payload.FileContent = composeFileContent
 
 	varsString, _ := request.RetrieveMultiPartFormValue(r, "Variables", true)
-	err = json.Unmarshal([]byte(varsString), &payload.Variables)
-	if err != nil {
-		return errors.New("Invalid variables. Ensure that the variables are valid JSON")
+	if varsString != "" {
+		err = json.Unmarshal([]byte(varsString), &payload.Variables)
+		if err != nil {
+			return errors.New("Invalid variables. Ensure that the variables are valid JSON")
+		}
+		return validateVariablesDefinitions(payload.Variables)
 	}
-
-	err = validateVariablesDefinitions(payload.Variables)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

api/http/handler/customtemplates/customtemplate_update.go

@@ -55,12 +55,7 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
-	err := validateVariablesDefinitions(payload.Variables)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return validateVariablesDefinitions(payload.Variables)
 }
 
 // @id CustomTemplateUpdate

api/http/handler/customtemplates/handler.go

@@ -8,7 +8,6 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 // Handler is the HTTP handler used to handle environment(endpoint) group operations.
@@ -42,20 +41,3 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 func userCanEditTemplate(customTemplate *portainer.CustomTemplate, securityContext *security.RestrictedRequestContext) bool {
 	return securityContext.IsAdmin || customTemplate.CreatedByUserID == securityContext.UserID
 }
-
-func userCanAccessTemplate(customTemplate portainer.CustomTemplate, securityContext *security.RestrictedRequestContext, resourceControl *portainer.ResourceControl) bool {
-	if securityContext.IsAdmin || customTemplate.CreatedByUserID == securityContext.UserID {
-		return true
-	}
-
-	userTeamIDs := make([]portainer.TeamID, 0)
-	for _, membership := range securityContext.UserMemberships {
-		userTeamIDs = append(userTeamIDs, membership.TeamID)
-	}
-
-	if resourceControl != nil && authorization.UserCanAccessResource(securityContext.UserID, userTeamIDs, resourceControl) {
-		return true
-	}
-
-	return false
-}

api/http/handler/edgegroups/associated_endpoints.go

@@ -1,7 +1,7 @@
 package edgegroups
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 )
 
 type endpointSetType map[portainer.EndpointID]bool

api/http/handler/edgegroups/edgegroup_create.go

@@ -4,11 +4,13 @@ import (
 	"errors"
 	"net/http"
 
-	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+
+	"github.com/asaskevich/govalidator"
 )
 
 type edgeGroupCreatePayload struct {
@@ -81,7 +83,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 				return httperror.InternalServerError("Unable to retrieve environment from the database", err)
 			}
 
-			if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+			if endpointutils.IsEdgeEndpoint(endpoint) {
 				endpointIDs = append(endpointIDs, endpoint.ID)
 			}
 		}

api/http/handler/edgegroups/edgegroup_delete.go

@@ -53,5 +53,4 @@ func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request)
 	}
 
 	return response.Empty(w)
-
 }

api/http/handler/edgegroups/edgegroup_list.go

@@ -81,7 +81,7 @@ func getEndpointTypes(endpointService dataservices.EndpointService, endpointIds
 	for _, endpointID := range endpointIds {
 		endpoint, err := endpointService.Endpoint(endpointID)
 		if err != nil {
-			return nil, fmt.Errorf("failed fetching endpoint: %w", err)
+			return nil, fmt.Errorf("failed fetching environment: %w", err)
 		}
 
 		typeSet[endpoint.Type] = true

api/http/handler/edgegroups/edgegroup_update.go

@@ -4,12 +4,14 @@ import (
 	"errors"
 	"net/http"
 
-	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+
+	"github.com/asaskevich/govalidator"
 )
 
 type edgeGroupUpdatePayload struct {
@@ -102,7 +104,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 				return httperror.InternalServerError("Unable to retrieve environment from the database", err)
 			}
 
-			if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+			if endpointutils.IsEdgeEndpoint(endpoint) {
 				endpointIDs = append(endpointIDs, endpoint.ID)
 			}
 		}

api/http/handler/edgejobs/edgejob_create.go

@@ -12,6 +12,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
 // @id EdgeJobCreate
@@ -200,7 +201,7 @@ func (handler *Handler) addAndPersistEdgeJob(edgeJob *portainer.EdgeJob, file []
 			return err
 		}
 
-		if endpoint.Type != portainer.EdgeAgentOnDockerEnvironment && endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
+		if !endpointutils.IsEdgeEndpoint(endpoint) {
 			delete(edgeJob.Endpoints, ID)
 		}
 	}

api/http/handler/edgejobs/edgejob_update.go

@@ -5,11 +5,12 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/asaskevich/govalidator"
 )
 
 type edgeJobUpdatePayload struct {

api/http/handler/edgestacks/edgestack_create.go

@@ -1,14 +1,12 @@
 package edgestacks
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -16,6 +14,9 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/internal/edge"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 )
 
 // @id EdgeStackCreate
@@ -271,7 +272,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 
 	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
+		return nil, fmt.Errorf("Unable to update environment relations: %w", err)
 	}
 
 	err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
@@ -378,7 +379,7 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 
 	err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
+		return nil, fmt.Errorf("Unable to update environment relations: %w", err)
 	}
 
 	err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
@@ -408,14 +409,14 @@ func updateEndpointRelations(endpointRelationService dataservices.EndpointRelati
 	for _, endpointID := range relatedEndpointIds {
 		relation, err := endpointRelationService.EndpointRelation(endpointID)
 		if err != nil {
-			return fmt.Errorf("unable to find endpoint relation in database: %w", err)
+			return fmt.Errorf("unable to find environment relation in database: %w", err)
 		}
 
 		relation.EdgeStacks[edgeStackID] = true
 
 		err = endpointRelationService.UpdateEndpointRelation(endpointID, relation)
 		if err != nil {
-			return fmt.Errorf("unable to persist endpoint relation in database: %w", err)
+			return fmt.Errorf("unable to persist environment relation in database: %w", err)
 		}
 	}
 

api/http/handler/edgestacks/edgestack_create_test.go

@@ -5,6 +5,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+
 	"github.com/stretchr/testify/assert"
 )
 

api/http/handler/edgestacks/edgestack_delete.go

@@ -42,7 +42,7 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 
 	relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
 	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve environments relations config from database", err)
+		return httperror.InternalServerError("Unable to find environment relations in database", err)
 	}
 
 	relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)

api/http/handler/edgestacks/edgestack_file.go

@@ -46,7 +46,7 @@ func (handler *Handler) edgeStackFile(w http.ResponseWriter, r *http.Request) *h
 
 	stackFileContent, err := handler.FileService.GetFileContent(stack.ProjectPath, fileName)
 	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve Compose file from disk", err)
+		return httperror.InternalServerError("Unable to retrieve stack file from disk", err)
 	}
 
 	return response.JSON(w, &stackFileResponse{StackFileContent: string(stackFileContent)})

api/http/handler/edgestacks/edgestack_test.go

@@ -3,7 +3,6 @@ package edgestacks
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -19,6 +18,8 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/jwt"
+
+	"github.com/pkg/errors"
 )
 
 type gitService struct {

api/http/handler/edgestacks/edgestack_update.go

@@ -2,7 +2,6 @@ package edgestacks
 
 import (
 	"errors"
-	"github.com/portainer/portainer/api/internal/endpointutils"
 	"net/http"
 	"strconv"
 
@@ -12,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
 type updateEdgeStackPayload struct {

api/http/handler/edgeupdateschedules/edgeupdateschedule_create.go

@@ -31,14 +31,14 @@ func (payload *createPayload) Validate(r *http.Request) error {
 		return errors.New("Required to choose at least one group")
 	}
 
-	if payload.Type != edgetypes.UpdateScheduleRollback && payload.Type != edgetypes.UpdateScheduleUpdate {
-		return errors.New("Invalid schedule type")
-	}
-
 	if len(payload.Environments) == 0 {
 		return errors.New("No Environment is scheduled for update")
 	}
 
+	if payload.Type != edgetypes.UpdateScheduleRollback && payload.Type != edgetypes.UpdateScheduleUpdate {
+		return errors.New("Invalid schedule type")
+	}
+
 	if payload.Time < time.Now().Unix() {
 		return errors.New("Invalid time")
 	}

api/http/handler/endpointedge/handler.go

@@ -3,14 +3,13 @@ package endpointedge
 import (
 	"net/http"
 
-	"github.com/portainer/portainer/api/http/middlewares"
-
 	httperror "github.com/portainer/libhttp/error"
-
-	"github.com/gorilla/mux"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/middlewares"
 	"github.com/portainer/portainer/api/http/security"
+
+	"github.com/gorilla/mux"
 )
 
 // Handler is the HTTP handler used to handle edge environment(endpoint) operations.

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -91,15 +91,13 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		tag, err := handler.DataStore.Tag().Tag(tagID)
-		if err != nil {
-			return httperror.InternalServerError("Unable to retrieve tag from the database", err)
-		}
+		handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+			tag.EndpointGroups[endpointGroup.ID] = true
+		})
 
-		tag.EndpointGroups[endpointGroup.ID] = true
-
-		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
-		if err != nil {
+		if handler.DataStore.IsErrObjectNotFound(err) {
+			return httperror.InternalServerError("Unable to find a tag inside the database", err)
+		} else if err != nil {
 			return httperror.InternalServerError("Unable to persist tag changes inside the database", err)
 		}
 	}

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -66,15 +66,13 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		tag, err := handler.DataStore.Tag().Tag(tagID)
-		if err != nil {
-			return httperror.InternalServerError("Unable to retrieve tag from the database", err)
-		}
+		handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+			delete(tag.EndpointGroups, endpointGroup.ID)
+		})
 
-		delete(tag.EndpointGroups, endpointGroup.ID)
-
-		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
-		if err != nil {
+		if handler.DataStore.IsErrObjectNotFound(err) {
+			return httperror.InternalServerError("Unable to find a tag inside the database", err)
+		} else if err != nil {
 			return httperror.InternalServerError("Unable to persist tag changes inside the database", err)
 		}
 	}

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -81,28 +81,26 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 			removeTags := tag.Difference(endpointGroupTagSet, payloadTagSet)
 
 			for tagID := range removeTags {
-				tag, err := handler.DataStore.Tag().Tag(tagID)
-				if err != nil {
+				handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+					delete(tag.EndpointGroups, endpointGroup.ID)
+				})
+
+				if handler.DataStore.IsErrObjectNotFound(err) {
 					return httperror.InternalServerError("Unable to find a tag inside the database", err)
-				}
-				delete(tag.EndpointGroups, endpointGroup.ID)
-				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
-				if err != nil {
+				} else if err != nil {
 					return httperror.InternalServerError("Unable to persist tag changes inside the database", err)
 				}
 			}
 
 			endpointGroup.TagIDs = payload.TagIDs
 			for _, tagID := range payload.TagIDs {
-				tag, err := handler.DataStore.Tag().Tag(tagID)
-				if err != nil {
+				handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+					tag.EndpointGroups[endpointGroup.ID] = true
+				})
+
+				if handler.DataStore.IsErrObjectNotFound(err) {
 					return httperror.InternalServerError("Unable to find a tag inside the database", err)
-				}
-
-				tag.EndpointGroups[endpointGroup.ID] = true
-
-				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
-				if err != nil {
+				} else if err != nil {
 					return httperror.InternalServerError("Unable to persist tag changes inside the database", err)
 				}
 			}

api/http/handler/endpointproxy/proxy_docker.go

@@ -2,13 +2,13 @@ package endpointproxy
 
 import (
 	"errors"
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	portainer "github.com/portainer/portainer/api"
+	"net/http"
 	"strconv"
 	"strings"
 
-	"net/http"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	portainer "github.com/portainer/portainer/api"
 )
 
 func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -3,12 +3,12 @@ package endpointproxy
 import (
 	"errors"
 	"fmt"
+	"net/http"
+	"strings"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
-	"strings"
-
-	"net/http"
 )
 
 func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {

api/http/handler/endpoints/endpoint_create.go

@@ -530,14 +530,9 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.
 	}
 
 	for _, tagID := range endpoint.TagIDs {
-		tag, err := handler.DataStore.Tag().Tag(tagID)
-		if err != nil {
-			return err
-		}
-
-		tag.Endpoints[endpoint.ID] = true
-
-		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
+		err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+			tag.Endpoints[endpoint.ID] = true
+		})
 		if err != nil {
 			return err
 		}

api/http/handler/endpoints/endpoint_delete.go

@@ -62,15 +62,13 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 	}
 
 	for _, tagID := range endpoint.TagIDs {
-		tag, err := handler.DataStore.Tag().Tag(tagID)
-		if err != nil {
+		err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
+			delete(tag.Endpoints, endpoint.ID)
+		})
+
+		if handler.DataStore.IsErrObjectNotFound(err) {
 			return httperror.NotFound("Unable to find tag inside the database", err)
-		}
-
-		delete(tag.Endpoints, endpoint.ID)
-
-		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
-		if err != nil {
+		} else if err != nil {
 			return httperror.InternalServerError("Unable to persist tag relation inside the database", err)
 		}
 	}

api/http/handler/endpoints/endpoint_list_test.go

@@ -102,7 +102,6 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
 			is.ElementsMatch(test.expected, respIds)
 		})
 	}
-
 }
 
 func Test_endpointList_edgeDeviceFilter(t *testing.T) {

api/http/handler/endpoints/endpoint_snapshots.go

@@ -32,6 +32,10 @@ func (handler *Handler) endpointSnapshots(w http.ResponseWriter, r *http.Request
 			continue
 		}
 
+		if endpoint.URL == "" {
+			continue
+		}
+
 		snapshotError := handler.SnapshotService.SnapshotEndpoint(&endpoint)
 
 		latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)