fix(frontend): populate data between simple and advanced modes (#4503)

* fix(k8s/application): validate load balancer ports inputs

* fix(k8s/application): allow user to only change the protocol on the first port mapping

.babelrc

@@ -5,8 +5,7 @@
       "@babel/preset-env",
       {
         "modules": false,
-        "useBuiltIns": "entry",
-        "corejs": "2"
+        "useBuiltIns": "entry"
       }
     ]
   ]

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -9,7 +9,7 @@ Thanks for reporting a bug for Portainer !
 
 You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/.
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this
@@ -40,6 +40,7 @@ You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#ho
 
 - Portainer version:
 - Docker version (managed by Portainer):
+- Kubernetes version (managed by Portainer):
 - Platform (windows/linux):
 - Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
 - Browser:

.github/ISSUE_TEMPLATE/Custom.md

@@ -1,17 +1,16 @@
----
-name: Question
-about: Ask us a question about Portainer usage or deployment
-
----
-
-<!--
-
-You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
-
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
-
-Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
--->
-
-**Question**:
-How can I deploy Portainer on... ?
+---
+name: Question
+about: Ask us a question about Portainer usage or deployment
+---
+
+<!--
+
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
+
+Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
+-->
+
+**Question**:
+How can I deploy Portainer on... ?

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -1,31 +1,30 @@
----
-name: Feature request
-about: Suggest a feature/enhancement that should be added in Portainer
-
----
-
-<!--
-
-Thanks for opening a feature request for Portainer !
-
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
-
-Before opening a new issue, make sure that we do not have any duplicates
-already open. You can ensure this by searching the issue list for this
-repository. If there is a duplicate, please close your issue and add a comment
-to the existing issue instead.
-
-Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
--->
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.
+---
+name: Feature request
+about: Suggest a feature/enhancement that should be added in Portainer
+---
+
+<!--
+
+Thanks for opening a feature request for Portainer !
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
+
+Before opening a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
+-->
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/stale.yml

@@ -15,6 +15,7 @@ issues:
     - kind/question
     - kind/style
     - kind/workaround
+    - kind/refactor
     - bug/need-confirmation
     - bug/confirmed
     - status/discuss

.gitignore

@@ -8,10 +8,3 @@ api/cmd/portainer/portainer*
 **/.vscode/tasks.json
 
 .eslintcache
-.idea
-test/e2e/cypress/screenshots
-*.db
-*.log
-__debug_bin
-api/docs
-.env

.vscode.example/launch.json

@@ -1,19 +0,0 @@
-{
-  // Use IntelliSense to learn about possible attributes.
-  // Hover to view descriptions of existing attributes.
-  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Launch",
-      "type": "go",
-      "request": "launch",
-      "mode": "debug",
-      "program": "${workspaceRoot}/api/cmd/portainer/main.go",
-      "cwd": "${workspaceRoot}",
-      "env": {},
-      "showLog": true,
-      "args": ["--data", "${env:HOME}/portainer-data", "--assets", "${workspaceRoot}/dist"]
-    }
-  ]
-}

.vscode/portainer.code-snippets

@@ -21,11 +21,11 @@
     "description": "Dummy Angularjs Component",
     "body": [
       "import angular from 'angular';",
-      "import controller from './${TM_FILENAME_BASE}Controller'",
+      "import ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/}Controller from './${TM_FILENAME_BASE}Controller'",
       "",
       "angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').component('$TM_FILENAME_BASE', {",
       "  templateUrl: './$TM_FILENAME_BASE.html',",
-      "  controller,",
+      "  controller: ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/}Controller,",
       "});",
       ""
     ]
@@ -44,6 +44,25 @@
     ],
     "description": "Dummy ES6+ controller"
   },
+  "Model": {
+    "scope": "javascript",
+    "prefix": "mymodel",
+    "description": "Dummy ES6+ model",
+    "body": [
+      "/**",
+      " * $1 Model",
+      " */",
+      "const _$1 = Object.freeze({",
+      "  $0",
+      "});",
+      "",
+      "export class $1 {",
+      "  constructor() {",
+      "    Object.assign(this, JSON.parse(JSON.stringify(_$1)));",
+      "  }",
+      "}"
+    ]
+  },
   "Service": {
     "scope": "javascript",
     "prefix": "myservice",
@@ -139,29 +158,5 @@
       "export default $1;",
       "angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').service('$1', $1);"
     ]
-  },
-  "swagger-api-doc": {
-    "prefix": "swapi",
-    "scope": "go",
-    "description": "Snippet for a api doc",
-    "body": [
-      "// @id ",
-      "// @summary ",
-      "// @description ",
-      "// @description **Access policy**: ",
-      "// @tags ",
-      "// @security jwt",
-      "// @accept json",
-      "// @produce json",
-      "// @param id path int true \"identifier\"",
-      "// @param body body Object true \"details\"",
-      "// @success 200 {object} portainer. \"Success\"",
-      "// @success 204 \"Success\"",
-      "// @failure 400 \"Invalid request\"",
-      "// @failure 403 \"Permission denied\"",
-      "// @failure 404 \" not found\"",
-      "// @failure 500 \"Server error\"",
-      "// @router /{id} [get]"
-    ]
   }
 }

CONTRIBUTING.md

@@ -94,36 +94,3 @@ $ yarn start
 Portainer can now be accessed at <http://localhost:9000>.
 
 Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
-
-## Adding api docs
-
-When adding a new resource (or a route handler), we should add a new tag to api/http/handler/handler.go#L136 like this:
-
-```
-// @tag.name <Name of resource>
-// @tag.description a short description
-```
-
-When adding a new route to an existing handler use the following as a template (you can use `swapi` snippet if you're using vscode):
-
-```
-// @id
-// @summary
-// @description
-// @description **Access policy**:
-// @tags
-// @security jwt
-// @accept json
-// @produce json
-// @param id path int true "identifier"
-// @param body body Object true "details"
-// @success 200 {object} portainer. "Success"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 " not found"
-// @failure 500 "Server error"
-// @router /{id} [get]
-```
-
-explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]

README.md

@@ -30,6 +30,7 @@ Unlike the public demo, the playground sessions are deleted after 4 hours. Apart
 
 - [Deploy Portainer](https://www.portainer.io/installation/)
 - [Documentation](https://documentation.portainer.io)
+- [Building Portainer](https://documentation.portainer.io/contributing/instructions/)
 
 ## Getting help
 
@@ -44,7 +45,7 @@ For community support: You can find more information about Portainer's community
 ## Reporting bugs and contributing
 
 - Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
-- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://www.portainer.io/documentation/how-to-contribute/) to build it locally and make a pull request. We need all the help we can get!
+- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://documentation.portainer.io/contributing/instructions/) to build it locally and make a pull request. We need all the help we can get!
 
 ## Security
 

api/adminmonitor/admin_monitor.go

@@ -1,69 +0,0 @@
-package adminmonitor
-
-import (
-	"context"
-	"log"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-var logFatalf = log.Fatalf
-
-type Monitor struct {
-	timeout          time.Duration
-	datastore        portainer.DataStore
-	shutdownCtx      context.Context
-	cancellationFunc context.CancelFunc
-}
-
-// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
-func New(timeout time.Duration, datastore portainer.DataStore, shutdownCtx context.Context) *Monitor {
-	return &Monitor{
-		timeout:     timeout,
-		datastore:   datastore,
-		shutdownCtx: shutdownCtx,
-	}
-}
-
-// Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
-func (m *Monitor) Start() {
-	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
-	m.cancellationFunc = cancellationFunc
-
-	go func() {
-		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
-		select {
-		case <-time.After(m.timeout):
-			initialized, err := m.WasInitialized()
-			if err != nil {
-				logFatalf("failed getting admin user: %s", err)
-			}
-			if !initialized {
-				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
-			}
-		case <-cancellationCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
-		case <-m.shutdownCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
-		}
-	}()
-}
-
-// Stop stops monitor. Safe to call even if monitor wasn't started.
-func (m *Monitor) Stop() {
-	if m.cancellationFunc == nil {
-		return
-	}
-	m.cancellationFunc()
-	m.cancellationFunc = nil
-}
-
-// WasInitialized is a system initialization check
-func (m *Monitor) WasInitialized() (bool, error) {
-	users, err := m.datastore.User().UsersByRole(portainer.AdministratorRole)
-	if err != nil {
-		return false, err
-	}
-	return len(users) > 0, nil
-}

api/adminmonitor/admin_monitor_test.go

@@ -1,50 +0,0 @@
-package adminmonitor
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	i "github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_stopWithoutStarting(t *testing.T) {
-	monitor := New(1*time.Minute, nil, nil)
-	monitor.Stop()
-}
-
-func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
-	monitor := New(1*time.Minute, nil, nil)
-	monitor.Stop()
-	monitor.Stop()
-}
-
-func Test_canStopStartedMonitor(t *testing.T) {
-	monitor := New(1*time.Minute, nil, context.Background())
-	monitor.Start()
-	assert.NotNil(t, monitor.cancellationFunc, "cancellation function is missing in started monitor")
-
-	monitor.Stop()
-	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
-}
-
-func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
-	timeout := 10 * time.Millisecond
-
-	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
-
-	var fataled bool
-	origLogFatalf := logFatalf
-	logFatalf = func(s string, v ...interface{}) { fataled = true }
-	defer func() {
-		logFatalf = origLogFatalf
-	}()
-
-	monitor := New(timeout, datastore, context.Background())
-	monitor.Start()
-	<-time.After(2 * timeout)
-
-	assert.True(t, fataled, "monitor should been timeout and fatal")
-}

api/api-description.md

@@ -1,53 +0,0 @@
-Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
-Examples are available at https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8
-You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
-
-# Authentication
-
-Most of the API endpoints require to be authenticated as well as some level of authorization to be used.
-Portainer API uses JSON Web Token to manage authentication and thus requires you to provide a token in the **Authorization** header of each request
-with the **Bearer** authentication mechanism.
-
-Example:
-
-```
-Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
-```
-
-# Security
-
-Each API endpoint has an associated access policy, it is documented in the description of each endpoint.
-
-Different access policies are available:
-
-- Public access
-- Authenticated access
-- Restricted access
-- Administrator access
-
-### Public access
-
-No authentication is required to access the endpoints with this access policy.
-
-### Authenticated access
-
-Authentication is required to access the endpoints with this access policy.
-
-### Restricted access
-
-Authentication is required to access the endpoints with this access policy.
-Extra-checks might be added to ensure access to the resource is granted. Returned data might also be filtered.
-
-### Administrator access
-
-Authentication as well as an administrator role are required to access the endpoints with this access policy.
-
-# Execute Docker requests
-
-Portainer **DO NOT** expose specific endpoints to manage your Docker resources (create a container, remove a volume, etc...).
-
-Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you can execute Docker requests **via** the Portainer HTTP API.
-
-To do so, you can use the `/endpoints/{id}/docker` Portainer API endpoint (which is not documented below due to Swagger limitations). This endpoint has a restricted access policy so you still need to be authenticated to be able to query this endpoint. Any query on this endpoint will be proxied to the Docker API of the associated endpoint (requests and responses objects are the same as documented in the Docker API).
-
-**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8).

api/archive/targz.go

@@ -1,119 +0,0 @@
-package archive
-
-import (
-	"archive/tar"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// TarGzDir creates a tar.gz archive and returns it's path.
-// abosolutePath should be an absolute path to a directory.
-// Archive name will be <directoryName>.tar.gz and will be placed next to the directory.
-func TarGzDir(absolutePath string) (string, error) {
-	targzPath := filepath.Join(absolutePath, fmt.Sprintf("%s.tar.gz", filepath.Base(absolutePath)))
-	outFile, err := os.Create(targzPath)
-	if err != nil {
-		return "", err
-	}
-	defer outFile.Close()
-
-	zipWriter := gzip.NewWriter(outFile)
-	defer zipWriter.Close()
-	tarWriter := tar.NewWriter(zipWriter)
-	defer tarWriter.Close()
-
-	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if path == targzPath {
-			return nil // skip archive file
-		}
-
-		pathInArchive := filepath.Clean(strings.TrimPrefix(path, absolutePath))
-		if pathInArchive == "" {
-			return nil // skip root dir
-		}
-
-		return addToArchive(tarWriter, pathInArchive, path, info)
-	})
-
-	return targzPath, err
-}
-
-func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info os.FileInfo) error {
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-
-	header.Name = pathInArchive // use relative paths in archive
-
-	err = tarWriter.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
-	if info.IsDir() {
-		return nil
-	}
-
-	file, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(tarWriter, file)
-	return err
-}
-
-// ExtractTarGz reads a .tar.gz archive from the reader and extracts it into outputDirPath directory
-func ExtractTarGz(r io.Reader, outputDirPath string) error {
-	zipReader, err := gzip.NewReader(r)
-	if err != nil {
-		return err
-	}
-	defer zipReader.Close()
-
-	tarReader := tar.NewReader(zipReader)
-
-	for {
-		header, err := tarReader.Next()
-
-		if err == io.EOF {
-			break
-		}
-
-		if err != nil {
-			return err
-		}
-
-		switch header.Typeflag {
-		case tar.TypeDir:
-			// skip, dir will be created with a file
-		case tar.TypeReg:
-			p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
-			if err := os.MkdirAll(filepath.Dir(p), 0744); err != nil {
-				return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
-			}
-			outFile, err := os.Create(p)
-			if err != nil {
-				return fmt.Errorf("Failed to create file %s", header.Name)
-			}
-			if _, err := io.Copy(outFile, tarReader); err != nil {
-				return fmt.Errorf("Failed to extract file %s", header.Name)
-			}
-			outFile.Close()
-		default:
-			return fmt.Errorf("Tar: uknown type: %v in %s",
-				header.Typeflag,
-				header.Name)
-		}
-	}
-
-	return nil
-}

api/archive/targz_test.go

@@ -1,98 +0,0 @@
-package archive
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func Test_shouldCreateArhive(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
-	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
-
-	gzPath, err := TarGzDir(tmpdir)
-	assert.Nil(t, err)
-	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
-
-	extractionDir, _ := ioutil.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
-	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
-	err = cmd.Run()
-	if err != nil {
-		t.Fatal("Failed to extract archive: ", err)
-	}
-	extractedFiles := listFiles(extractionDir)
-
-	wasExtracted := func(p string) {
-		fullpath := path.Join(extractionDir, p)
-		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
-		assert.Equal(t, content, copyContent)
-	}
-
-	wasExtracted("outer")
-	wasExtracted("dir/inner")
-	wasExtracted("dir/.dotfile")
-}
-
-func Test_shouldCreateArhiveXXXXX(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
-	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
-
-	gzPath, err := TarGzDir(tmpdir)
-	assert.Nil(t, err)
-	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
-
-	extractionDir, _ := ioutil.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
-	r, _ := os.Open(gzPath)
-	ExtractTarGz(r, extractionDir)
-	if err != nil {
-		t.Fatal("Failed to extract archive: ", err)
-	}
-	extractedFiles := listFiles(extractionDir)
-
-	wasExtracted := func(p string) {
-		fullpath := path.Join(extractionDir, p)
-		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
-		assert.Equal(t, content, copyContent)
-	}
-
-	wasExtracted("outer")
-	wasExtracted("dir/inner")
-	wasExtracted("dir/.dotfile")
-}

api/archive/zip.go

@@ -3,13 +3,10 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
-	"fmt"
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -55,60 +52,3 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 
 	return outFile.Close()
 }
-
-// UnzipFile will decompress a zip archive, moving all files and folders
-// within the zip file (parameter 1) to an output directory (parameter 2).
-func UnzipFile(src string, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer r.Close()
-
-	for _, f := range r.File {
-		p := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
-		if !strings.HasPrefix(p, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("%s: illegal file path", p)
-		}
-
-		if f.FileInfo().IsDir() {
-			// Make Folder
-			os.MkdirAll(p, os.ModePerm)
-			continue
-		}
-
-		err = unzipFile(f, p)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func unzipFile(f *zip.File, p string) error {
-	// Make File
-	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
-		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
-	}
-	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
-	}
-	defer outFile.Close()
-	rc, err := f.Open()
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
-	}
-	defer rc.Close()
-
-	_, err = io.Copy(outFile, rc)
-
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
-	}
-
-	return nil
-}

api/archive/zip_test.go

@@ -1,32 +0,0 @@
-package archive
-
-import (
-	"github.com/stretchr/testify/assert"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestUnzipFile(t *testing.T) {
-	dir, err := ioutil.TempDir("", "unzip-test-")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dir)
-	/*
-		Archive structure.
-		├── 0
-		│	├── 1
-		│ 	│   └── 2.txt
-		│   └── 1.txt
-		└── 0.txt
-	*/
-
-	err = UnzipFile("./testdata/sample_archive.zip", dir)
-
-	assert.NoError(t, err)
-	archiveDir := dir + "/sample_archive"
-	assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
-	assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
-	assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
-
-}

api/backup/backup.go

@@ -1,112 +0,0 @@
-package backup
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/http/offlinegate"
-	"github.com/portainer/portainer/api/s3"
-)
-
-const rwxr__r__ os.FileMode = 0744
-
-var filesToBackup = []string{"compose", "config.json", "custom_templates", "edge_jobs", "edge_stacks", "extensions", "portainer.key", "portainer.pub", "tls"}
-
-func BackupToS3(settings portainer.S3BackupSettings, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) error {
-	archivePath, err := CreateBackupArchive(settings.Password, gate, datastore, filestorePath)
-	if err != nil {
-		log.Printf("[ERROR] failed to backup: %s \n", err)
-		return err
-	}
-	archiveReader, err := os.Open(archivePath)
-	if err != nil {
-		log.Println("[ERROR] failed to open backup file")
-		return err
-	}
-	defer os.RemoveAll(filepath.Dir(archivePath))
-
-	archiveName := fmt.Sprintf("portainer-backup_%s", filepath.Base(archivePath))
-
-	s3session, err := s3.NewSession(settings.Region, settings.AccessKeyID, settings.SecretAccessKey)
-	if err != nil {
-		log.Printf("[ERROR] %s \n", err)
-		return err
-	}
-	if err := s3.Upload(s3session, archiveReader, settings.BucketName, archiveName); err != nil {
-		log.Printf("[ERROR] failed to upload backup to S3: %s \n", err)
-		return err
-	}
-	return nil
-}
-
-// Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
-func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
-	unlock := gate.Lock()
-	defer unlock()
-
-	backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
-	if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
-		return "", errors.Wrap(err, "Failed to create backup dir")
-	}
-
-	if err := backupDb(backupDirPath, datastore); err != nil {
-		return "", errors.Wrap(err, "Failed to backup database")
-	}
-
-	for _, filename := range filesToBackup {
-		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
-		if err != nil {
-			return "", errors.Wrap(err, "Failed to create backup file")
-		}
-	}
-
-	archivePath, err := archive.TarGzDir(backupDirPath)
-	if err != nil {
-		return "", errors.Wrap(err, "Failed to make an archive")
-	}
-
-	if password != "" {
-		archivePath, err = encrypt(archivePath, password)
-		if err != nil {
-			return "", errors.Wrap(err, "Failed to encrypt backup with the password")
-		}
-	}
-
-	return archivePath, nil
-}
-
-func backupDb(backupDirPath string, datastore portainer.DataStore) error {
-	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
-	if err != nil {
-		return err
-	}
-	if err = datastore.BackupTo(backupWriter); err != nil {
-		return err
-	}
-	return backupWriter.Close()
-}
-
-func encrypt(path string, passphrase string) (string, error) {
-	in, err := os.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer in.Close()
-
-	outFileName := fmt.Sprintf("%s.encrypted", path)
-	out, err := os.Create(outFileName)
-	if err != nil {
-		return "", err
-	}
-
-	err = crypto.AesEncrypt(in, out, []byte(passphrase))
-
-	return outFileName, err
-}

api/backup/backup_scheduler.go

@@ -1,118 +0,0 @@
-package backup
-
-import (
-	"context"
-	"log"
-	"time"
-
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/offlinegate"
-	"github.com/robfig/cron/v3"
-)
-
-// BackupScheduler orchestrates S3 settings and active backup cron jobs
-type BackupScheduler struct {
-	cronmanager     *cron.Cron
-	s3backupService portainer.S3BackupService
-	gate            *offlinegate.OfflineGate
-	datastore       portainer.DataStore
-	filestorePath   string
-}
-
-func NewBackupScheduler(offlineGate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) *BackupScheduler {
-	crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger)))
-	s3backupService := datastore.S3Backup()
-
-	return &BackupScheduler{
-		cronmanager:     crontab,
-		s3backupService: s3backupService,
-		gate:            offlineGate,
-		datastore:       datastore,
-		filestorePath:   filestorePath,
-	}
-}
-
-// Start fetches latest backup settings and starts cron job if configured
-func (s *BackupScheduler) Start() error {
-	s.cronmanager.Start()
-
-	settings, err := s.s3backupService.GetSettings()
-	if err != nil {
-		return errors.Wrap(err, "failed to fetch settings")
-	}
-
-	if canBeScheduled(settings) {
-		return s.startJob(settings)
-	}
-
-	return nil
-}
-
-// Stop stops the scheduler if it is running; otherwise it does nothing.
-// A context is returned so the caller can wait for running jobs to complete.
-func (s *BackupScheduler) Stop() context.Context {
-	if s.cronmanager != nil {
-		log.Println("[DEBUG] Stopping backup scheduler")
-		return s.cronmanager.Stop()
-	}
-
-	return nil
-}
-
-// Update updates stored S3 backup settings and orchestrates cron jobs.
-// When scheduler has an active cron job, then it shuts it down.
-// When a provided settings has a cron, then starts a new cron job.
-// When ever current cron is being shut down, last cron error going to be dropped.
-func (s *BackupScheduler) Update(settings portainer.S3BackupSettings) error {
-
-	if err := s.s3backupService.UpdateSettings(settings); err != nil {
-		return errors.Wrap(err, "failed to update settings")
-	}
-
-	if err := s.stopJobs(); err != nil {
-		return errors.Wrap(err, "failed to stop current cronjob")
-	}
-
-	if canBeScheduled(settings) {
-		return s.startJob(settings)
-	}
-
-	return nil
-}
-
-// stops current backup cron job and drops last cron error if any
-func (s *BackupScheduler) stopJobs() error {
-	// stopping all cron jobs as there should be only one (c)
-	for _, job := range s.cronmanager.Entries() {
-		s.cronmanager.Remove(job.ID)
-	}
-
-	return s.s3backupService.DropStatus()
-}
-
-func (s *BackupScheduler) startJob(settings portainer.S3BackupSettings) error {
-	_, err := s.cronmanager.AddFunc(settings.CronRule, s.backup(settings))
-	if err != nil {
-		return errors.Wrap(err, "failed to start a new backup cron job")
-	}
-
-	return nil
-}
-
-func canBeScheduled(s portainer.S3BackupSettings) bool {
-	return s.AccessKeyID != "" && s.SecretAccessKey != "" && s.Region != "" && s.BucketName != "" && s.CronRule != ""
-}
-
-func (s *BackupScheduler) backup(settings portainer.S3BackupSettings) func() {
-	return func() {
-		err := BackupToS3(settings, s.gate, s.datastore, s.filestorePath)
-		status := portainer.S3BackupStatus{
-			Failed:    err != nil,
-			Timestamp: time.Now(),
-		}
-		if err = s.s3backupService.UpdateStatus(status); err != nil {
-			log.Printf("[ERROR] failed to update status of last scheduled backup. Status: %+v . Err: %s \n", status, err)
-		}
-	}
-}

api/backup/backup_scheduler_test.go

@@ -1,112 +0,0 @@
-package backup
-
-import (
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	i "github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
-func newScheduler(status *portainer.S3BackupStatus, settings *portainer.S3BackupSettings) *BackupScheduler {
-	scheduler := NewBackupScheduler(nil, i.NewDatastore(i.WithS3BackupService(status, settings)), "")
-	scheduler.Start()
-
-	return scheduler
-}
-
-func settings(cronRule string,
-	accessKeyID string,
-	secretAccessKey string,
-	region string,
-	bucketName string) *portainer.S3BackupSettings {
-	return &portainer.S3BackupSettings{
-		CronRule:        cronRule,
-		AccessKeyID:     accessKeyID,
-		SecretAccessKey: secretAccessKey,
-		Region:          region,
-		BucketName:      bucketName,
-	}
-}
-
-func Test_startWithoutCron_shouldNotStartAJob(t *testing.T) {
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
-	defer scheduler.Stop()
-
-	jobs := scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 0, "should have empty job list")
-}
-
-func Test_startWitACron_shouldAlsoStartAJob(t *testing.T) {
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, settings("*/10 * * * *", "id", "key", "region", "bucket"))
-	defer scheduler.Stop()
-
-	jobs := scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 1, "should have 1 active job")
-}
-
-func Test_update_shouldDropStatus(t *testing.T) {
-	storedStatus := &portainer.S3BackupStatus{Failed: true, Timestamp: time.Now().Add(-time.Hour)}
-	scheduler := newScheduler(storedStatus, &portainer.S3BackupSettings{})
-	defer scheduler.Stop()
-
-	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
-	assert.Equal(t, portainer.S3BackupStatus{}, *storedStatus, "stasus should be dropped")
-}
-
-func Test_update_shouldUpdateSettings(t *testing.T) {
-	storedSettings := &portainer.S3BackupSettings{}
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, storedSettings)
-	defer scheduler.Stop()
-
-	newSettings := settings("", "id2", "key2", "region2", "bucket2")
-	scheduler.Update(*newSettings)
-
-	assert.EqualValues(t, *storedSettings, *newSettings, "updated settings should match stored settings")
-}
-
-func Test_updateWithCron_shouldStartAJob(t *testing.T) {
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
-	defer scheduler.Stop()
-
-	jobs := scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 0, "should have empty job list upon startup")
-
-	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
-
-	jobs = scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 1, "should have 1 active job")
-}
-
-func Test_updateWithoutCron_shouldStopActiveJob(t *testing.T) {
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
-	defer scheduler.Stop()
-
-	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
-
-	jobs := scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 1, "should have 1 active job")
-
-	scheduler.Update(*settings("", "id2", "key2", "region2", "bucket2"))
-
-	jobs = scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 0, "should have no active jobs")
-}
-
-func Test_updateWithACron_shouldStopActiveJob_andStartNewJob(t *testing.T) {
-	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
-	defer scheduler.Stop()
-
-	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
-
-	jobs := scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 1, "should have 1 active job")
-	initJobId := jobs[0].ID
-
-	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
-
-	jobs = scheduler.cronmanager.Entries()
-	assert.Len(t, jobs, 1, "should have 1 active job")
-	assert.NotEqual(t, initJobId, jobs[0].ID, "new job should have a diffent id")
-}

api/backup/copy.go

@@ -1,68 +0,0 @@
-package backup
-
-import (
-	"errors"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-func copyPath(path string, toDir string) error {
-	info, err := os.Stat(path)
-	if err != nil && errors.Is(err, os.ErrNotExist) {
-		// skip copy if file does not exist
-		return nil
-	}
-
-	if !info.IsDir() {
-		destination := filepath.Join(toDir, info.Name())
-		return copyFile(path, destination)
-	}
-
-	return copyDir(path, toDir)
-}
-
-func copyDir(fromDir, toDir string) error {
-	cleanedSourcePath := filepath.Clean(fromDir)
-	parentDirectory := filepath.Dir(cleanedSourcePath)
-	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
-		if info.IsDir() {
-			return nil // skip directory creations
-		}
-
-		if info.Mode()&os.ModeSymlink != 0 { // entry is a symlink
-			return nil // don't copy symlinks
-		}
-
-		return copyFile(path, destination)
-	})
-
-	return err
-}
-
-// copies regular a file from src to dst
-func copyFile(src, dst string) error {
-	from, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer from.Close()
-
-	// has to include 'execute' bit, otherwise fails. MkdirAll follows `mkdir -m` restrictions
-	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
-		return err
-	}
-	to, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer to.Close()
-
-	_, err = io.Copy(to, from)
-	return err
-}

api/backup/copy_test.go

@@ -1,104 +0,0 @@
-package backup
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func contains(t *testing.T, list []string, path string) {
-	assert.Contains(t, list, path)
-	copyContent, _ := ioutil.ReadFile(path)
-	assert.Equal(t, "content\n", string(copyContent))
-}
-
-func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyFile("does-not-exist", tmpdir)
-	assert.NotNil(t, err)
-}
-
-func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
-
-	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
-	assert.Nil(t, err)
-
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyDir("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}
-
-func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyPath("does-not-exists", tmpdir)
-	assert.Nil(t, err)
-
-	assert.Empty(t, listFiles(tmpdir))
-}
-
-func Test_backupPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
-
-	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
-	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
-	assert.Nil(t, err)
-
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
-	assert.Nil(t, err)
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_backupPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyPath("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}

api/backup/restore.go

@@ -1,68 +0,0 @@
-package backup
-
-import (
-	"context"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/http/offlinegate"
-)
-
-var filesToRestore = append(filesToBackup, "portainer.db")
-
-// Restores system state from backup archive, will trigger system shutdown, when finished.
-func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, shutdownTrigger context.CancelFunc) error {
-	var err error
-	if password != "" {
-		archive, err = decrypt(archive, password)
-		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
-		}
-	}
-
-	restorePath := filepath.Join(filestorePath, "restore", time.Now().Format("20060102150405"))
-	defer os.RemoveAll(filepath.Dir(restorePath))
-
-	err = extractArchive(archive, restorePath)
-	if err != nil {
-		return errors.Wrap(err, "cannot extract files from the archive. Please ensure the password is correct and try again")
-	}
-
-	unlock := gate.Lock()
-	defer unlock()
-
-	if err = datastore.Close(); err != nil {
-		return errors.Wrap(err, "Failed to stop db")
-	}
-
-	if err = restoreFiles(restorePath, filestorePath); err != nil {
-		return errors.Wrap(err, "failed to restore the system state")
-	}
-
-	shutdownTrigger()
-	return nil
-}
-
-func decrypt(r io.Reader, password string) (io.Reader, error) {
-	return crypto.AesDecrypt(r, []byte(password))
-}
-
-func extractArchive(r io.Reader, destinationDirPath string) error {
-	return archive.ExtractTarGz(r, destinationDirPath)
-}
-
-func restoreFiles(srcDir string, destinationDir string) error {
-	for _, filename := range filesToRestore {
-		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

api/backup/test_assets/copy_test/dir/.dotfile

@@ -1 +0,0 @@
-content

api/backup/test_assets/copy_test/dir/inner

@@ -1 +0,0 @@
-content

api/backup/test_assets/copy_test/outer

@@ -1 +0,0 @@
-content

api/bolt/backup.go

@@ -1,181 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	plog "github.com/portainer/portainer/api/bolt/log"
-)
-
-var backupDefaults = struct {
-	backupDir        string
-	editions         []string
-	databaseFileName string
-}{
-	"backups",
-	[]string{"CE", "BE", "EE"},
-	databaseFileName,
-}
-
-var backupLog = plog.NewScopedLog("bolt, backup")
-
-//
-// Backup Helpers
-//
-
-// createBackupFolders create initial folders for backups
-func (store *Store) createBackupFolders() {
-	for _, e := range backupDefaults.editions {
-
-		p := path.Join(store.path, backupDefaults.backupDir, e)
-
-		if exists, _ := store.fileService.FileExists(p); !exists {
-			err := os.MkdirAll(p, 0700)
-			if err != nil {
-				backupLog.Error("Error while creating backup folders", err)
-			}
-		}
-	}
-}
-
-func (store *Store) databasePath() string {
-	return path.Join(store.path, databaseFileName)
-}
-
-func (store *Store) editionBackupDir(edition portainer.SoftwareEdition) string {
-	return path.Join(store.path, backupDefaults.backupDir, edition.GetEditionLabel())
-}
-
-func (store *Store) copyDBFile(from string, to string) error {
-	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
-	err := store.fileService.Copy(from, to, true)
-	if err != nil {
-		backupLog.Error("Failed", err)
-	}
-	return err
-}
-
-// BackupOptions provide a helper to inject backup options
-type BackupOptions struct {
-	Edition        portainer.SoftwareEdition
-	Version        int
-	BackupDir      string
-	BackupFileName string
-	BackupPath     string
-}
-
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-	if options == nil {
-		options = &BackupOptions{}
-	}
-	if options.Edition == 0 {
-		options.Edition = store.edition()
-	}
-	if options.Version == 0 {
-		options.Version, _ = store.version()
-	}
-	if options.BackupDir == "" {
-		options.BackupDir = store.editionBackupDir(options.Edition)
-	}
-	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
-	}
-	if options.BackupPath == "" {
-		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-	}
-	return options
-}
-
-func (store *Store) listEditionBackups(edition portainer.SoftwareEdition) ([]string, error) {
-	var fileNames = []string{}
-
-	files, err := ioutil.ReadDir(store.editionBackupDir(edition))
-
-	if err != nil {
-		backupLog.Error("Error while retrieving backup files", err)
-		return fileNames, err
-	}
-
-	for _, f := range files {
-		fileNames = append(fileNames, f.Name())
-	}
-
-	return fileNames, nil
-}
-
-func (store *Store) lastestEditionBackup() (string, error) {
-	edition := store.edition()
-
-	files, err := store.listEditionBackups(edition)
-	if err != nil {
-		backupLog.Error("Error while retrieving backup files", err)
-		return "", err
-	}
-
-	if len(files) == 0 {
-		return "", nil
-	}
-
-	return files[len(files)-1], nil
-}
-
-// BackupWithOptions backup current database with options
-func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
-	backupLog.Info("creating db backup")
-	store.createBackupFolders()
-
-	options = store.setupOptions(options)
-
-	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
-}
-
-// Backup current database with default options
-func (store *Store) Backup() (string, error) {
-	return store.BackupWithOptions(nil)
-}
-
-// RestoreWithOptions previously saved backup for the current Edition  with options
-// Restore strategies:
-// - default: restore latest from current edition
-// - restore a specific
-func (store *Store) RestoreWithOptions(options *BackupOptions) error {
-
-	// Check if backup file exist before restoring
-
-	options = store.setupOptions(options)
-
-	_, err := os.Stat(options.BackupPath)
-
-	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
-		return err
-	}
-
-	err = store.Close()
-	if err != nil {
-		backupLog.Error("Error while closing store before restore", err)
-		return err
-	}
-	backupLog.Info("Restoring db backup")
-	err = store.copyDBFile(options.BackupPath, store.databasePath())
-	if err != nil {
-		return err
-	}
-	return store.Open()
-}
-
-// Restore previously saved backup for the current Edition  with default options
-func (store *Store) Restore() error {
-	var options = &BackupOptions{}
-	var err error
-	options.BackupFileName, err = store.lastestEditionBackup()
-	if err != nil {
-		return err
-	}
-	return store.RestoreWithOptions(options)
-
-}

api/bolt/backup_test.go

@@ -1,118 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"log"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-func TestCreateBackupFolders(t *testing.T) {
-	store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false)
-	if exists, _ := store.fileService.FileExists("tmp/backups"); exists {
-		t.Error("Expect backups folder to not exist")
-	}
-	store.createBackupFolders()
-	if exists, _ := store.fileService.FileExists("tmp/backups"); !exists {
-		t.Error("Expect backups folder to exist")
-	}
-	store.createBackupFolders()
-	store.Close()
-	teardown()
-}
-
-func TestStoreCreation(t *testing.T) {
-	store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false)
-	if store == nil {
-		t.Error("Expect to create a store")
-	}
-
-	if store.edition() != portainer.PortainerEE {
-		t.Error("Expect to get EE Edition")
-	}
-
-	version, err := store.version()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if version != portainer.DBVersionEE {
-		t.Error("Expect to get EE DBVersion")
-	}
-
-	store.Close()
-	teardown()
-}
-
-func TestBackup(t *testing.T) {
-
-	tests := []struct {
-		edition portainer.SoftwareEdition
-		version int
-	}{
-		{edition: portainer.PortainerCE, version: portainer.DBVersion},
-		{edition: portainer.PortainerEE, version: portainer.DBVersionEE},
-	}
-
-	for _, tc := range tests {
-		backupFileName := fmt.Sprintf("tmp/backups/%s/portainer.db.%03d.*", tc.edition.GetEditionLabel(), tc.version)
-		t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) {
-			store := NewTestStore(tc.edition, tc.version, false)
-			store.Backup()
-
-			if !isFileExist(backupFileName) {
-				t.Errorf("Expect backup file to be created %s", backupFileName)
-			}
-			store.Close()
-		})
-	}
-	t.Run("BackupWithOption should create a name specific backup", func(t *testing.T) {
-		edition := portainer.PortainerCE
-		version := portainer.DBVersion
-		store := NewTestStore(edition, version, false)
-		store.BackupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerUpgradeToEEBackup,
-			Edition:        portainer.PortainerCE,
-		})
-		backupFileName := fmt.Sprintf("tmp/backups/%s/%s", edition.GetEditionLabel(), beforePortainerUpgradeToEEBackup)
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-		store.Close()
-	})
-
-	teardown()
-}
-
-// TODO restore / backup failed test cases
-func TestRestore(t *testing.T) {
-
-	editions := []portainer.SoftwareEdition{portainer.PortainerCE, portainer.PortainerEE}
-	var currentVersion = 0
-
-	for i, e := range editions {
-		editionLabel := e.GetEditionLabel()
-		currentVersion = 10 ^ i + 1
-		store := NewTestStore(e, currentVersion, false)
-		t.Run(fmt.Sprintf("Basic Restore for %s", editionLabel), func(t *testing.T) {
-			store.Backup()
-			updateVersion(store, currentVersion+1)
-			testVersion(store, currentVersion+1, t)
-			store.Restore()
-			testVersion(store, currentVersion, t)
-		})
-		t.Run(fmt.Sprintf("Basic Restore After Multiple Backup for %s", editionLabel), func(t *testing.T) {
-			currentVersion = currentVersion + 5
-			updateVersion(store, currentVersion)
-			store.Backup()
-			updateVersion(store, currentVersion+2)
-			testVersion(store, currentVersion+2, t)
-			store.Restore()
-			testVersion(store, currentVersion, t)
-		})
-		store.Close()
-	}
-
-	teardown()
-}

api/bolt/bolttest/datastore.go

@@ -1,73 +0,0 @@
-package bolttest
-
-import (
-	"io/ioutil"
-	"log"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/bolt"
-	"github.com/portainer/portainer/api/filesystem"
-)
-
-var errTempDir = errors.New("can't create a temp dir")
-
-func MustNewTestStore(init bool) (*bolt.Store, func()) {
-	store, teardown, err := NewTestStore(init)
-	if err != nil {
-		if !errors.Is(err, errTempDir) {
-			teardown()
-		}
-		log.Fatal(err)
-	}
-
-	return store, teardown
-}
-
-func NewTestStore(init bool) (*bolt.Store, func(), error) {
-	// Creates unique temp directory in a concurrency friendly manner.
-	dataStorePath, err := ioutil.TempDir("", "boltdb")
-	if err != nil {
-		return nil, nil, errors.Wrap(errTempDir, err.Error())
-	}
-
-	fileService, err := filesystem.NewService(dataStorePath, "")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	store, err := bolt.NewStore(dataStorePath, fileService)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	err = store.Open()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if init {
-		err = store.Init()
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	teardown := func() {
-		teardown(store, dataStorePath)
-	}
-
-	return store, teardown, nil
-}
-
-func teardown(store *bolt.Store, dataStorePath string) {
-	err := store.Close()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = os.RemoveAll(dataStorePath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

api/bolt/customtemplate/customtemplate.go

@@ -2,7 +2,7 @@ package customtemplate
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 	var customTemplates = make([]portainer.CustomTemplate, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 	var customTemplate portainer.CustomTemplate
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &customTemplate)
+	err := internal.GetObject(service.db, BucketName, identifier, &customTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 // UpdateCustomTemplate updates an custom template.
 func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, customTemplate)
+	return internal.UpdateObject(service.db, BucketName, identifier, customTemplate)
 }
 
 // DeleteCustomTemplate deletes an custom template.
 func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateCustomTemplate assign an ID to a new custom template and saves it.
 func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTemplate) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(customTemplate)
@@ -92,5 +92,5 @@ func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTem
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/datastore.go

@@ -1,17 +1,12 @@
 package bolt
 
 import (
-	"io"
+	"log"
 	"path"
 	"time"
 
-	"github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/license"
-	"github.com/portainer/portainer/api/bolt/s3backup"
-
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/customtemplate"
 	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/edgegroup"
@@ -20,7 +15,9 @@ import (
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -34,9 +31,10 @@ import (
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/bolt/webhook"
+	"github.com/portainer/portainer/api/internal/authorization"
 )
 
-var (
+const (
 	databaseFileName = "portainer.db"
 )
 
@@ -44,7 +42,7 @@ var (
 // BoltDB as the storage system.
 type Store struct {
 	path                    string
-	connection              *internal.DbConnection
+	db                      *bolt.DB
 	isNew                   bool
 	fileService             portainer.FileService
 	CustomTemplateService   *customtemplate.Service
@@ -56,11 +54,9 @@ type Store struct {
 	EndpointService         *endpoint.Service
 	EndpointRelationService *endpointrelation.Service
 	ExtensionService        *extension.Service
-	LicenseService          *license.Service
 	RegistryService         *registry.Service
 	ResourceControlService  *resourcecontrol.Service
 	RoleService             *role.Service
-	S3BackupService         *s3backup.Service
 	ScheduleService         *schedule.Service
 	SettingsService         *settings.Service
 	StackService            *stack.Service
@@ -73,29 +69,12 @@ type Store struct {
 	WebhookService          *webhook.Service
 }
 
-func (store *Store) version() (int, error) {
-	version, err := store.VersionService.DBVersion()
-	if err == errors.ErrObjectNotFound {
-		version = 0
-	}
-	return version, err
-}
-
-func (store *Store) edition() portainer.SoftwareEdition {
-	edition, err := store.VersionService.Edition()
-	if err == errors.ErrObjectNotFound {
-		edition = portainer.PortainerCE
-	}
-	return edition
-}
-
 // NewStore initializes a new Store and the associated services
 func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
 	store := &Store{
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
-		connection:  &internal.DbConnection{},
 	}
 
 	databasePath := path.Join(storePath, databaseFileName)
@@ -118,15 +97,15 @@ func (store *Store) Open() error {
 	if err != nil {
 		return err
 	}
-	store.connection.DB = db
+	store.db = db
 
 	return store.initServices()
 }
 
 // Close closes the BoltDB database.
 func (store *Store) Close() error {
-	if store.connection.DB != nil {
-		return store.connection.Close()
+	if store.db != nil {
+		return store.db.Close()
 	}
 	return nil
 }
@@ -137,11 +116,286 @@ func (store *Store) IsNew() bool {
 	return store.isNew
 }
 
-// BackupTo backs up db to a provided writer.
-// It does hot backup and doesn't block other database reads and writes
-func (store *Store) BackupTo(w io.Writer) error {
-	return store.connection.View(func(tx *bolt.Tx) error {
-		_, err := tx.WriteTo(w)
+// MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
+func (store *Store) MigrateData() error {
+	if store.isNew {
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
+	}
+
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	} else if err != nil {
 		return err
-	})
+	}
+
+	if version < portainer.DBVersion {
+		migratorParams := &migrator.Parameters{
+			DB:                      store.db,
+			DatabaseVersion:         version,
+			EndpointGroupService:    store.EndpointGroupService,
+			EndpointService:         store.EndpointService,
+			EndpointRelationService: store.EndpointRelationService,
+			ExtensionService:        store.ExtensionService,
+			RegistryService:         store.RegistryService,
+			ResourceControlService:  store.ResourceControlService,
+			RoleService:             store.RoleService,
+			ScheduleService:         store.ScheduleService,
+			SettingsService:         store.SettingsService,
+			StackService:            store.StackService,
+			TagService:              store.TagService,
+			TeamMembershipService:   store.TeamMembershipService,
+			UserService:             store.UserService,
+			VersionService:          store.VersionService,
+			FileService:             store.fileService,
+			AuthorizationService:    authorization.NewService(store),
+		}
+		migrator := migrator.NewMigrator(migratorParams)
+
+		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
+		err = migrator.Migrate()
+		if err != nil {
+			log.Printf("An error occurred during database migration: %s\n", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
+	customTemplateService, err := customtemplate.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.CustomTemplateService = customTemplateService
+
+	dockerhubService, err := dockerhub.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	edgeStackService, err := edgestack.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackService = edgeStackService
+
+	edgeGroupService, err := edgegroup.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeGroupService = edgeGroupService
+
+	edgeJobService, err := edgejob.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
+	endpointgroupService, err := endpointgroup.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	endpointRelationService, err := endpointrelation.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	extensionService, err := extension.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
+	registryService, err := registry.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	settingsService, err := settings.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	stackService, err := stack.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	tunnelServerService, err := tunnelserver.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
+	userService, err := user.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	webhookService, err := webhook.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
+	return nil
+}
+
+// CustomTemplate gives access to the CustomTemplate data management layer
+func (store *Store) CustomTemplate() portainer.CustomTemplateService {
+	return store.CustomTemplateService
+}
+
+// DockerHub gives access to the DockerHub data management layer
+func (store *Store) DockerHub() portainer.DockerHubService {
+	return store.DockerHubService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Endpoint gives access to the Endpoint data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
 }

api/bolt/dockerhub/dockerhub.go

@@ -1,8 +1,10 @@
 package dockerhub
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -13,18 +15,18 @@ const (
 
 // Service represents a service for managing Dockerhub data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
-	err := internal.GetObject(service.connection, BucketName, []byte(dockerHubKey), &dockerhub)
+	err := internal.GetObject(service.db, BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(dockerHubKey), dockerhub)
+	return internal.UpdateObject(service.db, BucketName, []byte(dockerHubKey), dockerhub)
 }

api/bolt/edgegroup/edgegroup.go

@@ -2,7 +2,7 @@ package edgegroup
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 	var groups = make([]portainer.EdgeGroup, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 	var group portainer.EdgeGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &group)
+	err := internal.GetObject(service.db, BucketName, identifier, &group)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 // UpdateEdgeGroup updates an Edge group.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, group)
+	return internal.UpdateObject(service.db, BucketName, identifier, group)
 }
 
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) CreateEdgeGroup(group *portainer.EdgeGroup) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/edgejob/edgejob.go

@@ -2,7 +2,7 @@ package edgejob
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 	var edgeJob portainer.EdgeJob
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &edgeJob)
+	err := internal.GetObject(service.db, BucketName, identifier, &edgeJob)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 
 // CreateEdgeJob creates a new Edge job
 func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeJob.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
 // UpdateEdgeJob updates an Edge job by ID
 func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, edgeJob)
+	return internal.UpdateObject(service.db, BucketName, identifier, edgeJob)
 }
 
 // DeleteEdgeJob deletes an Edge job
 func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/edgestack/edgestack.go

@@ -2,7 +2,7 @@ package edgestack
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 	var stack portainer.EdgeStack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	err := internal.GetObject(service.db, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 
 // CreateEdgeStack assign an ID to a new Edge stack and saves it.
 func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeStack.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
 // UpdateEdgeStack updates an Edge stack.
 func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, edgeStack)
+	return internal.UpdateObject(service.db, BucketName, identifier, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/endpoint/endpoint.go

@@ -2,7 +2,7 @@ package endpoint
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 	var endpoint portainer.Endpoint
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpoint)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -44,20 +44,20 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 // UpdateEndpoint updates an endpoint.
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpoint)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpoint)
 }
 
 // DeleteEndpoint deletes an endpoint.
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // Endpoints return an array containing all the endpoints.
 func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -78,7 +78,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 
 // CreateEndpoint assign an ID to a new endpoint and saves it.
 func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for endpoints
@@ -98,12 +98,12 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }
 
 // Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		for _, endpoint := range toCreate {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,7 +1,7 @@
 package endpointgroup
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 	var endpointGroup portainer.EndpointGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpointGroup)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpointGroup)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 // UpdateEndpointGroup updates an endpoint group.
 func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpointGroup)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpointGroup)
 }
 
 // DeleteEndpointGroup deletes an endpoint group.
 func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // EndpointGroups return an array containing all the endpoint groups.
 func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -79,7 +79,7 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 
 // CreateEndpointGroup assign an ID to a new endpoint group and saves it.
 func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/endpointrelation/endpointrelation.go

@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint relation data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 	var endpointRelation portainer.EndpointRelation
 	identifier := internal.Itob(int(endpointID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpointRelation)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpointRelation)
 	if err != nil {
 		return nil, err
 	}
@@ -43,7 +43,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 
 // CreateEndpointRelation saves endpointRelation
 func (service *Service) CreateEndpointRelation(endpointRelation *portainer.EndpointRelation) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(endpointRelation)
@@ -58,11 +58,11 @@ func (service *Service) CreateEndpointRelation(endpointRelation *portainer.Endpo
 // UpdateEndpointRelation updates an Endpoint relation object
 func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpointRelation)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpointRelation)
 }
 
 // DeleteEndpointRelation deletes an Endpoint relation object
 func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/errors/errors.go

@@ -4,5 +4,4 @@ import "errors"
 
 var (
 	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrMigrationToCE  = errors.New("DB is already on CE edition")
 )

api/bolt/extension/extension.go

@@ -1,7 +1,7 @@
 package extension
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 	var extension portainer.Extension
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &extension)
+	err := internal.GetObject(service.db, BucketName, identifier, &extension)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(extension)
@@ -82,5 +82,5 @@ func (service *Service) Persist(extension *portainer.Extension) error {
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/helpers_test.go

@@ -1,115 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"log"
-	"math/rand"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
-)
-
-var (
-	dataStorePath  string
-	testBackupPath string
-)
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-	databaseFileName = fmt.Sprintf("portainer-%08d.db", rand.Intn(100000000))
-
-	pwd, err := os.Getwd()
-	if err != nil {
-		log.Println(err)
-	}
-
-	dataStorePath = path.Join(pwd, "tmp")
-	testBackupPath = path.Join(dataStorePath, "backups")
-
-	teardown()
-}
-
-func NewTestStore(edition portainer.SoftwareEdition, version int, init bool) *Store {
-	fileService, err := filesystem.NewService(dataStorePath, "")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	store, err := NewStore(dataStorePath, fileService)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = store.Open()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if init {
-		err = store.Init()
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-
-	err = store.VersionService.StoreEdition(edition)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = store.VersionService.StoreDBVersion(version)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return store
-}
-
-func teardown() {
-	err := os.RemoveAll(testBackupPath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	files, err := filepath.Glob(path.Join(dataStorePath, "portainer-*.*"))
-	if err != nil {
-		log.Fatalln(err)
-	}
-	for _, f := range files {
-		if err := os.Remove(f); err != nil {
-			log.Fatalln(err)
-		}
-	}
-}
-
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}
-
-func updateVersion(store *Store, v int) {
-	err := store.VersionService.StoreDBVersion(v)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func testVersion(store *Store, versionWant int, t *testing.T) {
-	if v, _ := store.version(); v != versionWant {
-		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
-	}
-}
-
-func testEdition(store *Store, editionWant portainer.SoftwareEdition, t *testing.T) {
-	if e := store.edition(); e != editionWant {
-		t.Errorf("Expect store edition to be %s but was %s", editionWant.GetEditionLabel(), e.GetEditionLabel())
-	}
-}

api/bolt/init.go

@@ -33,7 +33,6 @@ func (store *Store) Init() error {
 				AnonymousMode:   true,
 				AutoCreateUsers: true,
 				TLSConfig:       portainer.TLSConfiguration{},
-				URLs:            []string{},
 				SearchSettings: []portainer.LDAPSearchSettings{
 					portainer.LDAPSearchSettings{},
 				},
@@ -41,14 +40,18 @@ func (store *Store) Init() error {
 					portainer.LDAPGroupSearchSettings{},
 				},
 			},
-			OAuthSettings: portainer.OAuthSettings{
-				TeamMemberships: portainer.TeamMemberships{
-					OAuthClaimMappings: make([]portainer.OAuthClaimMappings, 0),
-				},
-			},
-			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-			TemplatesURL:             portainer.DefaultTemplatesURL,
-			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
+			OAuthSettings:                             portainer.OAuthSettings{},
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowVolumeBrowserForRegularUsers:         false,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			EnableHostManagementFeatures:              false,
+			EdgeAgentCheckinInterval:                  portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
+			TemplatesURL:                              portainer.DefaultTemplatesURL,
+			UserSessionTimeout:                        portainer.DefaultUserSessionTimeout,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)
@@ -96,17 +99,5 @@ func (store *Store) Init() error {
 		}
 	}
 
-	roles, err := store.RoleService.Roles()
-	if err != nil {
-		return err
-	}
-
-	if len(roles) == 0 {
-		err := store.RoleService.CreateOrUpdatePredefinedRoles()
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }

api/bolt/internal/db.go

@@ -7,10 +7,6 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
-type DbConnection struct {
-	*bolt.DB
-}
-
 // Itob returns an 8-byte big endian representation of v.
 // This function is typically used for encoding integer IDs to byte slices
 // so that they can be used as BoltDB keys.
@@ -21,8 +17,8 @@ func Itob(v int) []byte {
 }
 
 // CreateBucket is a generic function used to create a bucket inside a bolt database.
-func CreateBucket(connection *DbConnection, bucketName string) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func CreateBucket(db *bolt.DB, bucketName string) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
 		if err != nil {
 			return err
@@ -32,10 +28,10 @@ func CreateBucket(connection *DbConnection, bucketName string) error {
 }
 
 // GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
-func GetObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
 	var data []byte
 
-	err := connection.View(func(tx *bolt.Tx) error {
+	err := db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		value := bucket.Get(key)
@@ -56,8 +52,8 @@ func GetObject(connection *DbConnection, bucketName string, key []byte, object i
 }
 
 // UpdateObject is a generic function used to update an object inside a bolt database.
-func UpdateObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		data, err := MarshalObject(object)
@@ -75,18 +71,18 @@ func UpdateObject(connection *DbConnection, bucketName string, key []byte, objec
 }
 
 // DeleteObject is a generic function used to delete an object inside a bolt database.
-func DeleteObject(connection *DbConnection, bucketName string, key []byte) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func DeleteObject(db *bolt.DB, bucketName string, key []byte) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
 }
 
 // GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
-func GetNextIdentifier(connection *DbConnection, bucketName string) int {
+func GetNextIdentifier(db *bolt.DB, bucketName string) int {
 	var identifier int
 
-	connection.Update(func(tx *bolt.Tx) error {
+	db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {

api/bolt/license/license.go

@@ -1,92 +0,0 @@
-package license
-
-import (
-	"github.com/portainer/liblicense"
-	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "license"
-)
-
-// Service represents a service for managing endpoint data.
-type Service struct {
-	connection *internal.DbConnection
-}
-
-// NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// License returns a license by licenseKey
-func (service *Service) License(licenseKey string) (*liblicense.PortainerLicense, error) {
-	var license liblicense.PortainerLicense
-	identifier := []byte(licenseKey)
-
-	err := internal.GetObject(service.connection, BucketName, identifier, &license)
-	if err != nil {
-		return nil, err
-	}
-
-	return &license, nil
-}
-
-// Licenses return an array containing all the licenses.
-func (service *Service) Licenses() ([]liblicense.PortainerLicense, error) {
-	var licenses = make([]liblicense.PortainerLicense, 0)
-
-	err := service.connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var license liblicense.PortainerLicense
-			err := internal.UnmarshalObject(v, &license)
-			if err != nil {
-				return err
-			}
-			licenses = append(licenses, license)
-		}
-
-		return nil
-	})
-
-	return licenses, err
-}
-
-// AddLicense persists a license inside the database.
-func (service *Service) AddLicense(licenseKey string, license *liblicense.PortainerLicense) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		data, err := internal.MarshalObject(license)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put([]byte(licenseKey), data)
-	})
-}
-
-// UpdateLicense updates a license.
-func (service *Service) UpdateLicense(licenseKey string, license *liblicense.PortainerLicense) error {
-	identifier := []byte(licenseKey)
-	return internal.UpdateObject(service.connection, BucketName, identifier, license)
-}
-
-// DeleteLicense deletes a License.
-func (service *Service) DeleteLicense(licenseKey string) error {
-	identifier := []byte(licenseKey)
-	return internal.DeleteObject(service.connection, BucketName, identifier)
-}

api/bolt/log/log.go

@@ -1,41 +0,0 @@
-package log
-
-import (
-	"fmt"
-	"log"
-)
-
-const (
-	INFO  = "INFO"
-	ERROR = "ERROR"
-	DEBUG = "DEBUG"
-	FATAL = "FATAL"
-)
-
-type ScopedLog struct {
-	scope string
-}
-
-func NewScopedLog(scope string) *ScopedLog {
-	return &ScopedLog{scope: scope}
-}
-
-func (slog *ScopedLog) print(kind string, message string) {
-	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
-}
-
-func (slog *ScopedLog) Debug(message string) {
-	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Info(message string) {
-	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Error(message string, err error) {
-	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
-}
-
-func (slog *ScopedLog) NotImplemented(method string) {
-	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
-}

api/bolt/log/log.test.go

@@ -1 +0,0 @@
-package log

api/bolt/migrate_data.go

@@ -1,207 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-
-	portainer "github.com/portainer/portainer/api"
-	errors "github.com/portainer/portainer/api/bolt/errors"
-	plog "github.com/portainer/portainer/api/bolt/log"
-	"github.com/portainer/portainer/api/bolt/migrator"
-	"github.com/portainer/portainer/api/cli"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-const beforePortainerUpgradeToEEBackup = "portainer.db.before-EE-upgrade"
-
-var migrateLog = plog.NewScopedLog("bolt, migrate")
-
-// FailSafeMigrate backup and restore DB if migration fail
-func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version int) error {
-	defer func() {
-		if err := recover(); err != nil {
-			migrateLog.Info(fmt.Sprintf("Error during migration, recovering [%v]", err))
-			store.Restore()
-		}
-	}()
-	return migrator.Migrate(version)
-}
-
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-// if force is true, then migrate regardless.
-func (store *Store) MigrateData(force bool) error {
-	// 0 – if DB is new then we don't need to migrate any data and just set version and edition to latest EE
-	if store.isNew && !force {
-		err := store.VersionService.StoreDBVersion(portainer.DBVersionEE)
-		if err != nil {
-			return err
-		}
-
-		err = store.VersionService.StoreEdition(portainer.PortainerEE)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	migrator, err := store.newMigrator()
-	if err != nil {
-		return err
-	}
-
-	if migrator.Edition() == portainer.PortainerCE {
-		// backup before migrating
-		store.BackupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerUpgradeToEEBackup,
-			Edition:        portainer.PortainerCE,
-		})
-
-		store.VersionService.StorePreviousDBVersion(migrator.Version())
-
-		// 1 – We need to migrate DB to latest CE version
-
-		if migrator.Version() < portainer.DBVersion {
-			store.Backup()
-			err = store.FailSafeMigrate(migrator, portainer.DBVersion)
-			if err != nil {
-				store.Restore()
-				migrateLog.Error("An error occurred while migrating CE database to latest version", err)
-				return err
-			}
-		}
-
-	}
-
-	if portainer.Edition == portainer.PortainerEE {
-		// 2 – if DB is CE Edition we need to upgrade settings to EE
-		if migrator.Edition() < portainer.PortainerEE {
-			err = migrator.UpgradeToEE()
-			if err != nil {
-				migrateLog.Error("An error occurred while upgrading database to EE", err)
-				store.RollbackFailedUpgradeToEE()
-				return err
-			}
-
-		}
-
-		// 3 – if DB is EE Edition we need to migrate to latest version of EE
-		if migrator.Edition() == portainer.PortainerEE && migrator.Version() < portainer.DBVersionEE {
-			store.Backup()
-			err = store.FailSafeMigrate(migrator, portainer.DBVersionEE)
-			if err != nil {
-				migrateLog.Error("An error occurred while migrating EE database to latest version", err)
-				store.Restore()
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// RollbackFailedUpgradeToEE down migrate to previous version
-func (store *Store) RollbackFailedUpgradeToEE() error {
-	return store.RestoreWithOptions(&BackupOptions{
-		BackupFileName: beforePortainerUpgradeToEEBackup,
-		Edition:        portainer.PortainerCE,
-	})
-}
-
-// RollbackToCE rollbacks the store to the current ce version
-func (store *Store) RollbackToCE() error {
-
-	migrator, err := store.newMigrator()
-	if err != nil {
-		return err
-	}
-
-	migrateLog.Info(fmt.Sprintf("Current Software Edition: %s", migrator.Edition().GetEditionLabel()))
-	migrateLog.Info(fmt.Sprintf("Current DB Version: %d", migrator.Version()))
-
-	if migrator.Edition() == portainer.PortainerCE {
-		return errors.ErrMigrationToCE
-	}
-
-	previousVersion, err := store.VersionService.PreviousDBVersion()
-	if err != nil {
-		migrateLog.Error("An Error occurred with retrieving previous DB version", err)
-		return err
-	}
-
-	confirmed, err := cli.Confirm(fmt.Sprintf("Are you sure you want to rollback your database to %d?", previousVersion))
-	if err != nil || !confirmed {
-		return err
-	}
-
-	if previousVersion < 25 {
-		migrator.DowngradeSettingsFrom25()
-	}
-
-	err = store.VersionService.StoreDBVersion(previousVersion)
-	if err != nil {
-		migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err)
-		return err
-	}
-
-	err = store.VersionService.StoreEdition(portainer.PortainerCE)
-	if err != nil {
-		migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err)
-		return err
-	}
-
-	migrateLog.Info(fmt.Sprintf("Rolled back to CE Edition, DB Version %d", previousVersion))
-	return nil
-}
-
-func (store *Store) newMigrator() (*migrator.Migrator, error) {
-
-	version, err := store.version()
-	if err != nil {
-		return nil, err
-	}
-
-	edition := store.edition()
-
-	params := &migrator.Parameters{
-		DB:              store.connection.DB,
-		DatabaseVersion: version,
-		CurrentEdition:  edition,
-
-		EndpointGroupService:    store.EndpointGroupService,
-		EndpointService:         store.EndpointService,
-		EndpointRelationService: store.EndpointRelationService,
-		ExtensionService:        store.ExtensionService,
-		RegistryService:         store.RegistryService,
-		ResourceControlService:  store.ResourceControlService,
-		RoleService:             store.RoleService,
-		ScheduleService:         store.ScheduleService,
-		SettingsService:         store.SettingsService,
-		StackService:            store.StackService,
-		TagService:              store.TagService,
-		TeamMembershipService:   store.TeamMembershipService,
-		UserService:             store.UserService,
-		VersionService:          store.VersionService,
-		FileService:             store.fileService,
-		AuthorizationService:    authorization.NewService(store),
-	}
-
-	return migrator.NewMigrator(params), nil
-}
-
-// RollbackVersion down migrate to previous version
-func (store *Store) RollbackVersion(version int) error {
-	// TODO
-	backupLog.NotImplemented("RollbackVersion")
-	return nil
-}
-
-// RollbackEdition downgrade to previous edition
-func (store *Store) RollbackEdition(edition portainer.SoftwareEdition) error {
-	// TODO
-	backupLog.NotImplemented("RollbackEdition")
-	// Change Edition
-	// Migrate Services
-	// Restore Latest
-	return nil
-}

api/bolt/migrate_data_test.go

@@ -1,99 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"log"
-	"testing"
-
-	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
-)
-
-// New Database should be EE and DBVersion
-//
-
-func TestMigrateData(t *testing.T) {
-	var store *Store
-
-	t.Run("MigrateData for New Store", func(t *testing.T) {
-		fileService, err := filesystem.NewService(dataStorePath, "")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		store, err := NewStore(dataStorePath, fileService)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		err = store.Open()
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		err = store.Init()
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		store.MigrateData(false)
-
-		testVersion(store, portainer.DBVersionEE, t)
-		testEdition(store, portainer.PortainerEE, t)
-
-		store.Close()
-	})
-
-	tests := []struct {
-		edition         portainer.SoftwareEdition
-		version         int
-		expectedVersion int
-	}{
-		{edition: portainer.PortainerCE, version: 5, expectedVersion: portainer.DBVersionEE},
-		{edition: portainer.PortainerCE, version: 21, expectedVersion: portainer.DBVersionEE},
-	}
-
-	for _, tc := range tests {
-		store = NewTestStore(tc.edition, tc.version, true)
-		t.Run(fmt.Sprintf("MigrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) {
-			store.MigrateData(false)
-			testVersion(store, tc.expectedVersion, t)
-			testEdition(store, portainer.PortainerEE, t)
-		})
-
-		t.Run(fmt.Sprintf("Restoring DB after migrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) {
-			store.RollbackToCE()
-			testVersion(store, tc.version, t)
-			testEdition(store, tc.edition, t)
-		})
-
-		store.Close()
-	}
-
-	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		version := 21
-		store = NewTestStore(portainer.PortainerCE, version, true)
-
-		deleteBucket(store.connection.DB, "settings")
-		store.MigrateData(false)
-
-		testVersion(store, version, t)
-		testEdition(store, portainer.PortainerCE, t)
-
-		store.Close()
-	})
-
-	teardown()
-}
-
-func deleteBucket(db *bolt.DB, bucketName string) {
-	db.Update(func(tx *bolt.Tx) error {
-		log.Printf("Delete bucket %s\n", bucketName)
-		err := tx.DeleteBucket([]byte(bucketName))
-		if err != nil {
-			log.Println(err)
-		}
-		return err
-	})
-}

api/bolt/migrator/downgrade_dbversion25.go

@@ -1,15 +0,0 @@
-package migrator
-
-// DowngradeSettingsFrom25 downgrade template settings for portainer v1.2
-func (migrator *Migrator) DowngradeSettingsFrom25() error {
-	legacySettings, err := migrator.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	legacySettings.TemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates-1.20.0.json"
-
-	err = migrator.settingsService.UpdateSettings(legacySettings)
-
-	return err
-}

api/bolt/migrator/migrate_ce.go

@@ -1,308 +0,0 @@
-package migrator
-
-import (
-	"log"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// MigrateCE checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) MigrateCE() error {
-	// Portainer < 1.12
-	if m.currentDBVersion < 1 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.currentDBVersion < 2 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return err
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.13.x
-	if m.currentDBVersion < 3 {
-		err := m.updateSettingsToDBVersion3()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.14.0
-	if m.currentDBVersion < 4 {
-		err := m.updateEndpointsToDBVersion4()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1235
-	if m.currentDBVersion < 5 {
-		err := m.updateSettingsToVersion5()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1236
-	if m.currentDBVersion < 6 {
-		err := m.updateSettingsToVersion6()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1449
-	if m.currentDBVersion < 7 {
-		err := m.updateSettingsToVersion7()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 8 {
-		err := m.updateEndpointsToVersion8()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https: //github.com/portainer/portainer/issues/1396
-	if m.currentDBVersion < 9 {
-		err := m.updateEndpointsToVersion9()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/461
-	if m.currentDBVersion < 10 {
-		err := m.updateEndpointsToVersion10()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1906
-	if m.currentDBVersion < 11 {
-		err := m.updateEndpointsToVersion11()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.18.0
-	if m.currentDBVersion < 12 {
-		err := m.updateEndpointsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateStacksToVersion12()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.0
-	if m.currentDBVersion < 13 {
-		err := m.updateSettingsToVersion13()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.2
-	if m.currentDBVersion < 14 {
-		err := m.updateResourceControlsToDBVersion14()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.0
-	if m.currentDBVersion < 15 {
-		err := m.updateSettingsToDBVersion15()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateTemplatesToVersion15()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 16 {
-		err := m.updateSettingsToDBVersion16()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.1
-	if m.currentDBVersion < 17 {
-		err := m.updateExtensionsToDBVersion17()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		err := m.updateEndpointSettingsToDB26()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateRbacRolesToDB26()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer EE-2.4.0
-	if m.currentDBVersion < 28 {
-		err := m.updateUsersAndRolesToDBVersion28()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer EE-2.4.0
-	if m.currentDBVersion < 29 {
-		err := m.updateRbacRolesToDB29()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer EE-2.7.0
-	if m.currentDBVersion < 31 {
-		err := m.updateSettingsToDB31()
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Println("Update DB version to ", portainer.DBVersion)
-	return m.versionService.StoreDBVersion(portainer.DBVersion)
-}

api/bolt/migrator/migrate_dbversion20.go

@@ -29,6 +29,11 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
 	for _, user := range legacyUsers {
 		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
 		err = m.userService.UpdateUser(user.ID, &user)
@@ -51,7 +56,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	helpDeskRole.Priority = 2
-	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole()
+	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole(settings.AllowVolumeBrowserForRegularUsers)
 
 	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
 
@@ -60,7 +65,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	standardUserRole.Priority = 3
-	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole()
+	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
 	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
 
@@ -69,7 +74,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	readOnlyUserRole.Priority = 4
-	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole()
+	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
 	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
 	if err != nil {

api/bolt/migrator/migrate_dbversion25.go

@@ -1,76 +0,0 @@
-package migrator
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-func (m *Migrator) updateEndpointSettingsToDB26() error {
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	endpoints, err := m.endpointService.Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for i := range endpoints {
-		endpoint := endpoints[i]
-
-		securitySettings := portainer.EndpointSecuritySettings{}
-
-		if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment ||
-			endpoint.Type == portainer.AgentOnDockerEnvironment ||
-			endpoint.Type == portainer.DockerEnvironment {
-
-			securitySettings = portainer.EndpointSecuritySettings{
-				AllowBindMountsForRegularUsers:            settings.AllowBindMountsForRegularUsers,
-				AllowContainerCapabilitiesForRegularUsers: settings.AllowContainerCapabilitiesForRegularUsers,
-				AllowDeviceMappingForRegularUsers:         settings.AllowDeviceMappingForRegularUsers,
-				AllowHostNamespaceForRegularUsers:         settings.AllowHostNamespaceForRegularUsers,
-				AllowPrivilegedModeForRegularUsers:        settings.AllowPrivilegedModeForRegularUsers,
-				AllowStackManagementForRegularUsers:       settings.AllowStackManagementForRegularUsers,
-			}
-
-			if endpoint.Type == portainer.AgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-				securitySettings.AllowVolumeBrowserForRegularUsers = settings.AllowVolumeBrowserForRegularUsers
-				securitySettings.EnableHostManagementFeatures = settings.EnableHostManagementFeatures
-			}
-		}
-
-		endpoint.SecuritySettings = securitySettings
-
-		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (m *Migrator) updateRbacRolesToDB26() error {
-	defaultAuthorizationsOfRoles := map[portainer.RoleID]portainer.Authorizations{
-		portainer.RoleIDEndpointAdmin: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
-		portainer.RoleIDHelpdesk:      authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
-		portainer.RoleIDStandardUser:  authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
-		portainer.RoleIDReadonly:      authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
-	}
-
-	for roleID, defaultAuthorizations := range defaultAuthorizationsOfRoles {
-		role, err := m.roleService.Role(roleID)
-		if err != nil {
-			return err
-		}
-		role.Authorizations = defaultAuthorizations
-
-		err = m.roleService.UpdateRole(role.ID, role)
-		if err != nil {
-			return err
-		}
-	}
-
-	return m.authorizationService.UpdateUsersAuthorizations()
-}

api/bolt/migrator/migrate_dbversion26.go

@@ -1,43 +0,0 @@
-package migrator
-
-import (
-	"fmt"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-)
-
-func (m *Migrator) updateStackResourceControlToDB27() error {
-	resourceControls, err := m.resourceControlService.ResourceControls()
-	if err != nil {
-		return err
-	}
-
-	for _, resource := range resourceControls {
-		if resource.Type != portainer.StackResourceControl {
-			continue
-		}
-
-		stackName := resource.ResourceID
-		if err != nil {
-			return err
-		}
-
-		stack, err := m.stackService.StackByName(stackName)
-		if err != nil {
-			if err == errors.ErrObjectNotFound {
-				continue
-			}
-
-			return err
-		}
-
-		resource.ResourceID = fmt.Sprintf("%d_%s", stack.EndpointID, stack.Name)
-
-		err = m.resourceControlService.UpdateResourceControl(resource.ID, &resource)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

api/bolt/migrator/migrate_dbversion27.go

@@ -1,10 +0,0 @@
-package migrator
-
-func (m *Migrator) updateUsersAndRolesToDBVersion28() error {
-	err := m.roleService.CreateOrUpdatePredefinedRoles()
-	if err != nil {
-		return err
-	}
-
-	return m.authorizationService.UpdateUsersAuthorizations()
-}

api/bolt/migrator/migrate_dbversion28.go

@@ -1,31 +0,0 @@
-package migrator
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-func (m *Migrator) updateRbacRolesToDB29() error {
-	defaultAuthorizationsOfRoles := map[portainer.RoleID]portainer.Authorizations{
-		portainer.RoleIDEndpointAdmin: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
-		portainer.RoleIDHelpdesk:      authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
-		portainer.RoleIDOperator:      authorization.DefaultEndpointAuthorizationsForOperatorRole(),
-		portainer.RoleIDStandardUser:  authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
-		portainer.RoleIDReadonly:      authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
-	}
-
-	for roleID, defaultAuthorizations := range defaultAuthorizationsOfRoles {
-		role, err := m.roleService.Role(roleID)
-		if err != nil {
-			return err
-		}
-		role.Authorizations = defaultAuthorizations
-
-		err = m.roleService.UpdateRole(role.ID, role)
-		if err != nil {
-			return err
-		}
-	}
-
-	return m.authorizationService.UpdateUsersAuthorizations()
-}

api/bolt/migrator/migrate_dbversion30.go

@@ -1,12 +0,0 @@
-package migrator
-
-func (m *Migrator) updateSettingsToDB31() error {
-	legacySettings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-	legacySettings.OAuthSettings.SSO = false
-	legacySettings.OAuthSettings.HideInternalAuth = false
-	legacySettings.OAuthSettings.LogoutURI = ""
-	return m.settingsService.UpdateSettings(legacySettings)
-}

api/bolt/migrator/migrate_dbversion30_test.go

@@ -1,67 +0,0 @@
-package migrator
-
-import (
-	"os"
-	"testing"
-
-	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api/bolt/settings"
-)
-
-var (
-	testingDBStorePath string
-	testingDBFileName  string
-	dummyLogoURL       string
-	dbConn             *bolt.DB
-	settingsService    *settings.Service
-)
-
-func setup() error {
-	testingDBStorePath, _ = os.Getwd()
-	testingDBFileName = "portainer-ee-mig-30.db"
-	dummyLogoURL = "example.com"
-	var err error
-	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
-	if err != nil {
-		return err
-	}
-	dummySettingsObj := map[string]interface{}{
-		"LogoURL": dummyLogoURL,
-	}
-	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func TestUpdateSettingsToDB31(t *testing.T) {
-	if err := setup(); err != nil {
-		t.Errorf("failed to complete testing setups, err: %v", err)
-	}
-	defer dbConn.Close()
-	defer os.Remove(testingDBFileName)
-	m := &Migrator{
-		db:              dbConn,
-		settingsService: settingsService,
-	}
-	if err := m.updateSettingsToDB31(); err != nil {
-		t.Errorf("failed to update settings: %v", err)
-	}
-	updatedSettings, err := m.settingsService.Settings()
-	if err != nil {
-		t.Errorf("failed to retrieve the updated settings: %v", err)
-	}
-	if updatedSettings.LogoURL != dummyLogoURL {
-		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
-	}
-	if updatedSettings.OAuthSettings.SSO != false {
-		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
-	}
-	if updatedSettings.OAuthSettings.HideInternalAuth != false {
-		t.Errorf("unexpected default OAuth HideInternalAuth setting, want: false, got: %t", updatedSettings.OAuthSettings.HideInternalAuth)
-	}
-	if updatedSettings.OAuthSettings.LogoutURI != "" {
-		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
-	}
-}

api/bolt/migrator/migrate_test_helper.go

@@ -1,38 +0,0 @@
-package migrator
-
-import (
-	"path"
-	"time"
-
-	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/settings"
-)
-
-// initTestingDBConn creates a raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
-	databasePath := path.Join(storePath, fileName)
-	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return nil, err
-	}
-	return dbConn, nil
-}
-
-// initTestingDBConn creates a settings service with raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
-	internalDBConn := &internal.DbConnection{
-		DB: dbConn,
-	}
-	settingsService, err := settings.NewService(internalDBConn)
-	if err != nil {
-		return nil, err
-	}
-	//insert a obj
-	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
-		return nil, err
-	}
-	return settingsService, nil
-}

api/bolt/migrator/migrator.go

@@ -1,15 +1,12 @@
 package migrator
 
 import (
-	"fmt"
-
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
-	plog "github.com/portainer/portainer/api/bolt/log"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -23,15 +20,11 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
-var migrateLog = plog.NewScopedLog("bolt, migrate")
-
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		db               *bolt.DB
-		currentDBVersion int
-		currentEdition   portainer.SoftwareEdition
-
+		currentDBVersion        int
+		db                      *bolt.DB
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -52,10 +45,8 @@ type (
 
 	// Parameters represents the required parameters to create a new Migrator instance.
 	Parameters struct {
-		DB              *bolt.DB
-		DatabaseVersion int
-		CurrentEdition  portainer.SoftwareEdition
-
+		DB                      *bolt.DB
+		DatabaseVersion         int
 		EndpointGroupService    *endpointgroup.Service
 		EndpointService         *endpoint.Service
 		EndpointRelationService *endpointrelation.Service
@@ -80,7 +71,6 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	return &Migrator{
 		db:                      parameters.DB,
 		currentDBVersion:        parameters.DatabaseVersion,
-		currentEdition:          parameters.CurrentEdition,
 		endpointGroupService:    parameters.EndpointGroupService,
 		endpointService:         parameters.EndpointService,
 		endpointRelationService: parameters.EndpointRelationService,
@@ -100,43 +90,257 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	}
 }
 
-// Version exposes version of database
-func (migrator *Migrator) Version() int {
-	return migrator.currentDBVersion
-}
-
-// Edition exposes edition of portainer
-func (migrator *Migrator) Edition() portainer.SoftwareEdition {
-	return migrator.currentEdition
-}
-
-// Migrate helper to upgrade DB
-func (migrator *Migrator) Migrate(version int) error {
-
-	migrateLog.Info(fmt.Sprintf("Migrating %s database from version %d to %d.", migrator.Edition().GetEditionLabel(), migrator.currentDBVersion, version))
-	// TODO : run backup before migration and restore if failed
-	err := migrator.MigrateCE() //CE
-	if err != nil {
-		migrateLog.Error("An error occurred during database migration", err)
-		return err
+// Migrate checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) Migrate() error {
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return err
+		}
 	}
 
-	migrator.versionService.StoreDBVersion(version)
-	migrator.currentDBVersion = version
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return err
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return err
+		}
+	}
 
-	return nil
-}
-
-// RollbackVersion rolls back the db to version
-func (migrator *Migrator) RollbackVersion(version int) error {
-
-	err := migrator.versionService.StoreDBVersion(version) // portainer.DBVersion
-	return err
-}
-
-// RollbackEdition rolls back the db to portainer CE
-func (migrator *Migrator) RollbackEdition(edition portainer.SoftwareEdition) error {
-
-	err := migrator.versionService.StoreEdition(portainer.PortainerCE)
-	return err
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.23.0
+	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+	if m.currentDBVersion < 22 {
+		err := m.updateResourceControlsToDBVersion22()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateUsersAndRolesToDBVersion22()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.0
+	if m.currentDBVersion < 23 {
+		err := m.updateTagsToDBVersion23()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.1
+	if m.currentDBVersion < 24 {
+		err := m.updateSettingsToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.0.0
+	if m.currentDBVersion < 25 {
+		err := m.updateSettingsToDB25()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/migrator/migrator_test.go

@@ -1,4 +0,0 @@
-package migrator
-
-// test CE version is always upgraded to latest version of CE
-// test EE version is always upgraded to latest version of EE

api/bolt/migrator/upgrade_ee.go

@@ -1,228 +0,0 @@
-package migrator
-
-import (
-	"fmt"
-
-	portainer "github.com/portainer/portainer/api"
-
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-// UpgradeToEE will migrate the db from latest ce version to latest ee version
-// Latest version is v25 on 06/11/2020
-func (m *Migrator) UpgradeToEE() error {
-
-	migrateLog.Info(fmt.Sprintf("Migrating CE database version %d to EE database version %d.", m.Version(), portainer.DBVersion))
-
-	migrateLog.Info("Updating LDAP settings to EE")
-	err := m.updateSettingsToEE()
-	if err != nil {
-		return err
-	}
-
-	migrateLog.Info("Updating user roles to EE")
-	err = m.updateUserRolesToEE()
-	if err != nil {
-		return err
-	}
-	migrateLog.Info("Updating role authorizations to EE")
-	err = m.updateRoleAuthorizationsToEE()
-	if err != nil {
-		return err
-	}
-	migrateLog.Info("Updating user authorizations")
-	err = m.authorizationService.UpdateUsersAuthorizations()
-	if err != nil {
-		return err
-	}
-
-	migrateLog.Info(fmt.Sprintf("Setting db version to %d", portainer.DBVersionEE))
-	err = m.versionService.StoreDBVersion(portainer.DBVersionEE)
-	if err != nil {
-		return err
-	}
-
-	migrateLog.Info(fmt.Sprintf("Setting edition to %s", portainer.PortainerEE.GetEditionLabel()))
-	err = m.versionService.StoreEdition(portainer.PortainerEE)
-	if err != nil {
-		return err
-	}
-
-	m.currentDBVersion = portainer.DBVersionEE
-	m.currentEdition = portainer.PortainerEE
-
-	return nil
-}
-
-func (m *Migrator) updateSettingsToEE() error {
-	legacySettings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	legacySettings.LDAPSettings.URLs = []string{}
-	url := legacySettings.LDAPSettings.URL
-	if url != "" {
-		legacySettings.LDAPSettings.URLs = append(legacySettings.LDAPSettings.URLs, url)
-	}
-
-	legacySettings.LDAPSettings.ServerType = portainer.LDAPServerCustom
-
-	return m.settingsService.UpdateSettings(legacySettings)
-}
-
-// Updating role authorizations because of the new policies in Kube RBAC
-func (m *Migrator) updateRoleAuthorizationsToEE() error {
-	migrateLog.Debug("Retriving settings")
-
-	migrateLog.Debug("Updating Endpoint Admin Role")
-	endpointAdministratorRole, err := m.roleService.Role(portainer.RoleID(1))
-	if err != nil {
-		return err
-	}
-	endpointAdministratorRole.Priority = 1
-	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
-
-	err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
-
-	migrateLog.Debug("Updating Help Desk Role")
-	helpDeskRole, err := m.roleService.Role(portainer.RoleID(2))
-	if err != nil {
-		return err
-	}
-	helpDeskRole.Priority = 2
-	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole()
-
-	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
-
-	migrateLog.Debug("Updating Standard User Role")
-	standardUserRole, err := m.roleService.Role(portainer.RoleID(3))
-	if err != nil {
-		return err
-	}
-	standardUserRole.Priority = 3
-	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole()
-
-	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
-
-	migrateLog.Debug("Updating Read Only User Role")
-	readOnlyUserRole, err := m.roleService.Role(portainer.RoleID(4))
-	if err != nil {
-		return err
-	}
-	readOnlyUserRole.Priority = 4
-	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole()
-
-	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// If RBAC extension wasn't installed before, update all users in endpoints and
-// endpoint groups to have read only access.
-func (m *Migrator) updateUserRolesToEE() error {
-	err := m.updateUserAuthorizationToEE()
-	if err != nil {
-		return err
-	}
-
-	migrateLog.Debug("Retriving extension info")
-	extensions, err := m.extensionService.Extensions()
-	for _, extension := range extensions {
-		if extension.ID == 3 && extension.Enabled {
-			migrateLog.Info("RBAC extensions were enabled before; Skip updating User Roles")
-			return nil
-		}
-	}
-
-	migrateLog.Debug("Retriving endpoint groups")
-	endpointGroups, err := m.endpointGroupService.EndpointGroups()
-	if err != nil {
-		return err
-	}
-
-	for _, endpointGroup := range endpointGroups {
-		migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint group %v", endpointGroup.ID))
-		for key := range endpointGroup.UserAccessPolicies {
-			updateUserAccessPolicyToReadOnlyRole(endpointGroup.UserAccessPolicies, key)
-		}
-
-		for key := range endpointGroup.TeamAccessPolicies {
-			updateTeamAccessPolicyToReadOnlyRole(endpointGroup.TeamAccessPolicies, key)
-		}
-
-		err := m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
-		if err != nil {
-			return err
-		}
-	}
-
-	migrateLog.Debug("Retriving endpoints")
-	endpoints, err := m.endpointService.Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for _, endpoint := range endpoints {
-		migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint %v", endpoint.ID))
-		for key := range endpoint.UserAccessPolicies {
-			updateUserAccessPolicyToReadOnlyRole(endpoint.UserAccessPolicies, key)
-		}
-
-		for key := range endpoint.TeamAccessPolicies {
-			updateTeamAccessPolicyToReadOnlyRole(endpoint.TeamAccessPolicies, key)
-		}
-
-		err := m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (m *Migrator) updateUserAuthorizationToEE() error {
-	legacyUsers, err := m.userService.Users()
-	if err != nil {
-		return err
-	}
-
-	for _, user := range legacyUsers {
-		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
-
-		err = m.userService.UpdateUser(user.ID, &user)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func updateUserAccessPolicyToNoRole(policies portainer.UserAccessPolicies, key portainer.UserID) {
-	tmp := policies[key]
-	tmp.RoleID = 0
-	policies[key] = tmp
-}
-
-func updateTeamAccessPolicyToNoRole(policies portainer.TeamAccessPolicies, key portainer.TeamID) {
-	tmp := policies[key]
-	tmp.RoleID = 0
-	policies[key] = tmp
-}
-
-func updateUserAccessPolicyToReadOnlyRole(policies portainer.UserAccessPolicies, key portainer.UserID) {
-	tmp := policies[key]
-	tmp.RoleID = 4
-	policies[key] = tmp
-}
-
-func updateTeamAccessPolicyToReadOnlyRole(policies portainer.TeamAccessPolicies, key portainer.TeamID) {
-	tmp := policies[key]
-	tmp.RoleID = 4
-	policies[key] = tmp
-}

api/bolt/registry/registry.go

@@ -1,7 +1,7 @@
 package registry
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 	var registry portainer.Registry
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &registry)
+	err := internal.GetObject(service.db, BucketName, identifier, &registry)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 func (service *Service) Registries() ([]portainer.Registry, error) {
 	var registries = make([]portainer.Registry, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 
 // CreateRegistry creates a new registry.
 func (service *Service) CreateRegistry(registry *portainer.Registry) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateRegistry(registry *portainer.Registry) error {
 // UpdateRegistry updates an registry.
 func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, registry)
+	return internal.UpdateObject(service.db, BucketName, identifier, registry)
 }
 
 // DeleteRegistry deletes an registry.
 func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,7 +1,7 @@
 package resourcecontrol
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 	var resourceControl portainer.ResourceControl
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &resourceControl)
+	err := internal.GetObject(service.db, BucketName, identifier, &resourceControl)
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +48,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -82,7 +82,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -121,11 +121,11 @@ func (service *Service) CreateResourceControl(resourceControl *portainer.Resourc
 // UpdateResourceControl saves a ResourceControl object.
 func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, resourceControl)
+	return internal.UpdateObject(service.db, BucketName, identifier, resourceControl)
 }
 
 // DeleteResourceControl deletes a ResourceControl object by ID
 func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/role/predefined_roles.go

@@ -1,68 +0,0 @@
-package role
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-// CreateOrUpdatePredefinedRoles update the predefined roles. Create one if it does not exist yet.
-func (service *Service) CreateOrUpdatePredefinedRoles() error {
-	predefinedRoles := map[portainer.RoleID]*portainer.Role{
-		portainer.RoleIDEndpointAdmin: &portainer.Role{
-			Name:           "Endpoint administrator",
-			Description:    "Full control of all resources in an endpoint",
-			ID:             portainer.RoleIDEndpointAdmin,
-			Priority:       1,
-			Authorizations: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
-		},
-		portainer.RoleIDOperator: &portainer.Role{
-			Name:           "Operator",
-			Description:    "Operational control of all existing resources in an endpoint",
-			ID:             portainer.RoleIDOperator,
-			Priority:       2,
-			Authorizations: authorization.DefaultEndpointAuthorizationsForOperatorRole(),
-		},
-		portainer.RoleIDHelpdesk: &portainer.Role{
-			Name:           "Helpdesk",
-			Description:    "Read-only access of all resources in an endpoint",
-			ID:             portainer.RoleIDHelpdesk,
-			Priority:       3,
-			Authorizations: authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
-		},
-		portainer.RoleIDStandardUser: &portainer.Role{
-			Name:           "Standard user",
-			Description:    "Full control of assigned resources in an endpoint",
-			ID:             portainer.RoleIDStandardUser,
-			Priority:       4,
-			Authorizations: authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
-		},
-		portainer.RoleIDReadonly: &portainer.Role{
-			Name:           "Read-only user",
-			Description:    "Read-only access of assigned resources in an endpoint",
-			ID:             portainer.RoleIDReadonly,
-			Priority:       5,
-			Authorizations: authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
-		},
-	}
-
-	for roleID, predefinedRole := range predefinedRoles {
-		_, err := service.Role(roleID)
-
-		if err == errors.ErrObjectNotFound {
-			err := service.CreateRole(predefinedRole)
-			if err != nil {
-				return err
-			}
-		} else if err != nil {
-			return err
-		} else {
-			err = service.UpdateRole(predefinedRole.ID, predefinedRole)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

api/bolt/role/role.go

@@ -1,7 +1,7 @@
 package role
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &set)
+	err := internal.GetObject(service.db, BucketName, identifier, &set)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 func (service *Service) Roles() ([]portainer.Role, error) {
 	var sets = make([]portainer.Role, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,13 +67,11 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 
 // CreateRole creates a new Role.
 func (service *Service) CreateRole(role *portainer.Role) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
-		if role.ID == 0 {
-			role.ID = portainer.RoleID(id)
-		}
+		role.ID = portainer.RoleID(id)
 
 		data, err := internal.MarshalObject(role)
 		if err != nil {
@@ -87,5 +85,5 @@ func (service *Service) CreateRole(role *portainer.Role) error {
 // UpdateRole updates a role.
 func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, role)
+	return internal.UpdateObject(service.db, BucketName, identifier, role)
 }

api/bolt/s3backup/s3backup.go

@@ -1,66 +0,0 @@
-package s3backup
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/bolt/internal"
-)
-
-const (
-	bucketName  = "s3backup"
-	statusKey   = "lastRunStatus"
-	settingsKey = "settings"
-)
-
-type Service struct {
-	connection *internal.DbConnection
-}
-
-// NewService creates a new service and ensures corresponding bucket exist
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, bucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// GetStatus returns the status of the last scheduled backup run
-func (s *Service) GetStatus() (portainer.S3BackupStatus, error) {
-	var status portainer.S3BackupStatus
-	err := internal.GetObject(s.connection, bucketName, []byte(statusKey), &status)
-	if err == errors.ErrObjectNotFound {
-		return status, nil
-	}
-
-	return status, err
-}
-
-// DropStatus deletes the status of the last sheduled backup run
-func (s *Service) DropStatus() error {
-	return internal.DeleteObject(s.connection, bucketName, []byte(statusKey))
-}
-
-// UpdateStatus upserts a status of the last scheduled backup run
-func (s *Service) UpdateStatus(status portainer.S3BackupStatus) error {
-	return internal.UpdateObject(s.connection, bucketName, []byte(statusKey), status)
-}
-
-// UpdateSettings updates stored s3 backup settings
-func (s *Service) UpdateSettings(settings portainer.S3BackupSettings) error {
-	return internal.UpdateObject(s.connection, bucketName, []byte(settingsKey), settings)
-}
-
-// GetSettings returns stored s3 backup settings
-func (s *Service) GetSettings() (portainer.S3BackupSettings, error) {
-	var settings portainer.S3BackupSettings
-	err := internal.GetObject(s.connection, bucketName, []byte(settingsKey), &settings)
-	if err == errors.ErrObjectNotFound {
-		return settings, nil
-	}
-
-	return settings, err
-}

api/bolt/schedule/schedule.go

@@ -1,7 +1,7 @@
 package schedule
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing schedule data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 	var schedule portainer.Schedule
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &schedule)
+	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 // UpdateSchedule updates a schedule.
 func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, schedule)
+	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
 func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -82,7 +82,7 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -105,7 +105,7 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 
 // CreateSchedule assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for schedules
@@ -125,5 +125,5 @@ func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
 
 // GetNextIdentifier returns the next identifier for a schedule.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/services.go

@@ -1,287 +0,0 @@
-package bolt
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/customtemplate"
-	"github.com/portainer/portainer/api/bolt/dockerhub"
-	"github.com/portainer/portainer/api/bolt/edgegroup"
-	"github.com/portainer/portainer/api/bolt/edgejob"
-	"github.com/portainer/portainer/api/bolt/edgestack"
-	"github.com/portainer/portainer/api/bolt/endpoint"
-	"github.com/portainer/portainer/api/bolt/endpointgroup"
-	"github.com/portainer/portainer/api/bolt/endpointrelation"
-	"github.com/portainer/portainer/api/bolt/extension"
-	"github.com/portainer/portainer/api/bolt/license"
-	"github.com/portainer/portainer/api/bolt/registry"
-	"github.com/portainer/portainer/api/bolt/resourcecontrol"
-	"github.com/portainer/portainer/api/bolt/role"
-	"github.com/portainer/portainer/api/bolt/s3backup"
-	"github.com/portainer/portainer/api/bolt/schedule"
-	"github.com/portainer/portainer/api/bolt/settings"
-	"github.com/portainer/portainer/api/bolt/stack"
-	"github.com/portainer/portainer/api/bolt/tag"
-	"github.com/portainer/portainer/api/bolt/team"
-	"github.com/portainer/portainer/api/bolt/teammembership"
-	"github.com/portainer/portainer/api/bolt/tunnelserver"
-	"github.com/portainer/portainer/api/bolt/user"
-	"github.com/portainer/portainer/api/bolt/version"
-	"github.com/portainer/portainer/api/bolt/webhook"
-)
-
-func (store *Store) initServices() error {
-	authorizationsetService, err := role.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.RoleService = authorizationsetService
-
-	customTemplateService, err := customtemplate.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.CustomTemplateService = customTemplateService
-
-	dockerhubService, err := dockerhub.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.DockerHubService = dockerhubService
-
-	edgeStackService, err := edgestack.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeStackService = edgeStackService
-
-	edgeGroupService, err := edgegroup.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeGroupService = edgeGroupService
-
-	edgeJobService, err := edgejob.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeJobService = edgeJobService
-
-	endpointgroupService, err := endpointgroup.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointGroupService = endpointgroupService
-
-	endpointService, err := endpoint.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointService = endpointService
-
-	endpointRelationService, err := endpointrelation.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointRelationService = endpointRelationService
-
-	extensionService, err := extension.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ExtensionService = extensionService
-
-	licenseService, err := license.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.LicenseService = licenseService
-
-	registryService, err := registry.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.RegistryService = registryService
-
-	resourcecontrolService, err := resourcecontrol.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ResourceControlService = resourcecontrolService
-
-	s3backupService, err := s3backup.NewService(store.connection)
-	if err != nil {
-		return nil
-	}
-	store.S3BackupService = s3backupService
-
-	settingsService, err := settings.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.SettingsService = settingsService
-
-	stackService, err := stack.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.StackService = stackService
-
-	tagService, err := tag.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TagService = tagService
-
-	teammembershipService, err := teammembership.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TeamMembershipService = teammembershipService
-
-	teamService, err := team.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TeamService = teamService
-
-	tunnelServerService, err := tunnelserver.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TunnelServerService = tunnelServerService
-
-	userService, err := user.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.UserService = userService
-
-	versionService, err := version.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.VersionService = versionService
-
-	webhookService, err := webhook.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.WebhookService = webhookService
-
-	scheduleService, err := schedule.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ScheduleService = scheduleService
-
-	return nil
-}
-
-// CustomTemplate gives access to the CustomTemplate data management layer
-func (store *Store) CustomTemplate() portainer.CustomTemplateService {
-	return store.CustomTemplateService
-}
-
-// DockerHub gives access to the DockerHub data management layer
-func (store *Store) DockerHub() portainer.DockerHubService {
-	return store.DockerHubService
-}
-
-// EdgeGroup gives access to the EdgeGroup data management layer
-func (store *Store) EdgeGroup() portainer.EdgeGroupService {
-	return store.EdgeGroupService
-}
-
-// EdgeJob gives access to the EdgeJob data management layer
-func (store *Store) EdgeJob() portainer.EdgeJobService {
-	return store.EdgeJobService
-}
-
-// EdgeStack gives access to the EdgeStack data management layer
-func (store *Store) EdgeStack() portainer.EdgeStackService {
-	return store.EdgeStackService
-}
-
-// Endpoint gives access to the Endpoint data management layer
-func (store *Store) Endpoint() portainer.EndpointService {
-	return store.EndpointService
-}
-
-// EndpointGroup gives access to the EndpointGroup data management layer
-func (store *Store) EndpointGroup() portainer.EndpointGroupService {
-	return store.EndpointGroupService
-}
-
-// EndpointRelation gives access to the EndpointRelation data management layer
-func (store *Store) EndpointRelation() portainer.EndpointRelationService {
-	return store.EndpointRelationService
-}
-
-// License provides access to the License data management layer
-func (store *Store) License() portainer.LicenseRepository {
-	return store.LicenseService
-}
-
-// Registry gives access to the Registry data management layer
-func (store *Store) Registry() portainer.RegistryService {
-	return store.RegistryService
-}
-
-// ResourceControl gives access to the ResourceControl data management layer
-func (store *Store) ResourceControl() portainer.ResourceControlService {
-	return store.ResourceControlService
-}
-
-// Role gives access to the Role data management layer
-func (store *Store) Role() portainer.RoleService {
-	return store.RoleService
-}
-
-// S3Backup gives access to S3 backup settings and status
-func (store *Store) S3Backup() portainer.S3BackupService {
-	return store.S3BackupService
-}
-
-// Settings gives access to the Settings data management layer
-func (store *Store) Settings() portainer.SettingsService {
-	return store.SettingsService
-}
-
-// Stack gives access to the Stack data management layer
-func (store *Store) Stack() portainer.StackService {
-	return store.StackService
-}
-
-// Tag gives access to the Tag data management layer
-func (store *Store) Tag() portainer.TagService {
-	return store.TagService
-}
-
-// TeamMembership gives access to the TeamMembership data management layer
-func (store *Store) TeamMembership() portainer.TeamMembershipService {
-	return store.TeamMembershipService
-}
-
-// Team gives access to the Team data management layer
-func (store *Store) Team() portainer.TeamService {
-	return store.TeamService
-}
-
-// TunnelServer gives access to the TunnelServer data management layer
-func (store *Store) TunnelServer() portainer.TunnelServerService {
-	return store.TunnelServerService
-}
-
-// User gives access to the User data management layer
-func (store *Store) User() portainer.UserService {
-	return store.UserService
-}
-
-// Version gives access to the Version data management layer
-func (store *Store) Version() portainer.VersionService {
-	return store.VersionService
-}
-
-// Webhook gives access to the Webhook data management layer
-func (store *Store) Webhook() portainer.WebhookService {
-	return store.WebhookService
-}

api/bolt/settings/settings.go

@@ -1,8 +1,10 @@
 package settings
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -13,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings
 
-	err := internal.GetObject(service.connection, BucketName, []byte(settingsKey), &settings)
+	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(settingsKey), settings)
+	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), settings)
 }

api/bolt/stack/stack.go

@@ -1,7 +1,7 @@
 package stack
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -35,7 +35,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 	var stack portainer.Stack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	err := internal.GetObject(service.db, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -47,7 +47,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 	var stack *portainer.Stack
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -78,7 +78,7 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) Stacks() ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -99,12 +99,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) CreateStack(stack *portainer.Stack) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for stacks
@@ -125,11 +125,11 @@ func (service *Service) CreateStack(stack *portainer.Stack) error {
 // UpdateStack updates a stack.
 func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, stack)
+	return internal.UpdateObject(service.db, BucketName, identifier, stack)
 }
 
 // DeleteStack deletes a stack.
 func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/tag/tag.go

@@ -1,7 +1,7 @@
 package tag
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,7 +57,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 	var tag portainer.Tag
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &tag)
+	err := internal.GetObject(service.db, BucketName, identifier, &tag)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +67,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 
 // CreateTag creates a new tag.
 func (service *Service) CreateTag(tag *portainer.Tag) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateTag(tag *portainer.Tag) error {
 // UpdateTag updates a tag.
 func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, tag)
+	return internal.UpdateObject(service.db, BucketName, identifier, tag)
 }
 
 // DeleteTag deletes a tag.
 func (service *Service) DeleteTag(ID portainer.TagID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/team/team.go

@@ -1,11 +1,11 @@
 package team
 
 import (
-	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
-	"strings"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -35,7 +35,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 	var team portainer.Team
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &team)
+	err := internal.GetObject(service.db, BucketName, identifier, &team)
 	if err != nil {
 		return nil, err
 	}
@@ -47,7 +47,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	var team *portainer.Team
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -58,7 +58,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 				return err
 			}
 
-			if strings.EqualFold(t.Name, name) {
+			if t.Name == name {
 				team = &t
 				break
 			}
@@ -78,7 +78,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 func (service *Service) Teams() ([]portainer.Team, error) {
 	var teams = make([]portainer.Team, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -100,12 +100,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 // UpdateTeam saves a Team.
 func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, team)
+	return internal.UpdateObject(service.db, BucketName, identifier, team)
 }
 
 // CreateTeam creates a new Team.
 func (service *Service) CreateTeam(team *portainer.Team) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -123,5 +123,5 @@ func (service *Service) CreateTeam(team *portainer.Team) error {
 // DeleteTeam deletes a Team.
 func (service *Service) DeleteTeam(ID portainer.TeamID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/teammembership/teammembership.go

@@ -1,7 +1,7 @@
 package teammembership
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 	var membership portainer.TeamMembership
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &membership)
+	err := internal.GetObject(service.db, BucketName, identifier, &membership)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -69,7 +69,7 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -95,7 +95,7 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -120,12 +120,12 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 // UpdateTeamMembership saves a TeamMembership object.
 func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, membership)
+	return internal.UpdateObject(service.db, BucketName, identifier, membership)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) CreateTeamMembership(membership *portainer.TeamMembership) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -143,12 +143,12 @@ func (service *Service) CreateTeamMembership(membership *portainer.TeamMembershi
 // DeleteTeamMembership deletes a TeamMembership object.
 func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -173,7 +173,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()

api/bolt/tunnelserver/tunnelserver.go

@@ -1,8 +1,10 @@
 package tunnelserver
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -13,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 	var info portainer.TunnelServerInfo
 
-	err := internal.GetObject(service.connection, BucketName, []byte(infoKey), &info)
+	err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 
 // UpdateInfo persists a TunnelServerInfo object.
 func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(infoKey), settings)
+	return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
 }

api/bolt/user/user.go

@@ -1,11 +1,11 @@
 package user
 
 import (
-	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
-	"strings"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -35,7 +35,7 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 	var user portainer.User
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &user)
+	err := internal.GetObject(service.db, BucketName, identifier, &user)
 	if err != nil {
 		return nil, err
 	}
@@ -47,9 +47,7 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
 	var user *portainer.User
 
-	username = strings.ToLower(username)
-
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -60,7 +58,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 				return err
 			}
 
-			if strings.EqualFold(u.Username, username) {
+			if u.Username == username {
 				user = &u
 				break
 			}
@@ -79,7 +77,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 func (service *Service) Users() ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -101,7 +99,7 @@ func (service *Service) Users() ([]portainer.User, error) {
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -125,18 +123,16 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 // UpdateUser saves a user.
 func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
 	identifier := internal.Itob(int(ID))
-	user.Username = strings.ToLower(user.Username)
-	return internal.UpdateObject(service.connection, BucketName, identifier, user)
+	return internal.UpdateObject(service.db, BucketName, identifier, user)
 }
 
 // CreateUser creates a new user.
 func (service *Service) CreateUser(user *portainer.User) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
 		user.ID = portainer.UserID(id)
-		user.Username = strings.ToLower(user.Username)
 
 		data, err := internal.MarshalObject(user)
 		if err != nil {
@@ -150,5 +146,5 @@ func (service *Service) CreateUser(user *portainer.User) error {
 // DeleteUser deletes a user.
 func (service *Service) DeleteUser(ID portainer.UserID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/version/version.go

@@ -4,109 +4,42 @@ import (
 	"strconv"
 
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
 	// BucketName represents the name of the bucket where this service stores data.
-	BucketName         = "version"
-	versionKey         = "DB_VERSION"
-	previousVersionKey = "PREVIOUS_DB_VERSION"
-	instanceKey        = "INSTANCE_ID"
-	editionKey         = "EDITION"
+	BucketName  = "version"
+	versionKey  = "DB_VERSION"
+	instanceKey = "INSTANCE_ID"
 )
 
 // Service represents a service to manage stored versions.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
-// Edition retrieves the stored portainer edition.
-func (service *Service) Edition() (portainer.SoftwareEdition, error) {
-	editionData, err := service.getKey(editionKey)
-	if err != nil {
-		return 0, err
-	}
-
-	edition, err := strconv.Atoi(string(editionData))
-	if err != nil {
-		return 0, err
-	}
-
-	return portainer.SoftwareEdition(edition), nil
-}
-
-// StoreEdition store the portainer edition.
-func (service *Service) StoreEdition(edition portainer.SoftwareEdition) error {
-	return service.setKey(editionKey, strconv.Itoa(int(edition)))
-}
-
-// PreviousDBVersion retrieves the stored database version.
-func (service *Service) PreviousDBVersion() (int, error) {
-	version, err := service.getKey(previousVersionKey)
-	if err != nil {
-		return 0, err
-	}
-
-	return strconv.Atoi(string(version))
-}
-
 // DBVersion retrieves the stored database version.
 func (service *Service) DBVersion() (int, error) {
-	version, err := service.getKey(versionKey)
-	if err != nil {
-		return 0, err
-	}
-
-	return strconv.Atoi(string(version))
-}
-
-// StorePreviousDBVersion store the database version.
-func (service *Service) StorePreviousDBVersion(version int) error {
-	return service.setKey(previousVersionKey, strconv.Itoa(version))
-}
-
-// StoreDBVersion store the database version.
-func (service *Service) StoreDBVersion(version int) error {
-	return service.setKey(versionKey, strconv.Itoa(version))
-}
-
-// InstanceID retrieves the stored instance ID.
-func (service *Service) InstanceID() (string, error) {
-	instanceID, err := service.getKey(instanceKey)
-	if err != nil {
-		return "", err
-	}
-
-	return string(instanceID), nil
-}
-
-// StoreInstanceID store the instance ID.
-func (service *Service) StoreInstanceID(ID string) error {
-	return service.setKey(instanceKey, ID)
-}
-
-func (service *Service) getKey(key string) ([]byte, error) {
 	var data []byte
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		value := bucket.Get([]byte(key))
+		value := bucket.Get([]byte(versionKey))
 		if value == nil {
 			return errors.ErrObjectNotFound
 		}
@@ -116,19 +49,53 @@ func (service *Service) getKey(key string) ([]byte, error) {
 
 		return nil
 	})
-
 	if err != nil {
-		return nil, err
+		return 0, err
 	}
 
-	return data, nil
+	return strconv.Atoi(string(data))
 }
 
-func (service *Service) setKey(key string, value string) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+// StoreDBVersion store the database version.
+func (service *Service) StoreDBVersion(version int) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		data := []byte(value)
-		return bucket.Put([]byte(key), data)
+		data := []byte(strconv.Itoa(version))
+		return bucket.Put([]byte(versionKey), data)
+	})
+}
+
+// InstanceID retrieves the stored instance ID.
+func (service *Service) InstanceID() (string, error) {
+	var data []byte
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		value := bucket.Get([]byte(instanceKey))
+		if value == nil {
+			return errors.ErrObjectNotFound
+		}
+
+		data = make([]byte, len(value))
+		copy(data, value)
+
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}
+
+// StoreInstanceID store the instance ID.
+func (service *Service) StoreInstanceID(ID string) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data := []byte(ID)
+		return bucket.Put([]byte(instanceKey), data)
 	})
 }

api/bolt/webhook/webhook.go

@@ -1,7 +1,7 @@
 package webhook
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -58,7 +58,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 	var webhook portainer.Webhook
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &webhook)
+	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +70,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -101,7 +101,7 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -131,12 +131,12 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 // DeleteWebhook deletes a webhook.
 func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/chisel/service.go

@@ -1,7 +1,6 @@
 package chisel
 
 import (
-	"context"
 	"fmt"
 	"log"
 	"strconv"
@@ -10,7 +9,7 @@ import (
 	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
 	cmap "github.com/orcaman/concurrent-map"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
@@ -30,15 +29,13 @@ type Service struct {
 	dataStore         portainer.DataStore
 	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
-	shutdownCtx       context.Context
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore portainer.DataStore, shutdownCtx context.Context) *Service {
+func NewService(dataStore portainer.DataStore) *Service {
 	return &Service{
 		tunnelDetailsMap: cmap.New(),
 		dataStore:        dataStore,
-		shutdownCtx:      shutdownCtx,
 	}
 }
 
@@ -86,11 +83,6 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	return nil
 }
 
-// StopTunnelServer stops tunnel http server
-func (service *Service) StopTunnelServer() error {
-	return service.chiselServer.Close()
-}
-
 func (service *Service) retrievePrivateKeySeed() (string, error) {
 	var serverInfo *portainer.TunnelServerInfo
 
@@ -116,16 +108,13 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 func (service *Service) startTunnelVerificationLoop() {
 	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
 	ticker := time.NewTicker(tunnelCleanupInterval)
+	stopSignal := make(chan struct{})
 
 	for {
 		select {
 		case <-ticker.C:
 			service.checkTunnels()
-		case <-service.shutdownCtx.Done():
-			log.Println("[DEBUG] Shutting down tunnel service")
-			if err := service.StopTunnelServer(); err != nil {
-				log.Printf("Stopped tunnel service: %s", err)
-			}
+		case <-stopSignal:
 			ticker.Stop()
 			return
 		}

api/cli/cli.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"time"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -42,7 +42,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TLSCacert:                 kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:                   kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
 		TLSKey:                    kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
-		RollbackToCE:              kingpin.Flag("rollback-to-ce", "Rollback the database store to CE").Bool(),
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),

api/cli/confirm.go

@@ -1,24 +0,0 @@
-package cli
-
-import (
-	"bufio"
-	"log"
-	"os"
-	"strings"
-)
-
-// Confirm starts a rollback db cli application
-func Confirm(message string) (bool, error) {
-	log.Printf("%s [y/N]", message)
-
-	reader := bufio.NewReader(os.Stdin)
-	answer, err := reader.ReadString('\n')
-	if err != nil {
-		return false, err
-	}
-	answer = strings.Replace(answer, "\n", "", -1)
-	answer = strings.ToLower(answer)
-
-	return answer == "y" || answer == "yes", nil
-
-}

api/cmd/portainer/main.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"context"
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
@@ -12,108 +12,75 @@ import (
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/docker"
-
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/http"
 	"github.com/portainer/portainer/api/http/client"
-	"github.com/portainer/portainer/api/http/proxy"
-	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
-	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/libcompose"
-	"github.com/portainer/portainer/api/license"
 	"github.com/portainer/portainer/api/oauth"
-	"github.com/portainer/portainer/api/useractivity"
 )
 
 func initCLI() *portainer.CLIFlags {
 	var cliService portainer.CLIService = &cli.Service{}
 	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
-		log.Fatalf("failed parsing flags: %s", err)
+		log.Fatal(err)
 	}
 
 	err = cliService.ValidateFlags(flags)
 	if err != nil {
-		log.Fatalf("failed validating flags:%s", err)
+		log.Fatal(err)
 	}
 	return flags
 }
 
-func initUserActivityStore(dataStorePath string) portainer.UserActivityStore {
-	store, err := useractivity.NewUserActivityStore(dataStorePath)
-	if err != nil {
-		log.Fatalf("Failed initalizing user activity store: %s", err)
-	}
-
-	return store
-}
-
 func initFileService(dataStorePath string) portainer.FileService {
 	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
-		log.Fatalf("failed creating file service: %s", err)
+		log.Fatal(err)
 	}
 	return fileService
 }
 
-func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService) portainer.DataStore {
+func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
 	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
-		log.Fatalf("failed creating data store: %s", err)
+		log.Fatal(err)
 	}
 
 	err = store.Open()
 	if err != nil {
-		log.Fatalf("failed opening store: %s", err)
+		log.Fatal(err)
 	}
 
 	err = store.Init()
 	if err != nil {
-		log.Fatalf("failed initializing data store: %s", err)
+		log.Fatal(err)
 	}
 
-	if rollback {
-		err := store.RollbackToCE()
-		if err != nil {
-			log.Fatalf("failed rolling back to CE: %s", err)
-		}
-
-		log.Println("Exiting rollback")
-		os.Exit(0)
-		return nil
-	}
-
-	err = store.MigrateData(false)
+	err = store.MigrateData()
 	if err != nil {
-		log.Fatalf("failed migration: %s", err)
+		log.Fatal(err)
 	}
 	return store
 }
 
-func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
-	if err != nil {
-		log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err)
-		return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
-	}
-
-	return composeWrapper
+func initComposeStackManager(dataStorePath string, reverseTunnelService portainer.ReverseTunnelService) portainer.ComposeStackManager {
+	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 }
 
 func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
 	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath)
+func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(assetsPath)
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -122,11 +89,11 @@ func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error)
 		return nil, err
 	}
 
-	userSessionTimeout := settings.UserSessionTimeout
-	if userSessionTimeout == "" {
-		userSessionTimeout = portainer.DefaultUserSessionTimeout
+	if settings.UserSessionTimeout == "" {
+		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
+		dataStore.Settings().UpdateSettings(settings)
 	}
-	jwtService, err := jwt.NewService(userSessionTimeout)
+	jwtService, err := jwt.NewService(settings.UserSessionTimeout)
 	if err != nil {
 		return nil, err
 	}
@@ -161,11 +128,11 @@ func initKubernetesClientFactory(signatureService portainer.DigitalSignatureServ
 	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
 }
 
-func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
+	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter)
 	if err != nil {
 		return nil, err
 	}
@@ -173,6 +140,21 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
+func loadEdgeJobsFromDatabase(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService) error {
+	edgeJobs, err := dataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeJob := range edgeJobs {
+		for endpointID := range edgeJob.Endpoints {
+			reverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
+		}
+	}
+
+	return nil
+}
+
 func initStatus(flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
 		Version: portainer.APIVersion,
@@ -189,8 +171,6 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 	settings.SnapshotInterval = *flags.SnapshotInterval
 	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
 	settings.EnableTelemetry = true
-	settings.OAuthSettings.SSO = true
-	settings.OAuthSettings.HideInternalAuth = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -223,7 +203,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
 func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	existingKeyPair, err := fileService.KeyPairFilesExist()
 	if err != nil {
-		log.Fatalf("failed checking for existing key pair: %s", err)
+		log.Fatal(err)
 	}
 
 	if existingKeyPair {
@@ -261,19 +241,6 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},
 		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
 	}
 
 	if strings.HasPrefix(endpoint.URL, "tcp://") {
@@ -323,19 +290,6 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},
 		Kubernetes:         portainer.KubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
 	}
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
@@ -367,21 +321,32 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
-func buildServer(flags *portainer.CLIFlags) portainer.Server {
-	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
+func terminateIfNoAdminCreated(dataStore portainer.DataStore) {
+	timer1 := time.NewTimer(5 * time.Minute)
+	<-timer1.C
+
+	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(users) == 0 {
+		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
+		return
+	}
+}
+
+func main() {
+	flags := initCLI()
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, *flags.RollbackToCE, fileService)
+	dataStore := initDataStore(*flags.Data, fileService)
+	defer dataStore.Close()
 
 	jwtService, err := initJWTService(dataStore)
 	if err != nil {
-		log.Fatalf("failed initializing JWT service: %s", err)
-	}
-
-	licenseService := license.NewService(dataStore.License(), shutdownCtx)
-	if err = licenseService.Init(); err != nil {
-		log.Fatalf("failed initializing license service: %s", err)
+		log.Fatal(err)
 	}
 
 	ldapService := initLDAPService()
@@ -396,70 +361,62 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
-		log.Fatalf("failed initializing key pair: %s", err)
+		log.Fatal(err)
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+	reverseTunnelService := chisel.NewService(dataStore)
 
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
-		log.Fatalf("failed to get datastore version: %s", err)
+		log.Fatal(err)
 	}
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
 	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
 
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory)
 	if err != nil {
-		log.Fatalf("failed initializing snapshot service: %s", err)
+		log.Fatal(err)
 	}
 	snapshotService.Start()
 
-	authorizationService := authorization.NewService(dataStore)
-	authorizationService.K8sClientFactory = kubernetesClientFactory
-
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
 	if err != nil {
-		log.Fatalf("failed initializing swarm stack manager: %s", err)
+		log.Fatal(err)
 	}
-	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
-	userActivityStore := initUserActivityStore(*flags.Data)
+	composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)
 
-	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, authorizationService, userActivityStore)
-
-	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
-
-	kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
 		if err != nil {
-			log.Fatalf("failed updating settings from flags: %s", err)
+			log.Fatal(err)
 		}
 	}
 
-	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
+	err = loadEdgeJobsFromDatabase(dataStore, reverseTunnelService)
 	if err != nil {
-		log.Fatalf("failed loading edge jobs from database: %s", err)
+		log.Fatal(err)
 	}
 
 	applicationStatus := initStatus(flags)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatalf("failed initializing endpoint: %s", err)
+		log.Fatal(err)
 	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
 		content, err := fileService.GetFileContent(*flags.AdminPasswordFile)
 		if err != nil {
-			log.Fatalf("failed getting admin password file: %s", err)
+			log.Fatal(err)
 		}
 		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
-			log.Fatalf("failed hashing admin password: %s", err)
+			log.Fatal(err)
 		}
 	} else if *flags.AdminPassword != "" {
 		adminPasswordHash = *flags.AdminPassword
@@ -468,75 +425,59 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	if adminPasswordHash != "" {
 		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
-			log.Fatalf("failed getting admin user: %s", err)
+			log.Fatal(err)
 		}
 
 		if len(users) == 0 {
 			log.Println("Created admin user with the given password.")
 			user := &portainer.User{
-				Username:                "admin",
-				Role:                    portainer.AdministratorRole,
-				Password:                adminPasswordHash,
-				PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
+				Username: "admin",
+				Role:     portainer.AdministratorRole,
+				Password: adminPasswordHash,
 			}
 			err := dataStore.User().CreateUser(user)
 			if err != nil {
-				log.Fatalf("failed creating admin user: %s", err)
+				log.Fatal(err)
 			}
 		} else {
 			log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
 		}
 	}
 
+	go terminateIfNoAdminCreated(dataStore)
+
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting tunnel server: %s", err)
+		log.Fatal(err)
 	}
 
-	err = licenseService.Start()
+	var server portainer.Server = &http.Server{
+		ReverseTunnelService:    reverseTunnelService,
+		Status:                  applicationStatus,
+		BindAddress:             *flags.Addr,
+		AssetsPath:              *flags.Assets,
+		DataStore:               dataStore,
+		SwarmStackManager:       swarmStackManager,
+		ComposeStackManager:     composeStackManager,
+		KubernetesDeployer:      kubernetesDeployer,
+		CryptoService:           cryptoService,
+		JWTService:              jwtService,
+		FileService:             fileService,
+		LDAPService:             ldapService,
+		OAuthService:            oauthService,
+		GitService:              gitService,
+		SignatureService:        digitalSignatureService,
+		SnapshotService:         snapshotService,
+		SSL:                     *flags.SSL,
+		SSLCert:                 *flags.SSLCert,
+		SSLKey:                  *flags.SSLKey,
+		DockerClientFactory:     dockerClientFactory,
+		KubernetesClientFactory: kubernetesClientFactory,
+	}
+
+	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)
+	err = server.Start()
 	if err != nil {
-		log.Fatalf("failed starting license service: %s", err)
-	}
-
-	return &http.Server{
-		AuthorizationService:        authorizationService,
-		ReverseTunnelService:        reverseTunnelService,
-		Status:                      applicationStatus,
-		BindAddress:                 *flags.Addr,
-		AssetsPath:                  *flags.Assets,
-		DataStore:                   dataStore,
-		LicenseService:              licenseService,
-		SwarmStackManager:           swarmStackManager,
-		ComposeStackManager:         composeStackManager,
-		KubernetesDeployer:          kubernetesDeployer,
-		CryptoService:               cryptoService,
-		JWTService:                  jwtService,
-		FileService:                 fileService,
-		LDAPService:                 ldapService,
-		OAuthService:                oauthService,
-		GitService:                  gitService,
-		ProxyManager:                proxyManager,
-		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		SignatureService:            digitalSignatureService,
-		SnapshotService:             snapshotService,
-		SSL:                         *flags.SSL,
-		SSLCert:                     *flags.SSLCert,
-		SSLKey:                      *flags.SSLKey,
-		DockerClientFactory:         dockerClientFactory,
-		UserActivityStore:           userActivityStore,
-		KubernetesClientFactory:     kubernetesClientFactory,
-		ShutdownCtx:                 shutdownCtx,
-		ShutdownTrigger:             shutdownTrigger,
-	}
-}
-
-func main() {
-	flags := initCLI()
-
-	for {
-		server := buildServer(flags)
-		log.Printf("Starting Portainer %s on %s\n", portainer.APIVersion, *flags.Addr)
-		err := server.Start()
-		log.Printf("Http server exited: %s\n", err)
+		log.Fatal(err)
 	}
 }

api/crypto/aes.go

@@ -1,70 +0,0 @@
-package crypto
-
-import (
-	"crypto/aes"
-	"crypto/cipher"
-	"io"
-
-	"golang.org/x/crypto/scrypt"
-)
-
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
-
-var emptySalt []byte = make([]byte, 0, 0)
-
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
-func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
-	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
-func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
-	if err != nil {
-		return nil, err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	reader := &cipher.StreamReader{S: stream, R: input}
-
-	return reader, nil
-}

api/crypto/aes_test.go

@@ -1,131 +0,0 @@
-package crypto
-
-import (
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to decrypt file")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
-}
-
-func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte(""))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(""))
-	assert.Nil(t, err, "Failed to decrypt file")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
-}
-
-func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
-}

api/dev.sh

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-go run -v -ldflags="-X github.com/portainer/liblicense.LicenseServerBaseURL=http://localhost:8080" cmd/portainer/main.go --data=./tmp/data

api/docker/snapshot.go

@@ -118,7 +118,6 @@ func snapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error
 	}
 	snapshot.TotalCPU = int(nanoCpus / 1e9)
 	snapshot.TotalMemory = totalMem
-	snapshot.NodeCount = len(nodes)
 	return nil
 }
 

api/exec/compose_stack.go

@@ -1,120 +0,0 @@
-package exec
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"strings"
-
-	wrapper "github.com/portainer/docker-compose-wrapper"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy"
-	"github.com/portainer/portainer/api/http/proxy/factory"
-)
-
-// ComposeStackManager is a wrapper for docker-compose binary
-type ComposeStackManager struct {
-	wrapper      *wrapper.ComposeWrapper
-	dataPath     string
-	proxyManager *proxy.Manager
-}
-
-// NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeStackManager(binaryPath string, dataPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
-	wrap, err := wrapper.NewComposeWrapper(binaryPath)
-	if err != nil {
-		return nil, err
-	}
-
-	return &ComposeStackManager{
-		wrapper:      wrap,
-		proxyManager: proxyManager,
-		dataPath:     dataPath,
-	}, nil
-}
-
-// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
-func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
-	return portainer.ComposeSyntaxMaxVersion
-}
-
-// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
-func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	url, proxy, err := w.fetchEndpointProxy(endpoint)
-	if err != nil {
-		return err
-	}
-
-	if proxy != nil {
-		defer proxy.Close()
-	}
-
-	envFilePath, err := createEnvFile(stack)
-	if err != nil {
-		return err
-	}
-
-	filePath := stackFilePath(stack)
-
-	_, err = w.wrapper.Up(filePath, url, stack.Name, envFilePath, w.dataPath)
-	return err
-}
-
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
-func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	url, proxy, err := w.fetchEndpointProxy(endpoint)
-	if err != nil {
-		return err
-	}
-	if proxy != nil {
-		defer proxy.Close()
-	}
-
-	filePath := stackFilePath(stack)
-
-	_, err = w.wrapper.Down(filePath, url, stack.Name)
-	return err
-}
-
-// NormalizeStackName returns the passed stack name, for interface implementation only
-func (w *ComposeStackManager) NormalizeStackName(name string) string {
-	return name
-}
-
-func stackFilePath(stack *portainer.Stack) string {
-	return path.Join(stack.ProjectPath, stack.EntryPoint)
-}
-
-func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
-	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
-		return "", nil, nil
-	}
-
-	proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return fmt.Sprintf("http://127.0.0.1:%d", proxy.Port), proxy, nil
-}
-
-func createEnvFile(stack *portainer.Stack) (string, error) {
-	if stack.Env == nil || len(stack.Env) == 0 {
-		return "", nil
-	}
-
-	envFilePath := path.Join(stack.ProjectPath, "stack.env")
-
-	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return "", err
-	}
-
-	for _, v := range stack.Env {
-		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
-	}
-	envfile.Close()
-
-	return envFilePath, nil
-}

api/exec/compose_stack_integration_test.go

@@ -1,79 +0,0 @@
-// +build integration
-package exec
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-const composeFile = `version: "3.9"
-services:
-  busybox:
-    image: "alpine:latest"
-    container_name: "compose_wrapper_test"`
-const composedContainerName = "compose_wrapper_test"
-
-func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
-	dir := t.TempDir()
-	composeFileName := "compose_wrapper_test.yml"
-	f, _ := os.Create(filepath.Join(dir, composeFileName))
-	f.WriteString(composeFile)
-
-	stack := &portainer.Stack{
-		ProjectPath: dir,
-		EntryPoint:  composeFileName,
-		Name:        "project-name",
-	}
-
-	endpoint := &portainer.Endpoint{
-		URL: "unix://",
-	}
-
-	return stack, endpoint
-}
-
-func Test_UpAndDown(t *testing.T) {
-
-	stack, endpoint := setup(t)
-
-	w, err := NewComposeStackManager("", "", nil)
-	if err != nil {
-		t.Fatalf("Failed creating manager: %s", err)
-	}
-
-	err = w.Up(stack, endpoint)
-	if err != nil {
-		t.Fatalf("Error calling docker-compose up: %s", err)
-	}
-
-	if !containerExists(composedContainerName) {
-		t.Fatal("container should exist")
-	}
-
-	err = w.Down(stack, endpoint)
-	if err != nil {
-		t.Fatalf("Error calling docker-compose down: %s", err)
-	}
-
-	if containerExists(composedContainerName) {
-		t.Fatal("container should be removed")
-	}
-}
-
-func containerExists(containerName string) bool {
-	cmd := exec.Command("docker", "ps", "-a", "-f", fmt.Sprintf("name=%s", containerName))
-
-	out, err := cmd.Output()
-	if err != nil {
-		log.Fatalf("failed to list containers: %s", err)
-	}
-
-	return strings.Contains(string(out), containerName)
-}

api/exec/compose_stack_test.go

@@ -1,112 +0,0 @@
-package exec
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_stackFilePath(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected string
-	}{
-		// {
-		// 	name:     "should return empty result if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
-		// {
-		// 	name:     "should return empty result if stack don't have entrypoint",
-		// 	stack:    &portainer.Stack{},
-		// 	expected: "",
-		// },
-		{
-			name: "should allow file name and dir",
-			stack: &portainer.Stack{
-				ProjectPath: "dir",
-				EntryPoint:  "file",
-			},
-			expected: path.Join("dir", "file"),
-		},
-		{
-			name: "should allow file name only",
-			stack: &portainer.Stack{
-				EntryPoint: "file",
-			},
-			expected: "file",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := stackFilePath(tt.stack)
-			assert.Equal(t, tt.expected, result)
-		})
-	}
-}
-
-func Test_createEnvFile(t *testing.T) {
-	dir := t.TempDir()
-
-	tests := []struct {
-		name         string
-		stack        *portainer.Stack
-		expected     string
-		expectedFile bool
-	}{
-		// {
-		// 	name:     "should not add env file option if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
-		{
-			name: "should not add env file option if stack doesn't have env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-			},
-			expected: "",
-		},
-		{
-			name: "should not add env file option if stack's env variables are empty",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env:         []portainer.Pair{},
-			},
-			expected: "",
-		},
-		{
-			name: "should add env file option if stack has env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env: []portainer.Pair{
-					{Name: "var1", Value: "value1"},
-					{Name: "var2", Value: "value2"},
-				},
-			},
-			expected: "var1=value1\nvar2=value2\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result, _ := createEnvFile(tt.stack)
-
-			if tt.expected != "" {
-				assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result)
-
-				f, _ := os.Open(path.Join(dir, "stack.env"))
-				content, _ := ioutil.ReadAll(f)
-
-				assert.Equal(t, tt.expected, string(content))
-			} else {
-				assert.Equal(t, "", result)
-			}
-		})
-	}
-}

api/exec/kubernetes_deploy.go

@@ -2,188 +2,71 @@ package exec
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
-	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath           string
-	dataStore            portainer.DataStore
-	reverseTunnelService portainer.ReverseTunnelService
-	signatureService     portainer.DigitalSignatureService
+	binaryPath string
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath:           binaryPath,
-		dataStore:            datastore,
-		reverseTunnelService: reverseTunnelService,
-		signatureService:     signatureService,
+		binaryPath: binaryPath,
 	}
 }
 
 // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
+// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
 // Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
-	if endpoint.Type == portainer.KubernetesLocalEnvironment {
-		token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+	if composeFormat {
+		convertedData, err := deployer.convertComposeData(data)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
-
-		command := path.Join(deployer.binaryPath, "kubectl")
-		if runtime.GOOS == "windows" {
-			command = path.Join(deployer.binaryPath, "kubectl.exe")
-		}
-
-		args := make([]string, 0)
-		args = append(args, "--server", endpoint.URL)
-		args = append(args, "--insecure-skip-tls-verify")
-		args = append(args, "--token", string(token))
-		args = append(args, "--namespace", namespace)
-		args = append(args, "apply", "-f", "-")
-
-		var stderr bytes.Buffer
-		cmd := exec.Command(command, args...)
-		cmd.Stderr = &stderr
-		cmd.Stdin = strings.NewReader(stackConfig)
-
-		output, err := cmd.Output()
-		if err != nil {
-			return "", errors.New(stderr.String())
-		}
-
-		return string(output), nil
+		data = string(convertedData)
 	}
 
-	// agent
-
-	endpointURL := endpoint.URL
-	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
-		if tunnel.Status == portainer.EdgeAgentIdle {
-
-			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
-			if err != nil {
-				return "", err
-			}
-
-			settings, err := deployer.dataStore.Settings().Settings()
-			if err != nil {
-				return "", err
-			}
-
-			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
-			time.Sleep(waitForAgentToConnect * 2)
-		}
-
-		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
-	}
-
-	transport := &http.Transport{}
-
-	if endpoint.TLSConfig.TLS {
-		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
-		if err != nil {
-			return "", err
-		}
-		transport.TLSClientConfig = tlsConfig
-	}
-
-	httpCli := &http.Client{
-		Transport: transport,
-	}
-
-	if !strings.HasPrefix(endpointURL, "http") {
-		endpointURL = fmt.Sprintf("https://%s", endpointURL)
-	}
-
-	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
+	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	reqPayload, err := json.Marshal(
-		struct {
-			StackConfig string
-			Namespace   string
-		}{
-			StackConfig: stackConfig,
-			Namespace:   namespace,
-		})
+	command := path.Join(deployer.binaryPath, "kubectl")
+	if runtime.GOOS == "windows" {
+		command = path.Join(deployer.binaryPath, "kubectl.exe")
+	}
+
+	args := make([]string, 0)
+	args = append(args, "--server", endpoint.URL)
+	args = append(args, "--insecure-skip-tls-verify")
+	args = append(args, "--token", string(token))
+	args = append(args, "--namespace", namespace)
+	args = append(args, "apply", "-f", "-")
+
+	var stderr bytes.Buffer
+	cmd := exec.Command(command, args...)
+	cmd.Stderr = &stderr
+	cmd.Stdin = strings.NewReader(data)
+
+	output, err := cmd.Output()
 	if err != nil {
-		return "", err
+		return nil, errors.New(stderr.String())
 	}
 
-	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
-	if err != nil {
-		return "", err
-	}
-
-	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
-	if err != nil {
-		return "", err
-	}
-
-	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
-	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
-
-	resp, err := httpCli.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var errorResponseData struct {
-			Message string
-			Details string
-		}
-		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
-		if err != nil {
-			output, parseStringErr := ioutil.ReadAll(resp.Body)
-			if parseStringErr != nil {
-				return "", parseStringErr
-			}
-
-			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
-
-		}
-
-		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
-	}
-
-	var responseData struct{ Output string }
-	err = json.NewDecoder(resp.Body).Decode(&responseData)
-	if err != nil {
-		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
-		if parseStringErr != nil {
-			return "", parseStringErr
-		}
-
-		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
-	}
-
-	return responseData.Output, nil
-
+	return output, nil
 }
 
-// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
-func (deployer *KubernetesDeployer) ConvertCompose(data string) ([]byte, error) {
+func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")

api/exec/swarm_stack.go

@@ -10,7 +10,7 @@ import (
 	"path"
 	"runtime"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 )
 
 // SwarmStackManager represents a service for managing stacks.

api/filesystem/filesystem.go

@@ -9,7 +9,7 @@ import (
 	"io/ioutil"
 
 	"github.com/gofrs/uuid"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 
 	"io"
 	"os"
@@ -106,66 +106,6 @@ func (service *Service) GetStackProjectPath(stackIdentifier string) string {
 	return path.Join(service.fileStorePath, ComposeStorePath, stackIdentifier)
 }
 
-// Copy copies the file on fromFilePath to toFilePath
-// if toFilePath exists func will fail unless deleteIfExists is true
-func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error {
-	exists, err := service.FileExists(fromFilePath)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return errors.New("File doesn't exist")
-	}
-
-	finput, err := os.Open(fromFilePath)
-	if err != nil {
-		return err
-	}
-
-	defer finput.Close()
-
-	exists, err = service.FileExists(toFilePath)
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		if !deleteIfExists {
-			return errors.New("Destination file exists")
-		}
-
-		err := os.Remove(toFilePath)
-		if err != nil {
-			return err
-		}
-	}
-
-	foutput, err := os.Create(toFilePath)
-	if err != nil {
-		return err
-	}
-
-	defer foutput.Close()
-
-	buf := make([]byte, 1024)
-	for {
-		n, err := finput.Read(buf)
-		if err != nil && err != io.EOF {
-			return err
-		}
-		if n == 0 {
-			break
-		}
-
-		if _, err := foutput.Write(buf[:n]); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // StoreStackFileFromBytes creates a subfolder in the ComposeStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
@@ -565,8 +505,3 @@ func (service *Service) GetTemporaryPath() (string, error) {
 
 	return path.Join(service.fileStorePath, TempPath, uid.String()), nil
 }
-
-// GetDataStorePath returns path to data folder
-func (service *Service) GetDatastorePath() string {
-	return service.dataStorePath
-}

api/git/azure.go

@@ -1,219 +0,0 @@
-package git
-
-import (
-	"context"
-	"fmt"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-)
-
-const (
-	azureDevOpsHost        = "dev.azure.com"
-	visualStudioHostSuffix = ".visualstudio.com"
-)
-
-func isAzureUrl(s string) bool {
-	return strings.Contains(s, azureDevOpsHost) ||
-		strings.Contains(s, visualStudioHostSuffix)
-}
-
-type azureOptions struct {
-	organisation, project, repository string
-	// a user may pass credentials in a repository URL,
-	// for example https://<username>:<password>@<domain>/<path>
-	username, password string
-}
-
-type azureDownloader struct {
-	client  *http.Client
-	baseUrl string
-}
-
-func NewAzureDownloader(client *http.Client) *azureDownloader {
-	return &azureDownloader{
-		client: client,
-		baseUrl: "https://dev.azure.com",
-	}
-}
-
-func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
-	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
-	if err != nil {
-		return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
-	}
-	defer os.Remove(zipFilepath)
-
-	err = archive.UnzipFile(zipFilepath, destination)
-	if err != nil {
-		return errors.Wrap(err, "failed to unzip file")
-	}
-
-	return nil
-}
-
-func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
-	config, err := parseUrl(options.repositoryUrl)
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to parse url")
-	}
-	downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to build download url")
-	}
-	zipFile, err := ioutil.TempFile("", "azure-git-repo-*.zip")
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to create temp file")
-	}
-	defer zipFile.Close()
-
-	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
-	if options.username != "" || options.password != "" {
-		req.SetBasicAuth(options.username, options.password)
-	} else if config.username != "" || config.password != "" {
-		req.SetBasicAuth(config.username, config.password)
-	}
-
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to create a new HTTP request")
-	}
-
-	res, err := a.client.Do(req)
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to make an HTTP request")
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to download zip with a status \"%v\"", res.Status)
-	}
-
-	_, err = io.Copy(zipFile, res.Body)
-	if err != nil {
-		return "", errors.WithMessage(err, "failed to save HTTP response to a file")
-	}
-	return zipFile.Name(), nil
-}
-
-func parseUrl(rawUrl string) (*azureOptions, error) {
-	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
-		return parseHttpUrl(rawUrl)
-	}
-	if strings.HasPrefix(rawUrl, "git@ssh") {
-		return parseSshUrl(rawUrl)
-	}
-	if strings.HasPrefix(rawUrl, "ssh://") {
-		r := []rune(rawUrl)
-		return parseSshUrl(string(r[6:])) // remove the prefix
-	}
-
-	return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl)
-}
-
-var expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
-
-func parseSshUrl(rawUrl string) (*azureOptions, error) {
-	path := strings.Split(rawUrl, "/")
-
-	unexpectedUrlErr := errors.Errorf("want url %s, got %s", expectedSshUrl, rawUrl)
-	if len(path) != 4 {
-		return nil, unexpectedUrlErr
-	}
-	return &azureOptions{
-		organisation: path[1],
-		project:      path[2],
-		repository:   path[3],
-	}, nil
-}
-
-const expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository"
-const expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository"
-
-func parseHttpUrl(rawUrl string) (*azureOptions, error) {
-	u, err := url.Parse(rawUrl)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse HTTP url")
-	}
-
-	opt := azureOptions{}
-	switch {
-	case u.Host == azureDevOpsHost:
-		path := strings.Split(u.Path, "/")
-		if len(path) != 5 {
-			return nil, errors.Errorf("want url %s, got %s", expectedAzureDevOpsHttpUrl, u)
-		}
-		opt.organisation = path[1]
-		opt.project = path[2]
-		opt.repository = path[4]
-	case strings.HasSuffix(u.Host, visualStudioHostSuffix):
-		path := strings.Split(u.Path, "/")
-		if len(path) != 4 {
-			return nil, errors.Errorf("want url %s, got %s", expectedVisualStudioHttpUrl, u)
-		}
-		opt.organisation = strings.TrimSuffix(u.Host, visualStudioHostSuffix)
-		opt.project = path[1]
-		opt.repository = path[3]
-	default:
-		return nil, errors.Errorf("unknown azure host in url \"%s\"", rawUrl)
-	}
-
-	opt.username = u.User.Username()
-	opt.password, _ = u.User.Password()
-
-	return &opt, nil
-}
-
-func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
-	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
-		a.baseUrl,
-		url.PathEscape(config.organisation),
-		url.PathEscape(config.project),
-		url.PathEscape(config.repository))
-	u, err := url.Parse(rawUrl)
-
-	if err != nil {
-		return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl)
-	}
-	q := u.Query()
-	// scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0
-	q.Set("scopePath", "/")
-	q.Set("download", "true")
-	q.Set("versionDescriptor.versionType", getVersionType(referenceName))
-	q.Set("versionDescriptor.version", formatReferenceName(referenceName))
-	q.Set("$format", "zip")
-	q.Set("recursionLevel", "full")
-	q.Set("api-version", "6.0")
-	u.RawQuery = q.Encode()
-
-	return u.String(), nil
-}
-
-const (
-	branchPrefix = "refs/heads/"
-	tagPrefix    = "refs/tags/"
-)
-
-func formatReferenceName(name string) string {
-	if strings.HasPrefix(name, branchPrefix) {
-		return strings.TrimPrefix(name, branchPrefix)
-	}
-	if strings.HasPrefix(name, tagPrefix) {
-		return strings.TrimPrefix(name, tagPrefix)
-	}
-	return name
-}
-
-func getVersionType(name string) string {
-	if strings.HasPrefix(name, branchPrefix) {
-		return "branch"
-	}
-	if strings.HasPrefix(name, tagPrefix) {
-		return "tag"
-	}
-	return "commit"
-}

api/git/azure_integration_test.go

@@ -1,92 +0,0 @@
-package git
-
-import (
-	"fmt"
-	"github.com/docker/docker/pkg/ioutils"
-	_ "github.com/joho/godotenv/autoload"
-	"github.com/stretchr/testify/assert"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestService_ClonePublicRepository_Azure(t *testing.T) {
-	ensureIntegrationTest(t)
-
-	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
-
-	type args struct {
-		repositoryURLFormat string
-		referenceName       string
-		username            string
-		password            string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{
-			name: "Clone Azure DevOps repo branch",
-			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
-				referenceName:       "refs/heads/main",
-				username:            "",
-				password:            pat,
-			},
-			wantErr: false,
-		},
-		{
-			name: "Clone Azure DevOps repo tag",
-			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
-				referenceName:       "refs/tags/v1.1",
-				username:            "",
-				password:            pat,
-			},
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			dst, err := ioutils.TempDir("", "clone")
-			assert.NoError(t, err)
-			defer os.RemoveAll(dst)
-			repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
-			err = service.ClonePublicRepository(repositoryUrl, tt.args.referenceName, dst)
-			assert.NoError(t, err)
-			assert.FileExists(t, filepath.Join(dst, "README.md"))
-		})
-	}
-}
-
-func TestService_ClonePrivateRepository_Azure(t *testing.T) {
-	ensureIntegrationTest(t)
-
-	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
-
-	dst, err := ioutils.TempDir("", "clone")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dst)
-
-	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
-	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, "", pat)
-	assert.NoError(t, err)
-	assert.FileExists(t, filepath.Join(dst, "README.md"))
-}
-
-func getRequiredValue(t *testing.T, name string) string {
-	value, ok := os.LookupEnv(name)
-	if !ok {
-		t.Fatalf("can't find required env var \"%s\"", name)
-	}
-	return value
-}
-
-func ensureIntegrationTest(t *testing.T) {
-	if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
-		t.Skip("skip an integration test")
-	}
-}

api/git/azure_test.go

@@ -1,250 +0,0 @@
-package git
-
-import (
-	"context"
-	"github.com/stretchr/testify/assert"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"testing"
-)
-
-func Test_buildDownloadUrl(t *testing.T) {
-	a := NewAzureDownloader(nil)
-	u, err := a.buildDownloadUrl(&azureOptions{
-		organisation: "organisation",
-		project:      "project",
-		repository:   "repository",
-	}, "refs/heads/main")
-
-	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0&versionDescriptor.versionType=branch")
-	actualUrl, _ := url.Parse(u)
-	if assert.NoError(t, err) {
-		assert.Equal(t, expectedUrl.Host, actualUrl.Host)
-		assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
-		assert.Equal(t, expectedUrl.Path, actualUrl.Path)
-		assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
-	}
-}
-
-func Test_parseAzureUrl(t *testing.T) {
-	type args struct {
-		url string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *azureOptions
-		wantErr bool
-	}{
-		{
-			name: "Expected SSH URL format starting with ssh://",
-			args: args{
-				url: "ssh://git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
-			},
-			want: &azureOptions{
-				organisation: "Organisation",
-				project:      "Project",
-				repository:   "Repository",
-			},
-			wantErr: false,
-		},
-		{
-			name: "Expected SSH URL format starting with git@ssh",
-			args: args{
-				url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
-			},
-			want: &azureOptions{
-				organisation: "Organisation",
-				project:      "Project",
-				repository:   "Repository",
-			},
-			wantErr: false,
-		},
-		{
-			name: "Unexpected SSH URL format",
-			args: args{
-				url: "git@ssh.dev.azure.com:v3/Organisation/Repository",
-			},
-			wantErr: true,
-		},
-		{
-			name: "Expected HTTPS URL format",
-			args: args{
-				url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
-			},
-			want: &azureOptions{
-				organisation: "Organisation",
-				project:      "Project",
-				repository:   "Repository",
-				username:     "Organisation",
-			},
-			wantErr: false,
-		},
-		{
-			name: "HTTPS URL with credentials",
-			args: args{
-				url: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
-			},
-			want: &azureOptions{
-				organisation: "Organisation",
-				project:      "Project",
-				repository:   "Repository",
-				username:     "username",
-				password:     "password",
-			},
-			wantErr: false,
-		},
-		{
-			name: "HTTPS URL with password",
-			args: args{
-				url: "https://:password@dev.azure.com/Organisation/Project/_git/Repository",
-			},
-			want: &azureOptions{
-				organisation: "Organisation",
-				project:      "Project",
-				repository:   "Repository",
-				password:     "password",
-			},
-			wantErr: false,
-		},
-		{
-			name: "Visual Studio HTTPS URL with credentials",
-			args: args{
-				url: "https://username:password@organisation.visualstudio.com/project/_git/repository",
-			},
-			want: &azureOptions{
-				organisation: "organisation",
-				project:      "project",
-				repository:   "repository",
-				username:     "username",
-				password:     "password",
-			},
-			wantErr: false,
-		},
-		{
-			name: "Unexpected HTTPS URL format",
-			args: args{
-				url: "https://Organisation@dev.azure.com/Project/_git/Repository",
-			},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseUrl(tt.args.url)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseUrl() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
-func Test_isAzureUrl(t *testing.T) {
-	type args struct {
-		s string
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "Is Azure url",
-			args: args{
-				s: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
-			},
-			want: true,
-		},
-		{
-			name: "Is Azure url",
-			args: args{
-				s: "https://portainer.visualstudio.com/project/_git/repository",
-			},
-			want: true,
-		},
-		{
-			name: "Is NOT Azure url",
-			args: args{
-				s: "https://github.com/Organisation/Repository",
-			},
-			want: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, isAzureUrl(tt.args.s))
-		})
-	}
-}
-
-func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
-	type args struct {
-		options cloneOptions
-	}
-	type basicAuth struct {
-		username, password string
-	}
-	tests := []struct {
-		name string
-		args args
-		want *basicAuth
-	}{
-		{
-			name: "username, password embedded",
-			args: args{
-				options: cloneOptions{
-					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
-				},
-			},
-			want: &basicAuth{
-				username: "username",
-				password: "password",
-			},
-		},
-		{
-			name: "username, password embedded, clone options take precedence",
-			args: args{
-				options: cloneOptions{
-					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
-					username:      "u",
-					password:      "p",
-				},
-			},
-			want: &basicAuth{
-				username: "u",
-				password: "p",
-			},
-		},
-		{
-			name: "no credentials",
-			args: args{
-				options: cloneOptions{
-					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var zipRequestAuth *basicAuth
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				if username, password, ok := r.BasicAuth(); ok {
-					zipRequestAuth = &basicAuth{username, password}
-				}
-				w.WriteHeader(http.StatusNotFound) // this makes function under test to return an error
-			}))
-			defer server.Close()
-
-			a := &azureDownloader{
-				client:  server.Client(),
-				baseUrl: server.URL,
-			}
-			_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
-			assert.Error(t, err)
-			assert.Equal(t, tt.want, zipRequestAuth)
-		})
-	}
-}

api/git/git.go

@@ -1,71 +1,21 @@
 package git
 
 import (
-	"context"
 	"crypto/tls"
-	"github.com/pkg/errors"
 	"net/http"
-	"os"
-	"path/filepath"
+	"net/url"
+	"strings"
 	"time"
 
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/transport/client"
-	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 )
 
-type cloneOptions struct {
-	repositoryUrl string
-	username      string
-	password      string
-	referenceName string
-	depth         int
-}
-
-type downloader interface {
-	download(ctx context.Context, dst string, opt cloneOptions) error
-}
-
-type gitClient struct{
-	preserveGitDirectory bool
-}
-
-func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
-	gitOptions := git.CloneOptions{
-		URL:   opt.repositoryUrl,
-		Depth: opt.depth,
-	}
-
-	if opt.password != "" || opt.username != "" {
-		gitOptions.Auth = &githttp.BasicAuth{
-			Username: opt.username,
-			Password: opt.password,
-		}
-	}
-
-	if opt.referenceName != "" {
-		gitOptions.ReferenceName = plumbing.ReferenceName(opt.referenceName)
-	}
-
-	_, err := git.PlainCloneContext(ctx, dst, false, &gitOptions)
-
-	if err != nil {
-		return errors.Wrap(err, "failed to clone git repository")
-	}
-
-	if !c.preserveGitDirectory {
-		os.RemoveAll(filepath.Join(dst, ".git"))
-	}
-
-	return nil
-}
-
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
-	azure    downloader
-	git      downloader
 }
 
 // NewService initializes a new service.
@@ -81,37 +31,32 @@ func NewService() *Service {
 
 	return &Service{
 		httpsCli: httpsCli,
-		azure:    NewAzureDownloader(httpsCli),
-		git:      gitClient{},
 	}
 }
 
 // ClonePublicRepository clones a public git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, referenceName, destination string) error {
-	return service.cloneRepository(destination, cloneOptions{
-		repositoryUrl: repositoryURL,
-		referenceName: referenceName,
-		depth:         1,
-	})
+func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
 // ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
-// destination folder. It will use the specified Username and Password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName, destination, username, password string) error {
-	return service.cloneRepository(destination, cloneOptions{
-		repositoryUrl: repositoryURL,
-		username:      username,
-		password:      password,
-		referenceName: referenceName,
-		depth:         1,
-	})
+// destination folder. It will use the specified username and password for basic HTTP authentication.
+func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
+	credentials := username + ":" + url.PathEscape(password)
+	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
-func (service *Service) cloneRepository(destination string, options cloneOptions) error {
-	if isAzureUrl(options.repositoryUrl) {
-		return service.azure.download(context.TODO(), destination, options)
+func cloneRepository(repositoryURL, referenceName, destination string) error {
+	options := &git.CloneOptions{
+		URL: repositoryURL,
 	}
 
-	return service.git.download(context.TODO(), destination, options)
+	if referenceName != "" {
+		options.ReferenceName = plumbing.ReferenceName(referenceName)
+	}
+
+	_, err := git.PlainClone(destination, false, options)
+	return err
 }

api/git/git_integration_test.go

@@ -1,26 +0,0 @@
-package git
-
-import (
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
-	ensureIntegrationTest(t)
-
-	pat := getRequiredValue(t, "GITHUB_PAT")
-	username := getRequiredValue(t, "GITHUB_USERNAME")
-	service := NewService()
-
-	dst, err := ioutils.TempDir("", "clone")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dst)
-
-	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
-	err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, username, pat)
-	assert.NoError(t, err)
-	assert.FileExists(t, filepath.Join(dst, "README.md"))
-}

api/git/git_test.go

@@ -1,173 +0,0 @@
-package git
-
-import (
-	"context"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/stretchr/testify/assert"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-var bareRepoDir string
-
-func TestMain(m *testing.M) {
-	if err := testMain(m); err != nil {
-		log.Fatal(err)
-	}
-}
-
-// testMain does extra setup/teardown before/after testing.
-// The function is separated from TestMain due to necessity to call os.Exit/log.Fatal in the latter.
-func testMain(m *testing.M) error {
-	dir, err := ioutil.TempDir("", "git-repo-")
-	if err != nil {
-		return errors.Wrap(err, "failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
-
-	bareRepoDir = filepath.Join(dir, "test-clone.git")
-
-	file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0755)
-	if err != nil {
-		return errors.Wrap(err, "failed to open an archive")
-	}
-	err = archive.ExtractTarGz(file, dir)
-	if err != nil {
-		return errors.Wrapf(err, "failed to extract file from the archive to a folder %s\n", dir)
-	}
-
-	m.Run()
-
-	return nil
-}
-
-func Test_ClonePublicRepository_Shallow(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
-	repositoryURL := bareRepoDir
-	referenceName := "refs/heads/main"
-	destination := "shallow"
-
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
-	t.Logf("Cloning into %s", dir)
-	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
-}
-
-func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
-	repositoryURL := bareRepoDir
-	referenceName := "refs/heads/main"
-	destination := "shallow"
-
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
-	t.Logf("Cloning into %s", dir)
-	err = service.ClonePublicRepository(repositoryURL, referenceName, dir)
-	assert.NoError(t, err)
-	assert.NoDirExists(t, filepath.Join(dir, ".git"))
-}
-
-func Test_cloneRepository(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
-
-	repositoryURL := bareRepoDir
-	referenceName := "refs/heads/main"
-	destination := "shallow"
-
-	dir, err := ioutil.TempDir("", destination)
-	if err != nil {
-		t.Fatalf("failed to create a temp dir")
-	}
-	defer os.RemoveAll(dir)
-	t.Logf("Cloning into %s", dir)
-
-	err = service.cloneRepository(dir, cloneOptions{
-		repositoryUrl: repositoryURL,
-		referenceName: referenceName,
-		depth:         10,
-	})
-
-	assert.NoError(t, err)
-	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
-}
-
-func getCommitHistoryLength(t *testing.T, err error, dir string) int {
-	repo, err := git.PlainOpen(dir)
-	if err != nil {
-		t.Fatalf("can't open a git repo at %s with error %v", dir, err)
-	}
-	iter, err := repo.Log(&git.LogOptions{All: true})
-	if err != nil {
-		t.Fatalf("can't get a commit history iterator with error %v", err)
-	}
-	count := 0
-	err = iter.ForEach(func(_ *object.Commit) error {
-		count++
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("can't iterate over the commit history with error %v", err)
-	}
-	return count
-}
-
-type testDownloader struct {
-	called bool
-}
-
-func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
-	t.called = true
-	return nil
-}
-
-func Test_cloneRepository_azure(t *testing.T) {
-	tests := []struct {
-		name   string
-		url    string
-		called bool
-	}{
-		{
-			name:   "Azure HTTP URL",
-			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
-			called: true,
-		},
-		{
-			name:   "Azure SSH URL",
-			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
-			called: true,
-		},
-		{
-			name:   "Something else",
-			url:    "https://example.com",
-			called: false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			azure := &testDownloader{}
-			git := &testDownloader{}
-
-			s := &Service{azure: azure, git: git}
-			s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
-
-			// if azure API is called, git isn't and vice versa
-			assert.Equal(t, tt.called, azure.called)
-			assert.Equal(t, tt.called, !git.called)
-		})
-	}
-}