feat(settings): add admin mapping section EE-971

- fixed merge conflicts

.babelrc

@@ -5,7 +5,8 @@
       "@babel/preset-env",
       {
         "modules": false,
-        "useBuiltIns": "entry"
+        "useBuiltIns": "entry",
+        "corejs": "2"
       }
     ]
   ]

.codeclimate.yml

@@ -1,62 +1,44 @@
 version: "2"
 checks:
   argument-count:
-    enabled: true
-    config:
-      threshold: 4
+    enabled: false
   complex-logic:
-    enabled: true
-    config:
-      threshold: 4
+    enabled: false
   file-lines:
-    enabled: true
-    config:
-      threshold: 300
+    enabled: false
   method-complexity:
     enabled: false
   method-count:
-    enabled: true
-    config:
-      threshold: 20
+    enabled: false
   method-lines:
-    enabled: true
-    config:
-      threshold: 50
+    enabled: false
   nested-control-flow:
-    enabled: true
-    config:
-      threshold: 4
+    enabled: false
   return-statements:
     enabled: false
   similar-code:
-    enabled: true
-    config:
-      threshold: #language-specific defaults. overrides affect all languages.
+    enabled: false
   identical-code:
-    enabled: true
-    config:
-      threshold: #language-specific defaults. overrides affect all languages.
+    enabled: false
 plugins:
   gofmt:
     enabled: true
-  golint:
-    enabled: true
-  govet:
-    enabled: true
-  csslint:
-    enabled: true
-  duplication:
-    enabled: true
-    config:
-      languages:
-        javascript:
-          mass_threshold: 80
   eslint:
     enabled: true
     channel: "eslint-5"
     config:
       config: .eslintrc.yml
-  fixme:
-    enabled: true
 exclude_patterns:
+- assets/
+- build/
+- dist/
+- distribution/
+- node_modules
 - test/
+- webpack/
+- gruntfile.js
+- webpack.config.js
+- api/
+- "!app/kubernetes/**"
+- .github/
+- .tmp/

.eslintrc.yml

@@ -6,7 +6,6 @@ env:
 
 globals:
   angular: true
-  __CONFIG_GA_ID: true
 
 extends:
   - 'eslint:recommended'

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,47 +1,48 @@
----
-name: Bug report
-about: Create a bug report
-
----
-
-<!--
-
-Thanks for reporting a bug for Portainer !
-
-You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
-
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/.
-
-Before opening a new issue, make sure that we do not have any duplicates
-already open. You can ensure this by searching the issue list for this
-repository. If there is a duplicate, please close your issue and add a comment
-to the existing issue instead.
-
-Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
--->
-
-**Bug description**
-A clear and concise description of what the bug is.
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Portainer Logs**
-Provide the logs of your Portainer container or Service.
-You can see how [here](https://portainer.readthedocs.io/en/stable/faq.html#how-do-i-get-the-logs-from-portainer)
-
-**Steps to reproduce the issue:**
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Technical details:**
-* Portainer version:
-* Docker version (managed by Portainer):
-* Platform (windows/linux):
-* Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
-* Browser:
-
-**Additional context**
-Add any other context about the problem here.
+---
+name: Bug report
+about: Create a bug report
+---
+
+<!--
+
+Thanks for reporting a bug for Portainer !
+
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/.
+
+Before opening a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
+-->
+
+**Bug description**
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Portainer Logs**
+Provide the logs of your Portainer container or Service.
+You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#how-do-i-get-the-logs-from-portainer)
+
+**Steps to reproduce the issue:**
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Technical details:**
+
+- Portainer version:
+- Docker version (managed by Portainer):
+- Platform (windows/linux):
+- Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
+- Browser:
+
+**Additional context**
+Add any other context about the problem here.

.github/stale.yml

@@ -12,14 +12,13 @@ issues:
   # Issues with these labels will never be considered stale
   exemptLabels:
     - kind/enhancement
-    - kind/feature
     - kind/question
     - kind/style
     - kind/workaround
     - bug/need-confirmation
     - bug/confirmed
     - status/discuss
-  
+
   # Only issues with all of these labels are checked if stale. Defaults to `[]` (disabled)
   onlyLabels: []
 
@@ -35,9 +34,9 @@ issues:
 
   # Comment to post when marking an issue as stale. Set to `false` to disable
   markComment: >
-    This issue has been marked as stale as it has not had recent activity, 
+    This issue has been marked as stale as it has not had recent activity,
     it will be closed if no further activity occurs in the next 7 days.
-    If you believe that it has been incorrectly labelled as stale, 
+    If you believe that it has been incorrectly labelled as stale,
     leave a comment and the label will be removed.
 
   # Comment to post when removing the stale label.
@@ -48,8 +47,7 @@ issues:
   closeComment: >
     Since no further activity has appeared on this issue it will be closed.
     If you believe that it has been incorrectly closed, leave a comment
-    and mention @itsconquest. One of our staff will then review the issue.
-    
+    mentioning `ametdoohan`, `balasu` or `keverv` and one of our staff will then review the issue.
+
     Note - If it is an old bug report, make sure that it is reproduceable in the
     latest version of Portainer as it may have already been fixed.
-    

.gitignore

@@ -4,5 +4,14 @@ dist
 portainer-checksum.txt
 api/cmd/portainer/portainer*
 .tmp
-.vscode
-.eslintcache
+**/.vscode/settings.json
+**/.vscode/tasks.json
+
+.eslintcache
+.idea
+test/e2e/cypress/screenshots
+*.db
+*.log
+__debug_bin
+api/docs
+.env

.vscode.example/launch.json

@@ -0,0 +1,19 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceRoot}/api/cmd/portainer/main.go",
+      "cwd": "${workspaceRoot}",
+      "env": {},
+      "showLog": true,
+      "args": ["--data", "${env:HOME}/portainer-data", "--assets", "${workspaceRoot}/dist"]
+    }
+  ]
+}

.vscode.example/portainer.code-snippets

@@ -0,0 +1,167 @@
+{
+  // Place your portainer workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
+  // description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
+  // is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
+  // used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
+  // $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
+  // Placeholders with the same ids are connected.
+  // Example:
+  // "Print to console": {
+  // 	"scope": "javascript,typescript",
+  // 	"prefix": "log",
+  // 	"body": [
+  // 		"console.log('$1');",
+  // 		"$2"
+  // 	],
+  // 	"description": "Log output to console"
+  // }
+  "Component": {
+    "scope": "javascript",
+    "prefix": "mycomponent",
+    "description": "Dummy Angularjs Component",
+    "body": [
+      "import angular from 'angular';",
+      "import controller from './${TM_FILENAME_BASE}Controller'",
+      "",
+      "angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').component('$TM_FILENAME_BASE', {",
+      "  templateUrl: './$TM_FILENAME_BASE.html',",
+      "  controller,",
+      "});",
+      ""
+    ]
+  },
+  "Controller": {
+    "scope": "javascript",
+    "prefix": "mycontroller",
+    "body": [
+      "class ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/} {",
+      "\t/* @ngInject */",
+      "\tconstructor($0) {",
+      "\t}",
+      "}",
+      "",
+      "export default ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/};"
+    ],
+    "description": "Dummy ES6+ controller"
+  },
+  "Service": {
+    "scope": "javascript",
+    "prefix": "myservice",
+    "description": "Dummy ES6+ service",
+    "body": [
+      "import angular from 'angular';",
+      "import PortainerError from 'Portainer/error';",
+      "",
+      "class $1 {",
+      "  /* @ngInject */",
+      "  constructor(\\$async, $0) {",
+      "    this.\\$async = \\$async;",
+      "",
+      "    this.getAsync = this.getAsync.bind(this);",
+      "    this.getAllAsync = this.getAllAsync.bind(this);",
+      "    this.createAsync = this.createAsync.bind(this);",
+      "    this.updateAsync = this.updateAsync.bind(this);",
+      "    this.deleteAsync = this.deleteAsync.bind(this);",
+      "  }",
+      "",
+      "  /**",
+      "   * GET",
+      "   */",
+      "  async getAsync() {",
+      "    try {",
+      "",
+      "    } catch (err) {",
+      "      throw new PortainerError('', err);",
+      "    }",
+      "  }",
+      "",
+      "  async getAllAsync() {",
+      "    try {",
+      "",
+      "    } catch (err) {",
+      "      throw new PortainerError('', err);",
+      "    }",
+      "  }",
+      "",
+      "  get() {",
+      "    if () {",
+      "      return this.\\$async(this.getAsync);",
+      "    }",
+      "    return this.\\$async(this.getAllAsync);",
+      "  }",
+      "",
+      "  /**",
+      "   * CREATE",
+      "   */",
+      "  async createAsync() {",
+      "    try {",
+      "",
+      "    } catch (err) {",
+      "      throw new PortainerError('', err);",
+      "    }",
+      "  }",
+      "",
+      "  create() {",
+      "    return this.\\$async(this.createAsync);",
+      "  }",
+      "",
+      "  /**",
+      "   * UPDATE",
+      "   */",
+      "  async updateAsync() {",
+      "    try {",
+      "",
+      "    } catch (err) {",
+      "      throw new PortainerError('', err);",
+      "    }",
+      "  }",
+      "",
+      "  update() {",
+      "    return this.\\$async(this.updateAsync);",
+      "  }",
+      "",
+      "  /**",
+      "   * DELETE",
+      "   */",
+      "  async deleteAsync() {",
+      "    try {",
+      "",
+      "    } catch (err) {",
+      "      throw new PortainerError('', err);",
+      "    }",
+      "  }",
+      "",
+      "  delete() {",
+      "    return this.\\$async(this.deleteAsync);",
+      "  }",
+      "}",
+      "",
+      "export default $1;",
+      "angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').service('$1', $1);"
+    ]
+  },
+  "swagger-api-doc": {
+    "prefix": "swapi",
+    "scope": "go",
+    "description": "Snippet for a api doc",
+    "body": [
+      "// @id ",
+      "// @summary ",
+      "// @description ",
+      "// @description **Access policy**: ",
+      "// @tags ",
+      "// @security jwt",
+      "// @accept json",
+      "// @produce json",
+      "// @param id path int true \"identifier\"",
+      "// @param body body Object true \"details\"",
+      "// @success 200 {object} portainer. \"Success\"",
+      "// @success 204 \"Success\"",
+      "// @failure 400 \"Invalid request\"",
+      "// @failure 403 \"Permission denied\"",
+      "// @failure 404 \" not found\"",
+      "// @failure 500 \"Server error\"",
+      "// @router /{id} [get]"
+    ]
+  }
+}

CONTRIBUTING.md

@@ -6,10 +6,10 @@ Some basic conventions for contributing to this project.
 
 Please make sure that there aren't existing pull requests attempting to address the issue mentioned. Likewise, please check for issues related to update, as someone else may be working on the issue in a branch or fork.
 
-* Please open a discussion in a new issue / existing issue to talk about the changes you'd like to bring
-* Develop in a topic branch, not master/develop
+- Please open a discussion in a new issue / existing issue to talk about the changes you'd like to bring
+- Develop in a topic branch, not master/develop
 
-When creating a new branch, prefix it with the *type* of the change (see section **Commit Message Format** below), the associated opened issue number, a dash and some text describing the issue (using dash as a separator).
+When creating a new branch, prefix it with the _type_ of the change (see section **Commit Message Format** below), the associated opened issue number, a dash and some text describing the issue (using dash as a separator).
 
 For example, if you work on a bugfix for the issue #361, you could name the branch `fix361-template-selection`.
 
@@ -37,14 +37,14 @@ Lines should not exceed 100 characters. This allows the message to be easier to
 
 Must be one of the following:
 
-* **feat**: A new feature
-* **fix**: A bug fix
-* **docs**: Documentation only changes
-* **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing
+- **feat**: A new feature
+- **fix**: A bug fix
+- **docs**: Documentation only changes
+- **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing
   semi-colons, etc)
-* **refactor**: A code change that neither fixes a bug or adds a feature
-* **test**: Adding missing tests
-* **chore**: Changes to the build process or auxiliary tools and libraries such as documentation
+- **refactor**: A code change that neither fixes a bug or adds a feature
+- **test**: Adding missing tests
+- **chore**: Changes to the build process or auxiliary tools and libraries such as documentation
   generation
 
 ### Scope
@@ -57,9 +57,9 @@ You can use the **area** label tag associated on the issue here (for `area/conta
 
 The subject contains succinct description of the change:
 
-* use the imperative, present tense: "change" not "changed" nor "changes"
-* don't capitalize first letter
-* no dot (.) at the end
+- use the imperative, present tense: "change" not "changed" nor "changes"
+- don't capitalize first letter
+- no dot (.) at the end
 
 ## Contribution process
 
@@ -74,3 +74,56 @@ Our contribution process is described below. Some of the steps can be visualized
 The feature request process is similar to the bug report process but has an extra functional validation before the technical validation as well as a documentation validation before the testing phase.
 
 ![portainer_featurerequest_workflow](https://user-images.githubusercontent.com/5485061/45727229-5ad39f00-bbf5-11e8-9550-16ba66c50615.png)
+
+## Build Portainer locally
+
+Ensure you have Docker, Node.js, yarn, and Golang installed in the correct versions.
+
+Install dependencies with yarn:
+
+```sh
+$ yarn
+```
+
+Then build and run the project:
+
+```sh
+$ yarn start
+```
+
+Portainer can now be accessed at <http://localhost:9000>.
+
+Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
+
+## Adding api docs
+
+When adding a new resource (or a route handler), we should add a new tag to api/http/handler/handler.go#L136 like this:
+
+```
+// @tag.name <Name of resource>
+// @tag.description a short description
+```
+
+When adding a new route to an existing handler use the following as a template (you can use `swapi` snippet if you're using vscode):
+
+```
+// @id
+// @summary
+// @description
+// @description **Access policy**:
+// @tags
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "identifier"
+// @param body body Object true "details"
+// @success 200 {object} portainer. "Success"
+// @success 204 "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied"
+// @failure 404 " not found"
+// @failure 500 "Server error"
+// @router /{id} [get]
+```
+
+explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/assets/images/logo_alt.png?raw=true' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/logo_alt.png?raw=true' />
 </p>
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
@@ -10,7 +10,7 @@
 
 **_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
 **_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container, supports other platforms too).
-**_Portainer_** allows you to manage all your Docker resources (containers, images, volumes, networks and more) ! It is compatible with the _standalone Docker_ engine and with _Docker Swarm mode_.
+**_Portainer_** allows you to manage all your Docker resources (containers, images, volumes, networks and more!) It is compatible with the _standalone Docker_ engine and with _Docker Swarm mode_.
 
 ## Demo
 
@@ -24,12 +24,12 @@ Alternatively, you can deploy a copy of the demo stack inside a [play-with-docke
 - Sign in with your [Docker ID](https://docs.docker.com/docker-id)
 - Follow [these](https://github.com/portainer/portainer-demo/blob/master/play-with-docker/docker-stack.yml#L5-L8) steps.
 
-Unlike the public demo, the playground sessions are deleted after 4 hours. Apart from that, all the settings are same, including default credentials.
+Unlike the public demo, the playground sessions are deleted after 4 hours. Apart from that, all the settings are the same, including default credentials.
 
 ## Getting started
 
 - [Deploy Portainer](https://www.portainer.io/installation/)
-- [Documentation](https://www.portainer.io/documentation/)
+- [Documentation](https://documentation.portainer.io)
 
 ## Getting help
 
@@ -38,7 +38,7 @@ For FORMAL Support, please purchase a support subscription from here: https://ww
 For community support: You can find more information about Portainer's community support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
 
 - Issues: https://github.com/portainer/portainer/issues
-- FAQ: https://www.portainer.io/documentation/faqs/
+- FAQ: https://documentation.portainer.io
 - Slack (chat): https://portainer.io/slack/
 
 ## Reporting bugs and contributing
@@ -50,6 +50,12 @@ For community support: You can find more information about Portainer's community
 
 - Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
 
+## Privacy
+
+**To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
+
+When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/documentation/in-app-analytics-and-privacy-policy/). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
+
 ## Limitations
 
 Portainer supports "Current - 2 docker versions only. Prior versions may operate, however these are not supported.

api/access_control.go

@@ -1,154 +0,0 @@
-package portainer
-
-// NewPrivateResourceControl will create a new private resource control associated to the resource specified by the
-// identifier and type parameters. It automatically assigns it to the user specified by the userID parameter.
-func NewPrivateResourceControl(resourceIdentifier string, resourceType ResourceControlType, userID UserID) *ResourceControl {
-	return &ResourceControl{
-		Type:           resourceType,
-		ResourceID:     resourceIdentifier,
-		SubResourceIDs: []string{},
-		UserAccesses: []UserResourceAccess{
-			{
-				UserID:      userID,
-				AccessLevel: ReadWriteAccessLevel,
-			},
-		},
-		TeamAccesses:       []TeamResourceAccess{},
-		AdministratorsOnly: false,
-		Public:             false,
-		System:             false,
-	}
-}
-
-// NewSystemResourceControl will create a new public resource control with the System flag set to true.
-// These kind of resource control are not persisted and are created on the fly by the Portainer API.
-func NewSystemResourceControl(resourceIdentifier string, resourceType ResourceControlType) *ResourceControl {
-	return &ResourceControl{
-		Type:               resourceType,
-		ResourceID:         resourceIdentifier,
-		SubResourceIDs:     []string{},
-		UserAccesses:       []UserResourceAccess{},
-		TeamAccesses:       []TeamResourceAccess{},
-		AdministratorsOnly: false,
-		Public:             true,
-		System:             true,
-	}
-}
-
-// NewPublicResourceControl will create a new public resource control.
-func NewPublicResourceControl(resourceIdentifier string, resourceType ResourceControlType) *ResourceControl {
-	return &ResourceControl{
-		Type:               resourceType,
-		ResourceID:         resourceIdentifier,
-		SubResourceIDs:     []string{},
-		UserAccesses:       []UserResourceAccess{},
-		TeamAccesses:       []TeamResourceAccess{},
-		AdministratorsOnly: false,
-		Public:             true,
-		System:             false,
-	}
-}
-
-// NewRestrictedResourceControl will create a new resource control with user and team accesses restrictions.
-func NewRestrictedResourceControl(resourceIdentifier string, resourceType ResourceControlType, userIDs []UserID, teamIDs []TeamID) *ResourceControl {
-	userAccesses := make([]UserResourceAccess, 0)
-	teamAccesses := make([]TeamResourceAccess, 0)
-
-	for _, id := range userIDs {
-		access := UserResourceAccess{
-			UserID:      id,
-			AccessLevel: ReadWriteAccessLevel,
-		}
-
-		userAccesses = append(userAccesses, access)
-	}
-
-	for _, id := range teamIDs {
-		access := TeamResourceAccess{
-			TeamID:      id,
-			AccessLevel: ReadWriteAccessLevel,
-		}
-
-		teamAccesses = append(teamAccesses, access)
-	}
-
-	return &ResourceControl{
-		Type:               resourceType,
-		ResourceID:         resourceIdentifier,
-		SubResourceIDs:     []string{},
-		UserAccesses:       userAccesses,
-		TeamAccesses:       teamAccesses,
-		AdministratorsOnly: false,
-		Public:             false,
-		System:             false,
-	}
-}
-
-// DecorateStacks will iterate through a list of stacks, check for an associated resource control for each
-// stack and decorate the stack element if a resource control is found.
-func DecorateStacks(stacks []Stack, resourceControls []ResourceControl) []Stack {
-	for idx, stack := range stacks {
-
-		resourceControl := GetResourceControlByResourceIDAndType(stack.Name, StackResourceControl, resourceControls)
-		if resourceControl != nil {
-			stacks[idx].ResourceControl = resourceControl
-		}
-	}
-
-	return stacks
-}
-
-// FilterAuthorizedStacks returns a list of decorated stacks filtered through resource control access checks.
-func FilterAuthorizedStacks(stacks []Stack, user *User, userTeamIDs []TeamID, rbacEnabled bool) []Stack {
-	authorizedStacks := make([]Stack, 0)
-
-	for _, stack := range stacks {
-		_, ok := user.EndpointAuthorizations[stack.EndpointID][EndpointResourcesAccess]
-		if rbacEnabled && ok {
-			authorizedStacks = append(authorizedStacks, stack)
-			continue
-		}
-
-		if stack.ResourceControl != nil && UserCanAccessResource(user.ID, userTeamIDs, stack.ResourceControl) {
-			authorizedStacks = append(authorizedStacks, stack)
-		}
-	}
-
-	return authorizedStacks
-}
-
-// UserCanAccessResource will valide that a user has permissions defined in the specified resource control
-// based on its identifier and the team(s) he is part of.
-func UserCanAccessResource(userID UserID, userTeamIDs []TeamID, resourceControl *ResourceControl) bool {
-	for _, authorizedUserAccess := range resourceControl.UserAccesses {
-		if userID == authorizedUserAccess.UserID {
-			return true
-		}
-	}
-
-	for _, authorizedTeamAccess := range resourceControl.TeamAccesses {
-		for _, userTeamID := range userTeamIDs {
-			if userTeamID == authorizedTeamAccess.TeamID {
-				return true
-			}
-		}
-	}
-
-	return resourceControl.Public
-}
-
-// GetResourceControlByResourceIDAndType retrieves the first matching resource control in a set of resource controls
-// based on the specified id and resource type parameters.
-func GetResourceControlByResourceIDAndType(resourceID string, resourceType ResourceControlType, resourceControls []ResourceControl) *ResourceControl {
-	for _, resourceControl := range resourceControls {
-		if resourceID == resourceControl.ResourceID && resourceType == resourceControl.Type {
-			return &resourceControl
-		}
-		for _, subResourceID := range resourceControl.SubResourceIDs {
-			if resourceID == subResourceID {
-				return &resourceControl
-			}
-		}
-	}
-	return nil
-}

api/adminmonitor/admin_monitor.go

@@ -0,0 +1,69 @@
+package adminmonitor
+
+import (
+	"context"
+	"log"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+var logFatalf = log.Fatalf
+
+type Monitor struct {
+	timeout          time.Duration
+	datastore        portainer.DataStore
+	shutdownCtx      context.Context
+	cancellationFunc context.CancelFunc
+}
+
+// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
+func New(timeout time.Duration, datastore portainer.DataStore, shutdownCtx context.Context) *Monitor {
+	return &Monitor{
+		timeout:     timeout,
+		datastore:   datastore,
+		shutdownCtx: shutdownCtx,
+	}
+}
+
+// Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
+func (m *Monitor) Start() {
+	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
+	m.cancellationFunc = cancellationFunc
+
+	go func() {
+		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
+		select {
+		case <-time.After(m.timeout):
+			initialized, err := m.WasInitialized()
+			if err != nil {
+				logFatalf("failed getting admin user: %s", err)
+			}
+			if !initialized {
+				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
+			}
+		case <-cancellationCtx.Done():
+			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
+		case <-m.shutdownCtx.Done():
+			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
+		}
+	}()
+}
+
+// Stop stops monitor. Safe to call even if monitor wasn't started.
+func (m *Monitor) Stop() {
+	if m.cancellationFunc == nil {
+		return
+	}
+	m.cancellationFunc()
+	m.cancellationFunc = nil
+}
+
+// WasInitialized is a system initialization check
+func (m *Monitor) WasInitialized() (bool, error) {
+	users, err := m.datastore.User().UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		return false, err
+	}
+	return len(users) > 0, nil
+}

api/adminmonitor/admin_monitor_test.go

@@ -0,0 +1,50 @@
+package adminmonitor
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	i "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_stopWithoutStarting(t *testing.T) {
+	monitor := New(1*time.Minute, nil, nil)
+	monitor.Stop()
+}
+
+func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
+	monitor := New(1*time.Minute, nil, nil)
+	monitor.Stop()
+	monitor.Stop()
+}
+
+func Test_canStopStartedMonitor(t *testing.T) {
+	monitor := New(1*time.Minute, nil, context.Background())
+	monitor.Start()
+	assert.NotNil(t, monitor.cancellationFunc, "cancellation function is missing in started monitor")
+
+	monitor.Stop()
+	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
+}
+
+func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
+	timeout := 10 * time.Millisecond
+
+	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
+
+	var fataled bool
+	origLogFatalf := logFatalf
+	logFatalf = func(s string, v ...interface{}) { fataled = true }
+	defer func() {
+		logFatalf = origLogFatalf
+	}()
+
+	monitor := New(timeout, datastore, context.Background())
+	monitor.Start()
+	<-time.After(2 * timeout)
+
+	assert.True(t, fataled, "monitor should been timeout and fatal")
+}

api/api-description.md

@@ -0,0 +1,53 @@
+Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
+Examples are available at https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8
+You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
+
+# Authentication
+
+Most of the API endpoints require to be authenticated as well as some level of authorization to be used.
+Portainer API uses JSON Web Token to manage authentication and thus requires you to provide a token in the **Authorization** header of each request
+with the **Bearer** authentication mechanism.
+
+Example:
+
+```
+Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
+```
+
+# Security
+
+Each API endpoint has an associated access policy, it is documented in the description of each endpoint.
+
+Different access policies are available:
+
+- Public access
+- Authenticated access
+- Restricted access
+- Administrator access
+
+### Public access
+
+No authentication is required to access the endpoints with this access policy.
+
+### Authenticated access
+
+Authentication is required to access the endpoints with this access policy.
+
+### Restricted access
+
+Authentication is required to access the endpoints with this access policy.
+Extra-checks might be added to ensure access to the resource is granted. Returned data might also be filtered.
+
+### Administrator access
+
+Authentication as well as an administrator role are required to access the endpoints with this access policy.
+
+# Execute Docker requests
+
+Portainer **DO NOT** expose specific endpoints to manage your Docker resources (create a container, remove a volume, etc...).
+
+Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you can execute Docker requests **via** the Portainer HTTP API.
+
+To do so, you can use the `/endpoints/{id}/docker` Portainer API endpoint (which is not documented below due to Swagger limitations). This endpoint has a restricted access policy so you still need to be authenticated to be able to query this endpoint. Any query on this endpoint will be proxied to the Docker API of the associated endpoint (requests and responses objects are the same as documented in the Docker API).
+
+**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8).

api/archive/targz.go

@@ -0,0 +1,119 @@
+package archive
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// TarGzDir creates a tar.gz archive and returns it's path.
+// abosolutePath should be an absolute path to a directory.
+// Archive name will be <directoryName>.tar.gz and will be placed next to the directory.
+func TarGzDir(absolutePath string) (string, error) {
+	targzPath := filepath.Join(absolutePath, fmt.Sprintf("%s.tar.gz", filepath.Base(absolutePath)))
+	outFile, err := os.Create(targzPath)
+	if err != nil {
+		return "", err
+	}
+	defer outFile.Close()
+
+	zipWriter := gzip.NewWriter(outFile)
+	defer zipWriter.Close()
+	tarWriter := tar.NewWriter(zipWriter)
+	defer tarWriter.Close()
+
+	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == targzPath {
+			return nil // skip archive file
+		}
+
+		pathInArchive := filepath.Clean(strings.TrimPrefix(path, absolutePath))
+		if pathInArchive == "" {
+			return nil // skip root dir
+		}
+
+		return addToArchive(tarWriter, pathInArchive, path, info)
+	})
+
+	return targzPath, err
+}
+
+func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info os.FileInfo) error {
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return err
+	}
+
+	header.Name = pathInArchive // use relative paths in archive
+
+	err = tarWriter.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	if info.IsDir() {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(tarWriter, file)
+	return err
+}
+
+// ExtractTarGz reads a .tar.gz archive from the reader and extracts it into outputDirPath directory
+func ExtractTarGz(r io.Reader, outputDirPath string) error {
+	zipReader, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer zipReader.Close()
+
+	tarReader := tar.NewReader(zipReader)
+
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			// skip, dir will be created with a file
+		case tar.TypeReg:
+			p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
+			if err := os.MkdirAll(filepath.Dir(p), 0744); err != nil {
+				return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
+			}
+			outFile, err := os.Create(p)
+			if err != nil {
+				return fmt.Errorf("Failed to create file %s", header.Name)
+			}
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				return fmt.Errorf("Failed to extract file %s", header.Name)
+			}
+			outFile.Close()
+		default:
+			return fmt.Errorf("Tar: uknown type: %v in %s",
+				header.Typeflag,
+				header.Name)
+		}
+	}
+
+	return nil
+}

api/archive/targz_test.go

@@ -0,0 +1,98 @@
+package archive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func listFiles(dir string) []string {
+	items := make([]string, 0)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		items = append(items, path)
+		return nil
+	})
+
+	return items
+}
+
+func Test_shouldCreateArhive(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+
+	gzPath, err := TarGzDir(tmpdir)
+	assert.Nil(t, err)
+	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
+
+	extractionDir, _ := ioutil.TempDir("", "extract")
+	defer os.RemoveAll(extractionDir)
+
+	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
+	err = cmd.Run()
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+	extractedFiles := listFiles(extractionDir)
+
+	wasExtracted := func(p string) {
+		fullpath := path.Join(extractionDir, p)
+		assert.Contains(t, extractedFiles, fullpath)
+		copyContent, _ := ioutil.ReadFile(fullpath)
+		assert.Equal(t, content, copyContent)
+	}
+
+	wasExtracted("outer")
+	wasExtracted("dir/inner")
+	wasExtracted("dir/.dotfile")
+}
+
+func Test_shouldCreateArhiveXXXXX(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
+	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
+	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
+
+	gzPath, err := TarGzDir(tmpdir)
+	assert.Nil(t, err)
+	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
+
+	extractionDir, _ := ioutil.TempDir("", "extract")
+	defer os.RemoveAll(extractionDir)
+
+	r, _ := os.Open(gzPath)
+	ExtractTarGz(r, extractionDir)
+	if err != nil {
+		t.Fatal("Failed to extract archive: ", err)
+	}
+	extractedFiles := listFiles(extractionDir)
+
+	wasExtracted := func(p string) {
+		fullpath := path.Join(extractionDir, p)
+		assert.Contains(t, extractedFiles, fullpath)
+		copyContent, _ := ioutil.ReadFile(fullpath)
+		assert.Equal(t, content, copyContent)
+	}
+
+	wasExtracted("outer")
+	wasExtracted("dir/inner")
+	wasExtracted("dir/.dotfile")
+}

api/archive/zip.go

@@ -3,10 +3,13 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
+	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -52,3 +55,60 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 
 	return outFile.Close()
 }
+
+// UnzipFile will decompress a zip archive, moving all files and folders
+// within the zip file (parameter 1) to an output directory (parameter 2).
+func UnzipFile(src string, dest string) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		p := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+		if !strings.HasPrefix(p, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: illegal file path", p)
+		}
+
+		if f.FileInfo().IsDir() {
+			// Make Folder
+			os.MkdirAll(p, os.ModePerm)
+			continue
+		}
+
+		err = unzipFile(f, p)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func unzipFile(f *zip.File, p string) error {
+	// Make File
+	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
+		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
+	}
+	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
+	}
+	defer outFile.Close()
+	rc, err := f.Open()
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
+	}
+	defer rc.Close()
+
+	_, err = io.Copy(outFile, rc)
+
+	if err != nil {
+		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
+	}
+
+	return nil
+}

api/archive/zip_test.go

@@ -0,0 +1,32 @@
+package archive
+
+import (
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestUnzipFile(t *testing.T) {
+	dir, err := ioutil.TempDir("", "unzip-test-")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+	/*
+		Archive structure.
+		├── 0
+		│	├── 1
+		│ 	│   └── 2.txt
+		│   └── 1.txt
+		└── 0.txt
+	*/
+
+	err = UnzipFile("./testdata/sample_archive.zip", dir)
+
+	assert.NoError(t, err)
+	archiveDir := dir + "/sample_archive"
+	assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
+	assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
+
+}

api/authorizations.go

@@ -1,774 +0,0 @@
-package portainer
-
-// AuthorizationService represents a service used to
-// update authorizations associated to a user or team.
-type AuthorizationService struct {
-	dataStore DataStore
-}
-
-// NewAuthorizationService returns a point to a new AuthorizationService instance.
-func NewAuthorizationService(dataStore DataStore) *AuthorizationService {
-	return &AuthorizationService{
-		dataStore: dataStore,
-	}
-}
-
-// DefaultEndpointAuthorizationsForEndpointAdministratorRole returns the default endpoint authorizations
-// associated to the endpoint administrator role.
-func DefaultEndpointAuthorizationsForEndpointAdministratorRole() Authorizations {
-	return map[Authorization]bool{
-		OperationDockerContainerArchiveInfo:         true,
-		OperationDockerContainerList:                true,
-		OperationDockerContainerExport:              true,
-		OperationDockerContainerChanges:             true,
-		OperationDockerContainerInspect:             true,
-		OperationDockerContainerTop:                 true,
-		OperationDockerContainerLogs:                true,
-		OperationDockerContainerStats:               true,
-		OperationDockerContainerAttachWebsocket:     true,
-		OperationDockerContainerArchive:             true,
-		OperationDockerContainerCreate:              true,
-		OperationDockerContainerPrune:               true,
-		OperationDockerContainerKill:                true,
-		OperationDockerContainerPause:               true,
-		OperationDockerContainerUnpause:             true,
-		OperationDockerContainerRestart:             true,
-		OperationDockerContainerStart:               true,
-		OperationDockerContainerStop:                true,
-		OperationDockerContainerWait:                true,
-		OperationDockerContainerResize:              true,
-		OperationDockerContainerAttach:              true,
-		OperationDockerContainerExec:                true,
-		OperationDockerContainerRename:              true,
-		OperationDockerContainerUpdate:              true,
-		OperationDockerContainerPutContainerArchive: true,
-		OperationDockerContainerDelete:              true,
-		OperationDockerImageList:                    true,
-		OperationDockerImageSearch:                  true,
-		OperationDockerImageGetAll:                  true,
-		OperationDockerImageGet:                     true,
-		OperationDockerImageHistory:                 true,
-		OperationDockerImageInspect:                 true,
-		OperationDockerImageLoad:                    true,
-		OperationDockerImageCreate:                  true,
-		OperationDockerImagePrune:                   true,
-		OperationDockerImagePush:                    true,
-		OperationDockerImageTag:                     true,
-		OperationDockerImageDelete:                  true,
-		OperationDockerImageCommit:                  true,
-		OperationDockerImageBuild:                   true,
-		OperationDockerNetworkList:                  true,
-		OperationDockerNetworkInspect:               true,
-		OperationDockerNetworkCreate:                true,
-		OperationDockerNetworkConnect:               true,
-		OperationDockerNetworkDisconnect:            true,
-		OperationDockerNetworkPrune:                 true,
-		OperationDockerNetworkDelete:                true,
-		OperationDockerVolumeList:                   true,
-		OperationDockerVolumeInspect:                true,
-		OperationDockerVolumeCreate:                 true,
-		OperationDockerVolumePrune:                  true,
-		OperationDockerVolumeDelete:                 true,
-		OperationDockerExecInspect:                  true,
-		OperationDockerExecStart:                    true,
-		OperationDockerExecResize:                   true,
-		OperationDockerSwarmInspect:                 true,
-		OperationDockerSwarmUnlockKey:               true,
-		OperationDockerSwarmInit:                    true,
-		OperationDockerSwarmJoin:                    true,
-		OperationDockerSwarmLeave:                   true,
-		OperationDockerSwarmUpdate:                  true,
-		OperationDockerSwarmUnlock:                  true,
-		OperationDockerNodeList:                     true,
-		OperationDockerNodeInspect:                  true,
-		OperationDockerNodeUpdate:                   true,
-		OperationDockerNodeDelete:                   true,
-		OperationDockerServiceList:                  true,
-		OperationDockerServiceInspect:               true,
-		OperationDockerServiceLogs:                  true,
-		OperationDockerServiceCreate:                true,
-		OperationDockerServiceUpdate:                true,
-		OperationDockerServiceDelete:                true,
-		OperationDockerSecretList:                   true,
-		OperationDockerSecretInspect:                true,
-		OperationDockerSecretCreate:                 true,
-		OperationDockerSecretUpdate:                 true,
-		OperationDockerSecretDelete:                 true,
-		OperationDockerConfigList:                   true,
-		OperationDockerConfigInspect:                true,
-		OperationDockerConfigCreate:                 true,
-		OperationDockerConfigUpdate:                 true,
-		OperationDockerConfigDelete:                 true,
-		OperationDockerTaskList:                     true,
-		OperationDockerTaskInspect:                  true,
-		OperationDockerTaskLogs:                     true,
-		OperationDockerPluginList:                   true,
-		OperationDockerPluginPrivileges:             true,
-		OperationDockerPluginInspect:                true,
-		OperationDockerPluginPull:                   true,
-		OperationDockerPluginCreate:                 true,
-		OperationDockerPluginEnable:                 true,
-		OperationDockerPluginDisable:                true,
-		OperationDockerPluginPush:                   true,
-		OperationDockerPluginUpgrade:                true,
-		OperationDockerPluginSet:                    true,
-		OperationDockerPluginDelete:                 true,
-		OperationDockerSessionStart:                 true,
-		OperationDockerDistributionInspect:          true,
-		OperationDockerBuildPrune:                   true,
-		OperationDockerBuildCancel:                  true,
-		OperationDockerPing:                         true,
-		OperationDockerInfo:                         true,
-		OperationDockerVersion:                      true,
-		OperationDockerEvents:                       true,
-		OperationDockerSystem:                       true,
-		OperationDockerUndefined:                    true,
-		OperationDockerAgentPing:                    true,
-		OperationDockerAgentList:                    true,
-		OperationDockerAgentHostInfo:                true,
-		OperationDockerAgentBrowseDelete:            true,
-		OperationDockerAgentBrowseGet:               true,
-		OperationDockerAgentBrowseList:              true,
-		OperationDockerAgentBrowsePut:               true,
-		OperationDockerAgentBrowseRename:            true,
-		OperationDockerAgentUndefined:               true,
-		OperationPortainerResourceControlCreate:     true,
-		OperationPortainerResourceControlUpdate:     true,
-		OperationPortainerStackList:                 true,
-		OperationPortainerStackInspect:              true,
-		OperationPortainerStackFile:                 true,
-		OperationPortainerStackCreate:               true,
-		OperationPortainerStackMigrate:              true,
-		OperationPortainerStackUpdate:               true,
-		OperationPortainerStackDelete:               true,
-		OperationPortainerWebsocketExec:             true,
-		OperationPortainerWebhookList:               true,
-		OperationPortainerWebhookCreate:             true,
-		OperationPortainerWebhookDelete:             true,
-		OperationIntegrationStoridgeAdmin:           true,
-		EndpointResourcesAccess:                     true,
-	}
-}
-
-// DefaultEndpointAuthorizationsForHelpDeskRole returns the default endpoint authorizations
-// associated to the helpdesk role.
-func DefaultEndpointAuthorizationsForHelpDeskRole(volumeBrowsingAuthorizations bool) Authorizations {
-	authorizations := map[Authorization]bool{
-		OperationDockerContainerArchiveInfo: true,
-		OperationDockerContainerList:        true,
-		OperationDockerContainerChanges:     true,
-		OperationDockerContainerInspect:     true,
-		OperationDockerContainerTop:         true,
-		OperationDockerContainerLogs:        true,
-		OperationDockerContainerStats:       true,
-		OperationDockerImageList:            true,
-		OperationDockerImageSearch:          true,
-		OperationDockerImageGetAll:          true,
-		OperationDockerImageGet:             true,
-		OperationDockerImageHistory:         true,
-		OperationDockerImageInspect:         true,
-		OperationDockerNetworkList:          true,
-		OperationDockerNetworkInspect:       true,
-		OperationDockerVolumeList:           true,
-		OperationDockerVolumeInspect:        true,
-		OperationDockerSwarmInspect:         true,
-		OperationDockerNodeList:             true,
-		OperationDockerNodeInspect:          true,
-		OperationDockerServiceList:          true,
-		OperationDockerServiceInspect:       true,
-		OperationDockerServiceLogs:          true,
-		OperationDockerSecretList:           true,
-		OperationDockerSecretInspect:        true,
-		OperationDockerConfigList:           true,
-		OperationDockerConfigInspect:        true,
-		OperationDockerTaskList:             true,
-		OperationDockerTaskInspect:          true,
-		OperationDockerTaskLogs:             true,
-		OperationDockerPluginList:           true,
-		OperationDockerDistributionInspect:  true,
-		OperationDockerPing:                 true,
-		OperationDockerInfo:                 true,
-		OperationDockerVersion:              true,
-		OperationDockerEvents:               true,
-		OperationDockerSystem:               true,
-		OperationDockerAgentPing:            true,
-		OperationDockerAgentList:            true,
-		OperationDockerAgentHostInfo:        true,
-		OperationPortainerStackList:         true,
-		OperationPortainerStackInspect:      true,
-		OperationPortainerStackFile:         true,
-		OperationPortainerWebhookList:       true,
-		EndpointResourcesAccess:             true,
-	}
-
-	if volumeBrowsingAuthorizations {
-		authorizations[OperationDockerAgentBrowseGet] = true
-		authorizations[OperationDockerAgentBrowseList] = true
-	}
-
-	return authorizations
-}
-
-// DefaultEndpointAuthorizationsForStandardUserRole returns the default endpoint authorizations
-// associated to the standard user role.
-func DefaultEndpointAuthorizationsForStandardUserRole(volumeBrowsingAuthorizations bool) Authorizations {
-	authorizations := map[Authorization]bool{
-		OperationDockerContainerArchiveInfo:         true,
-		OperationDockerContainerList:                true,
-		OperationDockerContainerExport:              true,
-		OperationDockerContainerChanges:             true,
-		OperationDockerContainerInspect:             true,
-		OperationDockerContainerTop:                 true,
-		OperationDockerContainerLogs:                true,
-		OperationDockerContainerStats:               true,
-		OperationDockerContainerAttachWebsocket:     true,
-		OperationDockerContainerArchive:             true,
-		OperationDockerContainerCreate:              true,
-		OperationDockerContainerKill:                true,
-		OperationDockerContainerPause:               true,
-		OperationDockerContainerUnpause:             true,
-		OperationDockerContainerRestart:             true,
-		OperationDockerContainerStart:               true,
-		OperationDockerContainerStop:                true,
-		OperationDockerContainerWait:                true,
-		OperationDockerContainerResize:              true,
-		OperationDockerContainerAttach:              true,
-		OperationDockerContainerExec:                true,
-		OperationDockerContainerRename:              true,
-		OperationDockerContainerUpdate:              true,
-		OperationDockerContainerPutContainerArchive: true,
-		OperationDockerContainerDelete:              true,
-		OperationDockerImageList:                    true,
-		OperationDockerImageSearch:                  true,
-		OperationDockerImageGetAll:                  true,
-		OperationDockerImageGet:                     true,
-		OperationDockerImageHistory:                 true,
-		OperationDockerImageInspect:                 true,
-		OperationDockerImageLoad:                    true,
-		OperationDockerImageCreate:                  true,
-		OperationDockerImagePush:                    true,
-		OperationDockerImageTag:                     true,
-		OperationDockerImageDelete:                  true,
-		OperationDockerImageCommit:                  true,
-		OperationDockerImageBuild:                   true,
-		OperationDockerNetworkList:                  true,
-		OperationDockerNetworkInspect:               true,
-		OperationDockerNetworkCreate:                true,
-		OperationDockerNetworkConnect:               true,
-		OperationDockerNetworkDisconnect:            true,
-		OperationDockerNetworkDelete:                true,
-		OperationDockerVolumeList:                   true,
-		OperationDockerVolumeInspect:                true,
-		OperationDockerVolumeCreate:                 true,
-		OperationDockerVolumeDelete:                 true,
-		OperationDockerExecInspect:                  true,
-		OperationDockerExecStart:                    true,
-		OperationDockerExecResize:                   true,
-		OperationDockerSwarmInspect:                 true,
-		OperationDockerSwarmUnlockKey:               true,
-		OperationDockerSwarmInit:                    true,
-		OperationDockerSwarmJoin:                    true,
-		OperationDockerSwarmLeave:                   true,
-		OperationDockerSwarmUpdate:                  true,
-		OperationDockerSwarmUnlock:                  true,
-		OperationDockerNodeList:                     true,
-		OperationDockerNodeInspect:                  true,
-		OperationDockerNodeUpdate:                   true,
-		OperationDockerNodeDelete:                   true,
-		OperationDockerServiceList:                  true,
-		OperationDockerServiceInspect:               true,
-		OperationDockerServiceLogs:                  true,
-		OperationDockerServiceCreate:                true,
-		OperationDockerServiceUpdate:                true,
-		OperationDockerServiceDelete:                true,
-		OperationDockerSecretList:                   true,
-		OperationDockerSecretInspect:                true,
-		OperationDockerSecretCreate:                 true,
-		OperationDockerSecretUpdate:                 true,
-		OperationDockerSecretDelete:                 true,
-		OperationDockerConfigList:                   true,
-		OperationDockerConfigInspect:                true,
-		OperationDockerConfigCreate:                 true,
-		OperationDockerConfigUpdate:                 true,
-		OperationDockerConfigDelete:                 true,
-		OperationDockerTaskList:                     true,
-		OperationDockerTaskInspect:                  true,
-		OperationDockerTaskLogs:                     true,
-		OperationDockerPluginList:                   true,
-		OperationDockerPluginPrivileges:             true,
-		OperationDockerPluginInspect:                true,
-		OperationDockerPluginPull:                   true,
-		OperationDockerPluginCreate:                 true,
-		OperationDockerPluginEnable:                 true,
-		OperationDockerPluginDisable:                true,
-		OperationDockerPluginPush:                   true,
-		OperationDockerPluginUpgrade:                true,
-		OperationDockerPluginSet:                    true,
-		OperationDockerPluginDelete:                 true,
-		OperationDockerSessionStart:                 true,
-		OperationDockerDistributionInspect:          true,
-		OperationDockerBuildPrune:                   true,
-		OperationDockerBuildCancel:                  true,
-		OperationDockerPing:                         true,
-		OperationDockerInfo:                         true,
-		OperationDockerVersion:                      true,
-		OperationDockerEvents:                       true,
-		OperationDockerSystem:                       true,
-		OperationDockerUndefined:                    true,
-		OperationDockerAgentPing:                    true,
-		OperationDockerAgentList:                    true,
-		OperationDockerAgentHostInfo:                true,
-		OperationDockerAgentUndefined:               true,
-		OperationPortainerResourceControlUpdate:     true,
-		OperationPortainerStackList:                 true,
-		OperationPortainerStackInspect:              true,
-		OperationPortainerStackFile:                 true,
-		OperationPortainerStackCreate:               true,
-		OperationPortainerStackMigrate:              true,
-		OperationPortainerStackUpdate:               true,
-		OperationPortainerStackDelete:               true,
-		OperationPortainerWebsocketExec:             true,
-		OperationPortainerWebhookList:               true,
-		OperationPortainerWebhookCreate:             true,
-	}
-
-	if volumeBrowsingAuthorizations {
-		authorizations[OperationDockerAgentBrowseGet] = true
-		authorizations[OperationDockerAgentBrowseList] = true
-		authorizations[OperationDockerAgentBrowseDelete] = true
-		authorizations[OperationDockerAgentBrowsePut] = true
-		authorizations[OperationDockerAgentBrowseRename] = true
-	}
-
-	return authorizations
-}
-
-// DefaultEndpointAuthorizationsForReadOnlyUserRole returns the default endpoint authorizations
-// associated to the readonly user role.
-func DefaultEndpointAuthorizationsForReadOnlyUserRole(volumeBrowsingAuthorizations bool) Authorizations {
-	authorizations := map[Authorization]bool{
-		OperationDockerContainerArchiveInfo: true,
-		OperationDockerContainerList:        true,
-		OperationDockerContainerChanges:     true,
-		OperationDockerContainerInspect:     true,
-		OperationDockerContainerTop:         true,
-		OperationDockerContainerLogs:        true,
-		OperationDockerContainerStats:       true,
-		OperationDockerImageList:            true,
-		OperationDockerImageSearch:          true,
-		OperationDockerImageGetAll:          true,
-		OperationDockerImageGet:             true,
-		OperationDockerImageHistory:         true,
-		OperationDockerImageInspect:         true,
-		OperationDockerNetworkList:          true,
-		OperationDockerNetworkInspect:       true,
-		OperationDockerVolumeList:           true,
-		OperationDockerVolumeInspect:        true,
-		OperationDockerSwarmInspect:         true,
-		OperationDockerNodeList:             true,
-		OperationDockerNodeInspect:          true,
-		OperationDockerServiceList:          true,
-		OperationDockerServiceInspect:       true,
-		OperationDockerServiceLogs:          true,
-		OperationDockerSecretList:           true,
-		OperationDockerSecretInspect:        true,
-		OperationDockerConfigList:           true,
-		OperationDockerConfigInspect:        true,
-		OperationDockerTaskList:             true,
-		OperationDockerTaskInspect:          true,
-		OperationDockerTaskLogs:             true,
-		OperationDockerPluginList:           true,
-		OperationDockerDistributionInspect:  true,
-		OperationDockerPing:                 true,
-		OperationDockerInfo:                 true,
-		OperationDockerVersion:              true,
-		OperationDockerEvents:               true,
-		OperationDockerSystem:               true,
-		OperationDockerAgentPing:            true,
-		OperationDockerAgentList:            true,
-		OperationDockerAgentHostInfo:        true,
-		OperationPortainerStackList:         true,
-		OperationPortainerStackInspect:      true,
-		OperationPortainerStackFile:         true,
-		OperationPortainerWebhookList:       true,
-	}
-
-	if volumeBrowsingAuthorizations {
-		authorizations[OperationDockerAgentBrowseGet] = true
-		authorizations[OperationDockerAgentBrowseList] = true
-	}
-
-	return authorizations
-}
-
-// DefaultPortainerAuthorizations returns the default Portainer authorizations used by non-admin users.
-func DefaultPortainerAuthorizations() Authorizations {
-	return map[Authorization]bool{
-		OperationPortainerDockerHubInspect:        true,
-		OperationPortainerEndpointGroupList:       true,
-		OperationPortainerEndpointList:            true,
-		OperationPortainerEndpointInspect:         true,
-		OperationPortainerEndpointExtensionAdd:    true,
-		OperationPortainerEndpointExtensionRemove: true,
-		OperationPortainerExtensionList:           true,
-		OperationPortainerMOTD:                    true,
-		OperationPortainerRegistryList:            true,
-		OperationPortainerRegistryInspect:         true,
-		OperationPortainerTeamList:                true,
-		OperationPortainerTemplateList:            true,
-		OperationPortainerTemplateInspect:         true,
-		OperationPortainerUserList:                true,
-		OperationPortainerUserInspect:             true,
-		OperationPortainerUserMemberships:         true,
-	}
-}
-
-// UpdateVolumeBrowsingAuthorizations will update all the volume browsing authorizations for each role (except endpoint administrator)
-// based on the specified removeAuthorizations parameter. If removeAuthorizations is set to true, all
-// the authorizations will be dropped for the each role. If removeAuthorizations is set to false, the authorizations
-// will be reset based for each role.
-func (service AuthorizationService) UpdateVolumeBrowsingAuthorizations(remove bool) error {
-	roles, err := service.dataStore.Role().Roles()
-	if err != nil {
-		return err
-	}
-
-	for _, role := range roles {
-		// all roles except endpoint administrator
-		if role.ID != RoleID(1) {
-			updateRoleVolumeBrowsingAuthorizations(&role, remove)
-
-			err := service.dataStore.Role().UpdateRole(role.ID, &role)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func updateRoleVolumeBrowsingAuthorizations(role *Role, removeAuthorizations bool) {
-	if !removeAuthorizations {
-		delete(role.Authorizations, OperationDockerAgentBrowseDelete)
-		delete(role.Authorizations, OperationDockerAgentBrowseGet)
-		delete(role.Authorizations, OperationDockerAgentBrowseList)
-		delete(role.Authorizations, OperationDockerAgentBrowsePut)
-		delete(role.Authorizations, OperationDockerAgentBrowseRename)
-		return
-	}
-
-	role.Authorizations[OperationDockerAgentBrowseGet] = true
-	role.Authorizations[OperationDockerAgentBrowseList] = true
-
-	// Standard-user
-	if role.ID == RoleID(3) {
-		role.Authorizations[OperationDockerAgentBrowseDelete] = true
-		role.Authorizations[OperationDockerAgentBrowsePut] = true
-		role.Authorizations[OperationDockerAgentBrowseRename] = true
-	}
-}
-
-// RemoveTeamAccessPolicies will remove all existing access policies associated to the specified team
-func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) error {
-	endpoints, err := service.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for _, endpoint := range endpoints {
-		for policyTeamID := range endpoint.TeamAccessPolicies {
-			if policyTeamID == teamID {
-				delete(endpoint.TeamAccessPolicies, policyTeamID)
-
-				err := service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
-	if err != nil {
-		return err
-	}
-
-	for _, endpointGroup := range endpointGroups {
-		for policyTeamID := range endpointGroup.TeamAccessPolicies {
-			if policyTeamID == teamID {
-				delete(endpointGroup.TeamAccessPolicies, policyTeamID)
-
-				err := service.dataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	registries, err := service.dataStore.Registry().Registries()
-	if err != nil {
-		return err
-	}
-
-	for _, registry := range registries {
-		for policyTeamID := range registry.TeamAccessPolicies {
-			if policyTeamID == teamID {
-				delete(registry.TeamAccessPolicies, policyTeamID)
-
-				err := service.dataStore.Registry().UpdateRegistry(registry.ID, &registry)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	return service.UpdateUsersAuthorizations()
-}
-
-// RemoveUserAccessPolicies will remove all existing access policies associated to the specified user
-func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) error {
-	endpoints, err := service.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for _, endpoint := range endpoints {
-		for policyUserID := range endpoint.UserAccessPolicies {
-			if policyUserID == userID {
-				delete(endpoint.UserAccessPolicies, policyUserID)
-
-				err := service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
-	if err != nil {
-		return err
-	}
-
-	for _, endpointGroup := range endpointGroups {
-		for policyUserID := range endpointGroup.UserAccessPolicies {
-			if policyUserID == userID {
-				delete(endpointGroup.UserAccessPolicies, policyUserID)
-
-				err := service.dataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	registries, err := service.dataStore.Registry().Registries()
-	if err != nil {
-		return err
-	}
-
-	for _, registry := range registries {
-		for policyUserID := range registry.UserAccessPolicies {
-			if policyUserID == userID {
-				delete(registry.UserAccessPolicies, policyUserID)
-
-				err := service.dataStore.Registry().UpdateRegistry(registry.ID, &registry)
-				if err != nil {
-					return err
-				}
-
-				break
-			}
-		}
-	}
-
-	return nil
-}
-
-// UpdateUsersAuthorizations will trigger an update of the authorizations for all the users.
-func (service *AuthorizationService) UpdateUsersAuthorizations() error {
-	users, err := service.dataStore.User().Users()
-	if err != nil {
-		return err
-	}
-
-	for _, user := range users {
-		err := service.updateUserAuthorizations(user.ID)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (service *AuthorizationService) updateUserAuthorizations(userID UserID) error {
-	user, err := service.dataStore.User().User(userID)
-	if err != nil {
-		return err
-	}
-
-	endpointAuthorizations, err := service.getAuthorizations(user)
-	if err != nil {
-		return err
-	}
-
-	user.EndpointAuthorizations = endpointAuthorizations
-
-	return service.dataStore.User().UpdateUser(userID, user)
-}
-
-func (service *AuthorizationService) getAuthorizations(user *User) (EndpointAuthorizations, error) {
-	endpointAuthorizations := EndpointAuthorizations{}
-	if user.Role == AdministratorRole {
-		return endpointAuthorizations, nil
-	}
-
-	userMemberships, err := service.dataStore.TeamMembership().TeamMembershipsByUserID(user.ID)
-	if err != nil {
-		return endpointAuthorizations, err
-	}
-
-	endpoints, err := service.dataStore.Endpoint().Endpoints()
-	if err != nil {
-		return endpointAuthorizations, err
-	}
-
-	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
-	if err != nil {
-		return endpointAuthorizations, err
-	}
-
-	roles, err := service.dataStore.Role().Roles()
-	if err != nil {
-		return endpointAuthorizations, err
-	}
-
-	endpointAuthorizations = getUserEndpointAuthorizations(user, endpoints, endpointGroups, roles, userMemberships)
-
-	return endpointAuthorizations, nil
-}
-
-func getUserEndpointAuthorizations(user *User, endpoints []Endpoint, endpointGroups []EndpointGroup, roles []Role, userMemberships []TeamMembership) EndpointAuthorizations {
-	endpointAuthorizations := make(EndpointAuthorizations)
-
-	groupUserAccessPolicies := map[EndpointGroupID]UserAccessPolicies{}
-	groupTeamAccessPolicies := map[EndpointGroupID]TeamAccessPolicies{}
-	for _, endpointGroup := range endpointGroups {
-		groupUserAccessPolicies[endpointGroup.ID] = endpointGroup.UserAccessPolicies
-		groupTeamAccessPolicies[endpointGroup.ID] = endpointGroup.TeamAccessPolicies
-	}
-
-	for _, endpoint := range endpoints {
-		authorizations := getAuthorizationsFromUserEndpointPolicy(user, &endpoint, roles)
-		if len(authorizations) > 0 {
-			endpointAuthorizations[endpoint.ID] = authorizations
-			continue
-		}
-
-		authorizations = getAuthorizationsFromUserEndpointGroupPolicy(user, &endpoint, roles, groupUserAccessPolicies)
-		if len(authorizations) > 0 {
-			endpointAuthorizations[endpoint.ID] = authorizations
-			continue
-		}
-
-		authorizations = getAuthorizationsFromTeamEndpointPolicies(userMemberships, &endpoint, roles)
-		if len(authorizations) > 0 {
-			endpointAuthorizations[endpoint.ID] = authorizations
-			continue
-		}
-
-		authorizations = getAuthorizationsFromTeamEndpointGroupPolicies(userMemberships, &endpoint, roles, groupTeamAccessPolicies)
-		if len(authorizations) > 0 {
-			endpointAuthorizations[endpoint.ID] = authorizations
-		}
-	}
-
-	return endpointAuthorizations
-}
-
-func getAuthorizationsFromUserEndpointPolicy(user *User, endpoint *Endpoint, roles []Role) Authorizations {
-	policyRoles := make([]RoleID, 0)
-
-	policy, ok := endpoint.UserAccessPolicies[user.ID]
-	if ok {
-		policyRoles = append(policyRoles, policy.RoleID)
-	}
-
-	return getAuthorizationsFromRoles(policyRoles, roles)
-}
-
-func getAuthorizationsFromUserEndpointGroupPolicy(user *User, endpoint *Endpoint, roles []Role, groupAccessPolicies map[EndpointGroupID]UserAccessPolicies) Authorizations {
-	policyRoles := make([]RoleID, 0)
-
-	policy, ok := groupAccessPolicies[endpoint.GroupID][user.ID]
-	if ok {
-		policyRoles = append(policyRoles, policy.RoleID)
-	}
-
-	return getAuthorizationsFromRoles(policyRoles, roles)
-}
-
-func getAuthorizationsFromTeamEndpointPolicies(memberships []TeamMembership, endpoint *Endpoint, roles []Role) Authorizations {
-	policyRoles := make([]RoleID, 0)
-
-	for _, membership := range memberships {
-		policy, ok := endpoint.TeamAccessPolicies[membership.TeamID]
-		if ok {
-			policyRoles = append(policyRoles, policy.RoleID)
-		}
-	}
-
-	return getAuthorizationsFromRoles(policyRoles, roles)
-}
-
-func getAuthorizationsFromTeamEndpointGroupPolicies(memberships []TeamMembership, endpoint *Endpoint, roles []Role, groupAccessPolicies map[EndpointGroupID]TeamAccessPolicies) Authorizations {
-	policyRoles := make([]RoleID, 0)
-
-	for _, membership := range memberships {
-		policy, ok := groupAccessPolicies[endpoint.GroupID][membership.TeamID]
-		if ok {
-			policyRoles = append(policyRoles, policy.RoleID)
-		}
-	}
-
-	return getAuthorizationsFromRoles(policyRoles, roles)
-}
-
-func getAuthorizationsFromRoles(roleIdentifiers []RoleID, roles []Role) Authorizations {
-	var associatedRoles []Role
-
-	for _, id := range roleIdentifiers {
-		for _, role := range roles {
-			if role.ID == id {
-				associatedRoles = append(associatedRoles, role)
-				break
-			}
-		}
-	}
-
-	var authorizations Authorizations
-	highestPriority := 0
-	for _, role := range associatedRoles {
-		if role.Priority > highestPriority {
-			highestPriority = role.Priority
-			authorizations = role.Authorizations
-		}
-	}
-
-	return authorizations
-}

api/backup/backup.go

@@ -0,0 +1,112 @@
+package backup
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/offlinegate"
+	"github.com/portainer/portainer/api/s3"
+)
+
+const rwxr__r__ os.FileMode = 0744
+
+var filesToBackup = []string{"compose", "config.json", "custom_templates", "edge_jobs", "edge_stacks", "extensions", "portainer.key", "portainer.pub", "tls"}
+
+func BackupToS3(settings portainer.S3BackupSettings, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) error {
+	archivePath, err := CreateBackupArchive(settings.Password, gate, datastore, filestorePath)
+	if err != nil {
+		log.Printf("[ERROR] failed to backup: %s \n", err)
+		return err
+	}
+	archiveReader, err := os.Open(archivePath)
+	if err != nil {
+		log.Println("[ERROR] failed to open backup file")
+		return err
+	}
+	defer os.RemoveAll(filepath.Dir(archivePath))
+
+	archiveName := fmt.Sprintf("portainer-backup_%s", filepath.Base(archivePath))
+
+	s3session, err := s3.NewSession(settings.Region, settings.AccessKeyID, settings.SecretAccessKey)
+	if err != nil {
+		log.Printf("[ERROR] %s \n", err)
+		return err
+	}
+	if err := s3.Upload(s3session, archiveReader, settings.BucketName, archiveName); err != nil {
+		log.Printf("[ERROR] failed to upload backup to S3: %s \n", err)
+		return err
+	}
+	return nil
+}
+
+// Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
+func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
+	unlock := gate.Lock()
+	defer unlock()
+
+	backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
+	if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
+		return "", errors.Wrap(err, "Failed to create backup dir")
+	}
+
+	if err := backupDb(backupDirPath, datastore); err != nil {
+		return "", errors.Wrap(err, "Failed to backup database")
+	}
+
+	for _, filename := range filesToBackup {
+		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
+		if err != nil {
+			return "", errors.Wrap(err, "Failed to create backup file")
+		}
+	}
+
+	archivePath, err := archive.TarGzDir(backupDirPath)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to make an archive")
+	}
+
+	if password != "" {
+		archivePath, err = encrypt(archivePath, password)
+		if err != nil {
+			return "", errors.Wrap(err, "Failed to encrypt backup with the password")
+		}
+	}
+
+	return archivePath, nil
+}
+
+func backupDb(backupDirPath string, datastore portainer.DataStore) error {
+	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
+	if err != nil {
+		return err
+	}
+	if err = datastore.BackupTo(backupWriter); err != nil {
+		return err
+	}
+	return backupWriter.Close()
+}
+
+func encrypt(path string, passphrase string) (string, error) {
+	in, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer in.Close()
+
+	outFileName := fmt.Sprintf("%s.encrypted", path)
+	out, err := os.Create(outFileName)
+	if err != nil {
+		return "", err
+	}
+
+	err = crypto.AesEncrypt(in, out, []byte(passphrase))
+
+	return outFileName, err
+}

api/backup/backup_scheduler.go

@@ -0,0 +1,118 @@
+package backup
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/offlinegate"
+	"github.com/robfig/cron/v3"
+)
+
+// BackupScheduler orchestrates S3 settings and active backup cron jobs
+type BackupScheduler struct {
+	cronmanager     *cron.Cron
+	s3backupService portainer.S3BackupService
+	gate            *offlinegate.OfflineGate
+	datastore       portainer.DataStore
+	filestorePath   string
+}
+
+func NewBackupScheduler(offlineGate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) *BackupScheduler {
+	crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger)))
+	s3backupService := datastore.S3Backup()
+
+	return &BackupScheduler{
+		cronmanager:     crontab,
+		s3backupService: s3backupService,
+		gate:            offlineGate,
+		datastore:       datastore,
+		filestorePath:   filestorePath,
+	}
+}
+
+// Start fetches latest backup settings and starts cron job if configured
+func (s *BackupScheduler) Start() error {
+	s.cronmanager.Start()
+
+	settings, err := s.s3backupService.GetSettings()
+	if err != nil {
+		return errors.Wrap(err, "failed to fetch settings")
+	}
+
+	if canBeScheduled(settings) {
+		return s.startJob(settings)
+	}
+
+	return nil
+}
+
+// Stop stops the scheduler if it is running; otherwise it does nothing.
+// A context is returned so the caller can wait for running jobs to complete.
+func (s *BackupScheduler) Stop() context.Context {
+	if s.cronmanager != nil {
+		log.Println("[DEBUG] Stopping backup scheduler")
+		return s.cronmanager.Stop()
+	}
+
+	return nil
+}
+
+// Update updates stored S3 backup settings and orchestrates cron jobs.
+// When scheduler has an active cron job, then it shuts it down.
+// When a provided settings has a cron, then starts a new cron job.
+// When ever current cron is being shut down, last cron error going to be dropped.
+func (s *BackupScheduler) Update(settings portainer.S3BackupSettings) error {
+
+	if err := s.s3backupService.UpdateSettings(settings); err != nil {
+		return errors.Wrap(err, "failed to update settings")
+	}
+
+	if err := s.stopJobs(); err != nil {
+		return errors.Wrap(err, "failed to stop current cronjob")
+	}
+
+	if canBeScheduled(settings) {
+		return s.startJob(settings)
+	}
+
+	return nil
+}
+
+// stops current backup cron job and drops last cron error if any
+func (s *BackupScheduler) stopJobs() error {
+	// stopping all cron jobs as there should be only one (c)
+	for _, job := range s.cronmanager.Entries() {
+		s.cronmanager.Remove(job.ID)
+	}
+
+	return s.s3backupService.DropStatus()
+}
+
+func (s *BackupScheduler) startJob(settings portainer.S3BackupSettings) error {
+	_, err := s.cronmanager.AddFunc(settings.CronRule, s.backup(settings))
+	if err != nil {
+		return errors.Wrap(err, "failed to start a new backup cron job")
+	}
+
+	return nil
+}
+
+func canBeScheduled(s portainer.S3BackupSettings) bool {
+	return s.AccessKeyID != "" && s.SecretAccessKey != "" && s.Region != "" && s.BucketName != "" && s.CronRule != ""
+}
+
+func (s *BackupScheduler) backup(settings portainer.S3BackupSettings) func() {
+	return func() {
+		err := BackupToS3(settings, s.gate, s.datastore, s.filestorePath)
+		status := portainer.S3BackupStatus{
+			Failed:    err != nil,
+			Timestamp: time.Now(),
+		}
+		if err = s.s3backupService.UpdateStatus(status); err != nil {
+			log.Printf("[ERROR] failed to update status of last scheduled backup. Status: %+v . Err: %s \n", status, err)
+		}
+	}
+}

api/backup/backup_scheduler_test.go

@@ -0,0 +1,112 @@
+package backup
+
+import (
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	i "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
+)
+
+func newScheduler(status *portainer.S3BackupStatus, settings *portainer.S3BackupSettings) *BackupScheduler {
+	scheduler := NewBackupScheduler(nil, i.NewDatastore(i.WithS3BackupService(status, settings)), "")
+	scheduler.Start()
+
+	return scheduler
+}
+
+func settings(cronRule string,
+	accessKeyID string,
+	secretAccessKey string,
+	region string,
+	bucketName string) *portainer.S3BackupSettings {
+	return &portainer.S3BackupSettings{
+		CronRule:        cronRule,
+		AccessKeyID:     accessKeyID,
+		SecretAccessKey: secretAccessKey,
+		Region:          region,
+		BucketName:      bucketName,
+	}
+}
+
+func Test_startWithoutCron_shouldNotStartAJob(t *testing.T) {
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
+	defer scheduler.Stop()
+
+	jobs := scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 0, "should have empty job list")
+}
+
+func Test_startWitACron_shouldAlsoStartAJob(t *testing.T) {
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, settings("*/10 * * * *", "id", "key", "region", "bucket"))
+	defer scheduler.Stop()
+
+	jobs := scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 1, "should have 1 active job")
+}
+
+func Test_update_shouldDropStatus(t *testing.T) {
+	storedStatus := &portainer.S3BackupStatus{Failed: true, Timestamp: time.Now().Add(-time.Hour)}
+	scheduler := newScheduler(storedStatus, &portainer.S3BackupSettings{})
+	defer scheduler.Stop()
+
+	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
+	assert.Equal(t, portainer.S3BackupStatus{}, *storedStatus, "stasus should be dropped")
+}
+
+func Test_update_shouldUpdateSettings(t *testing.T) {
+	storedSettings := &portainer.S3BackupSettings{}
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, storedSettings)
+	defer scheduler.Stop()
+
+	newSettings := settings("", "id2", "key2", "region2", "bucket2")
+	scheduler.Update(*newSettings)
+
+	assert.EqualValues(t, *storedSettings, *newSettings, "updated settings should match stored settings")
+}
+
+func Test_updateWithCron_shouldStartAJob(t *testing.T) {
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
+	defer scheduler.Stop()
+
+	jobs := scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 0, "should have empty job list upon startup")
+
+	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
+
+	jobs = scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 1, "should have 1 active job")
+}
+
+func Test_updateWithoutCron_shouldStopActiveJob(t *testing.T) {
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
+	defer scheduler.Stop()
+
+	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
+
+	jobs := scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 1, "should have 1 active job")
+
+	scheduler.Update(*settings("", "id2", "key2", "region2", "bucket2"))
+
+	jobs = scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 0, "should have no active jobs")
+}
+
+func Test_updateWithACron_shouldStopActiveJob_andStartNewJob(t *testing.T) {
+	scheduler := newScheduler(&portainer.S3BackupStatus{}, &portainer.S3BackupSettings{})
+	defer scheduler.Stop()
+
+	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
+
+	jobs := scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 1, "should have 1 active job")
+	initJobId := jobs[0].ID
+
+	scheduler.Update(*settings("*/10 * * * *", "id", "key", "region", "bucket"))
+
+	jobs = scheduler.cronmanager.Entries()
+	assert.Len(t, jobs, 1, "should have 1 active job")
+	assert.NotEqual(t, initJobId, jobs[0].ID, "new job should have a diffent id")
+}

api/backup/copy.go

@@ -0,0 +1,68 @@
+package backup
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func copyPath(path string, toDir string) error {
+	info, err := os.Stat(path)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		// skip copy if file does not exist
+		return nil
+	}
+
+	if !info.IsDir() {
+		destination := filepath.Join(toDir, info.Name())
+		return copyFile(path, destination)
+	}
+
+	return copyDir(path, toDir)
+}
+
+func copyDir(fromDir, toDir string) error {
+	cleanedSourcePath := filepath.Clean(fromDir)
+	parentDirectory := filepath.Dir(cleanedSourcePath)
+	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		if info.IsDir() {
+			return nil // skip directory creations
+		}
+
+		if info.Mode()&os.ModeSymlink != 0 { // entry is a symlink
+			return nil // don't copy symlinks
+		}
+
+		return copyFile(path, destination)
+	})
+
+	return err
+}
+
+// copies regular a file from src to dst
+func copyFile(src, dst string) error {
+	from, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer from.Close()
+
+	// has to include 'execute' bit, otherwise fails. MkdirAll follows `mkdir -m` restrictions
+	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
+		return err
+	}
+	to, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer to.Close()
+
+	_, err = io.Copy(to, from)
+	return err
+}

api/backup/copy_test.go

@@ -0,0 +1,104 @@
+package backup
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func listFiles(dir string) []string {
+	items := make([]string, 0)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		items = append(items, path)
+		return nil
+	})
+
+	return items
+}
+
+func contains(t *testing.T, list []string, path string) {
+	assert.Contains(t, list, path)
+	copyContent, _ := ioutil.ReadFile(path)
+	assert.Equal(t, "content\n", string(copyContent))
+}
+
+func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyFile("does-not-exist", tmpdir)
+	assert.NotNil(t, err)
+}
+
+func Test_copyFile_shouldMakeAbackup(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+
+	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
+	assert.Nil(t, err)
+
+	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := copyDir("./test_assets/copy_test", destination)
+	assert.Nil(t, err)
+
+	createdFiles := listFiles(destination)
+
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
+}
+
+func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyPath("does-not-exists", tmpdir)
+	assert.Nil(t, err)
+
+	assert.Empty(t, listFiles(tmpdir))
+}
+
+func Test_backupPath_shouldCopyFile(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+
+	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
+	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
+	assert.Nil(t, err)
+
+	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	assert.Nil(t, err)
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_backupPath_shouldCopyDir(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := copyPath("./test_assets/copy_test", destination)
+	assert.Nil(t, err)
+
+	createdFiles := listFiles(destination)
+
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
+}

api/backup/restore.go

@@ -0,0 +1,68 @@
+package backup
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/offlinegate"
+)
+
+var filesToRestore = append(filesToBackup, "portainer.db")
+
+// Restores system state from backup archive, will trigger system shutdown, when finished.
+func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, shutdownTrigger context.CancelFunc) error {
+	var err error
+	if password != "" {
+		archive, err = decrypt(archive, password)
+		if err != nil {
+			return errors.Wrap(err, "failed to decrypt the archive")
+		}
+	}
+
+	restorePath := filepath.Join(filestorePath, "restore", time.Now().Format("20060102150405"))
+	defer os.RemoveAll(filepath.Dir(restorePath))
+
+	err = extractArchive(archive, restorePath)
+	if err != nil {
+		return errors.Wrap(err, "cannot extract files from the archive. Please ensure the password is correct and try again")
+	}
+
+	unlock := gate.Lock()
+	defer unlock()
+
+	if err = datastore.Close(); err != nil {
+		return errors.Wrap(err, "Failed to stop db")
+	}
+
+	if err = restoreFiles(restorePath, filestorePath); err != nil {
+		return errors.Wrap(err, "failed to restore the system state")
+	}
+
+	shutdownTrigger()
+	return nil
+}
+
+func decrypt(r io.Reader, password string) (io.Reader, error) {
+	return crypto.AesDecrypt(r, []byte(password))
+}
+
+func extractArchive(r io.Reader, destinationDirPath string) error {
+	return archive.ExtractTarGz(r, destinationDirPath)
+}
+
+func restoreFiles(srcDir string, destinationDir string) error {
+	for _, filename := range filesToRestore {
+		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/backup/test_assets/copy_test/dir/.dotfile

@@ -0,0 +1 @@
+content

api/backup/test_assets/copy_test/dir/inner

@@ -0,0 +1 @@
+content

api/backup/test_assets/copy_test/outer

@@ -0,0 +1 @@
+content

api/bolt/backup.go

@@ -0,0 +1,181 @@
+package bolt
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	plog "github.com/portainer/portainer/api/bolt/log"
+)
+
+var backupDefaults = struct {
+	backupDir        string
+	editions         []string
+	databaseFileName string
+}{
+	"backups",
+	[]string{"CE", "BE", "EE"},
+	databaseFileName,
+}
+
+var backupLog = plog.NewScopedLog("bolt, backup")
+
+//
+// Backup Helpers
+//
+
+// createBackupFolders create initial folders for backups
+func (store *Store) createBackupFolders() {
+	for _, e := range backupDefaults.editions {
+
+		p := path.Join(store.path, backupDefaults.backupDir, e)
+
+		if exists, _ := store.fileService.FileExists(p); !exists {
+			err := os.MkdirAll(p, 0700)
+			if err != nil {
+				backupLog.Error("Error while creating backup folders", err)
+			}
+		}
+	}
+}
+
+func (store *Store) databasePath() string {
+	return path.Join(store.path, databaseFileName)
+}
+
+func (store *Store) editionBackupDir(edition portainer.SoftwareEdition) string {
+	return path.Join(store.path, backupDefaults.backupDir, edition.GetEditionLabel())
+}
+
+func (store *Store) copyDBFile(from string, to string) error {
+	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
+	err := store.fileService.Copy(from, to, true)
+	if err != nil {
+		backupLog.Error("Failed", err)
+	}
+	return err
+}
+
+// BackupOptions provide a helper to inject backup options
+type BackupOptions struct {
+	Edition        portainer.SoftwareEdition
+	Version        int
+	BackupDir      string
+	BackupFileName string
+	BackupPath     string
+}
+
+func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
+	if options == nil {
+		options = &BackupOptions{}
+	}
+	if options.Edition == 0 {
+		options.Edition = store.edition()
+	}
+	if options.Version == 0 {
+		options.Version, _ = store.version()
+	}
+	if options.BackupDir == "" {
+		options.BackupDir = store.editionBackupDir(options.Edition)
+	}
+	if options.BackupFileName == "" {
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+	}
+	if options.BackupPath == "" {
+		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
+	}
+	return options
+}
+
+func (store *Store) listEditionBackups(edition portainer.SoftwareEdition) ([]string, error) {
+	var fileNames = []string{}
+
+	files, err := ioutil.ReadDir(store.editionBackupDir(edition))
+
+	if err != nil {
+		backupLog.Error("Error while retrieving backup files", err)
+		return fileNames, err
+	}
+
+	for _, f := range files {
+		fileNames = append(fileNames, f.Name())
+	}
+
+	return fileNames, nil
+}
+
+func (store *Store) lastestEditionBackup() (string, error) {
+	edition := store.edition()
+
+	files, err := store.listEditionBackups(edition)
+	if err != nil {
+		backupLog.Error("Error while retrieving backup files", err)
+		return "", err
+	}
+
+	if len(files) == 0 {
+		return "", nil
+	}
+
+	return files[len(files)-1], nil
+}
+
+// BackupWithOptions backup current database with options
+func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
+	backupLog.Info("creating db backup")
+	store.createBackupFolders()
+
+	options = store.setupOptions(options)
+
+	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
+}
+
+// Backup current database with default options
+func (store *Store) Backup() (string, error) {
+	return store.BackupWithOptions(nil)
+}
+
+// RestoreWithOptions previously saved backup for the current Edition  with options
+// Restore strategies:
+// - default: restore latest from current edition
+// - restore a specific
+func (store *Store) RestoreWithOptions(options *BackupOptions) error {
+
+	// Check if backup file exist before restoring
+
+	options = store.setupOptions(options)
+
+	_, err := os.Stat(options.BackupPath)
+
+	if os.IsNotExist(err) {
+		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
+		return err
+	}
+
+	err = store.Close()
+	if err != nil {
+		backupLog.Error("Error while closing store before restore", err)
+		return err
+	}
+	backupLog.Info("Restoring db backup")
+	err = store.copyDBFile(options.BackupPath, store.databasePath())
+	if err != nil {
+		return err
+	}
+	return store.Open()
+}
+
+// Restore previously saved backup for the current Edition  with default options
+func (store *Store) Restore() error {
+	var options = &BackupOptions{}
+	var err error
+	options.BackupFileName, err = store.lastestEditionBackup()
+	if err != nil {
+		return err
+	}
+	return store.RestoreWithOptions(options)
+
+}

api/bolt/backup_test.go

@@ -0,0 +1,118 @@
+package bolt
+
+import (
+	"fmt"
+	"log"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+func TestCreateBackupFolders(t *testing.T) {
+	store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false)
+	if exists, _ := store.fileService.FileExists("tmp/backups"); exists {
+		t.Error("Expect backups folder to not exist")
+	}
+	store.createBackupFolders()
+	if exists, _ := store.fileService.FileExists("tmp/backups"); !exists {
+		t.Error("Expect backups folder to exist")
+	}
+	store.createBackupFolders()
+	store.Close()
+	teardown()
+}
+
+func TestStoreCreation(t *testing.T) {
+	store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false)
+	if store == nil {
+		t.Error("Expect to create a store")
+	}
+
+	if store.edition() != portainer.PortainerEE {
+		t.Error("Expect to get EE Edition")
+	}
+
+	version, err := store.version()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if version != portainer.DBVersionEE {
+		t.Error("Expect to get EE DBVersion")
+	}
+
+	store.Close()
+	teardown()
+}
+
+func TestBackup(t *testing.T) {
+
+	tests := []struct {
+		edition portainer.SoftwareEdition
+		version int
+	}{
+		{edition: portainer.PortainerCE, version: portainer.DBVersion},
+		{edition: portainer.PortainerEE, version: portainer.DBVersionEE},
+	}
+
+	for _, tc := range tests {
+		backupFileName := fmt.Sprintf("tmp/backups/%s/portainer.db.%03d.*", tc.edition.GetEditionLabel(), tc.version)
+		t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) {
+			store := NewTestStore(tc.edition, tc.version, false)
+			store.Backup()
+
+			if !isFileExist(backupFileName) {
+				t.Errorf("Expect backup file to be created %s", backupFileName)
+			}
+			store.Close()
+		})
+	}
+	t.Run("BackupWithOption should create a name specific backup", func(t *testing.T) {
+		edition := portainer.PortainerCE
+		version := portainer.DBVersion
+		store := NewTestStore(edition, version, false)
+		store.BackupWithOptions(&BackupOptions{
+			BackupFileName: beforePortainerUpgradeToEEBackup,
+			Edition:        portainer.PortainerCE,
+		})
+		backupFileName := fmt.Sprintf("tmp/backups/%s/%s", edition.GetEditionLabel(), beforePortainerUpgradeToEEBackup)
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+		store.Close()
+	})
+
+	teardown()
+}
+
+// TODO restore / backup failed test cases
+func TestRestore(t *testing.T) {
+
+	editions := []portainer.SoftwareEdition{portainer.PortainerCE, portainer.PortainerEE}
+	var currentVersion = 0
+
+	for i, e := range editions {
+		editionLabel := e.GetEditionLabel()
+		currentVersion = 10 ^ i + 1
+		store := NewTestStore(e, currentVersion, false)
+		t.Run(fmt.Sprintf("Basic Restore for %s", editionLabel), func(t *testing.T) {
+			store.Backup()
+			updateVersion(store, currentVersion+1)
+			testVersion(store, currentVersion+1, t)
+			store.Restore()
+			testVersion(store, currentVersion, t)
+		})
+		t.Run(fmt.Sprintf("Basic Restore After Multiple Backup for %s", editionLabel), func(t *testing.T) {
+			currentVersion = currentVersion + 5
+			updateVersion(store, currentVersion)
+			store.Backup()
+			updateVersion(store, currentVersion+2)
+			testVersion(store, currentVersion+2, t)
+			store.Restore()
+			testVersion(store, currentVersion, t)
+		})
+		store.Close()
+	}
+
+	teardown()
+}

api/bolt/bolttest/datastore.go

@@ -0,0 +1,73 @@
+package bolttest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+var errTempDir = errors.New("can't create a temp dir")
+
+func MustNewTestStore(init bool) (*bolt.Store, func()) {
+	store, teardown, err := NewTestStore(init)
+	if err != nil {
+		if !errors.Is(err, errTempDir) {
+			teardown()
+		}
+		log.Fatal(err)
+	}
+
+	return store, teardown
+}
+
+func NewTestStore(init bool) (*bolt.Store, func(), error) {
+	// Creates unique temp directory in a concurrency friendly manner.
+	dataStorePath, err := ioutil.TempDir("", "boltdb")
+	if err != nil {
+		return nil, nil, errors.Wrap(errTempDir, err.Error())
+	}
+
+	fileService, err := filesystem.NewService(dataStorePath, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	store, err := bolt.NewStore(dataStorePath, fileService)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = store.Open()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if init {
+		err = store.Init()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	teardown := func() {
+		teardown(store, dataStorePath)
+	}
+
+	return store, teardown, nil
+}
+
+func teardown(store *bolt.Store, dataStorePath string) {
+	err := store.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	err = os.RemoveAll(dataStorePath)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

api/bolt/customtemplate/customtemplate.go

@@ -0,0 +1,96 @@
+package customtemplate
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "customtemplates"
+)
+
+// Service represents a service for managing custom template data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// CustomTemplates return an array containing all the custom templates.
+func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
+	var customTemplates = make([]portainer.CustomTemplate, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var customTemplate portainer.CustomTemplate
+			err := internal.UnmarshalObjectWithJsoniter(v, &customTemplate)
+			if err != nil {
+				return err
+			}
+			customTemplates = append(customTemplates, customTemplate)
+		}
+
+		return nil
+	})
+
+	return customTemplates, err
+}
+
+// CustomTemplate returns an custom template by ID.
+func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
+	var customTemplate portainer.CustomTemplate
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &customTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	return &customTemplate, nil
+}
+
+// UpdateCustomTemplate updates an custom template.
+func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, customTemplate)
+}
+
+// DeleteCustomTemplate deletes an custom template.
+func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// CreateCustomTemplate assign an ID to a new custom template and saves it.
+func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTemplate) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(customTemplate)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(customTemplate.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/datastore.go

@@ -1,20 +1,26 @@
 package bolt
 
 import (
-	"log"
+	"io"
 	"path"
 	"time"
 
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/license"
+	"github.com/portainer/portainer/api/bolt/s3backup"
+
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/customtemplate"
 	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
 	"github.com/portainer/portainer/api/bolt/edgestack"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
-	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -30,7 +36,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/webhook"
 )
 
-const (
+var (
 	databaseFileName = "portainer.db"
 )
 
@@ -38,19 +44,23 @@ const (
 // BoltDB as the storage system.
 type Store struct {
 	path                    string
-	db                      *bolt.DB
+	connection              *internal.DbConnection
 	isNew                   bool
 	fileService             portainer.FileService
+	CustomTemplateService   *customtemplate.Service
 	DockerHubService        *dockerhub.Service
 	EdgeGroupService        *edgegroup.Service
+	EdgeJobService          *edgejob.Service
 	EdgeStackService        *edgestack.Service
 	EndpointGroupService    *endpointgroup.Service
 	EndpointService         *endpoint.Service
 	EndpointRelationService *endpointrelation.Service
 	ExtensionService        *extension.Service
+	LicenseService          *license.Service
 	RegistryService         *registry.Service
 	ResourceControlService  *resourcecontrol.Service
 	RoleService             *role.Service
+	S3BackupService         *s3backup.Service
 	ScheduleService         *schedule.Service
 	SettingsService         *settings.Service
 	StackService            *stack.Service
@@ -63,12 +73,29 @@ type Store struct {
 	WebhookService          *webhook.Service
 }
 
+func (store *Store) version() (int, error) {
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	}
+	return version, err
+}
+
+func (store *Store) edition() portainer.SoftwareEdition {
+	edition, err := store.VersionService.Edition()
+	if err == errors.ErrObjectNotFound {
+		edition = portainer.PortainerCE
+	}
+	return edition
+}
+
 // NewStore initializes a new Store and the associated services
 func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
 	store := &Store{
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
+		connection:  &internal.DbConnection{},
 	}
 
 	databasePath := path.Join(storePath, databaseFileName)
@@ -91,15 +118,15 @@ func (store *Store) Open() error {
 	if err != nil {
 		return err
 	}
-	store.db = db
+	store.connection.DB = db
 
 	return store.initServices()
 }
 
 // Close closes the BoltDB database.
 func (store *Store) Close() error {
-	if store.db != nil {
-		return store.db.Close()
+	if store.connection.DB != nil {
+		return store.connection.Close()
 	}
 	return nil
 }
@@ -110,274 +137,11 @@ func (store *Store) IsNew() bool {
 	return store.isNew
 }
 
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-func (store *Store) MigrateData() error {
-	if store.isNew {
-		return store.VersionService.StoreDBVersion(portainer.DBVersion)
-	}
-
-	version, err := store.VersionService.DBVersion()
-	if err == portainer.ErrObjectNotFound {
-		version = 0
-	} else if err != nil {
+// BackupTo backs up db to a provided writer.
+// It does hot backup and doesn't block other database reads and writes
+func (store *Store) BackupTo(w io.Writer) error {
+	return store.connection.View(func(tx *bolt.Tx) error {
+		_, err := tx.WriteTo(w)
 		return err
-	}
-
-	if version < portainer.DBVersion {
-		migratorParams := &migrator.Parameters{
-			DB:                      store.db,
-			DatabaseVersion:         version,
-			EndpointGroupService:    store.EndpointGroupService,
-			EndpointService:         store.EndpointService,
-			EndpointRelationService: store.EndpointRelationService,
-			ExtensionService:        store.ExtensionService,
-			RegistryService:         store.RegistryService,
-			ResourceControlService:  store.ResourceControlService,
-			RoleService:             store.RoleService,
-			ScheduleService:         store.ScheduleService,
-			SettingsService:         store.SettingsService,
-			StackService:            store.StackService,
-			TagService:              store.TagService,
-			TeamMembershipService:   store.TeamMembershipService,
-			UserService:             store.UserService,
-			VersionService:          store.VersionService,
-			FileService:             store.fileService,
-			AuthorizationService:    portainer.NewAuthorizationService(store),
-		}
-		migrator := migrator.NewMigrator(migratorParams)
-
-		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
-		err = migrator.Migrate()
-		if err != nil {
-			log.Printf("An error occurred during database migration: %s\n", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (store *Store) initServices() error {
-	authorizationsetService, err := role.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.RoleService = authorizationsetService
-
-	dockerhubService, err := dockerhub.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.DockerHubService = dockerhubService
-
-	edgeStackService, err := edgestack.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EdgeStackService = edgeStackService
-
-	edgeGroupService, err := edgegroup.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EdgeGroupService = edgeGroupService
-
-	endpointgroupService, err := endpointgroup.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointGroupService = endpointgroupService
-
-	endpointService, err := endpoint.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointService = endpointService
-
-	endpointRelationService, err := endpointrelation.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.EndpointRelationService = endpointRelationService
-
-	extensionService, err := extension.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ExtensionService = extensionService
-
-	registryService, err := registry.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.RegistryService = registryService
-
-	resourcecontrolService, err := resourcecontrol.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ResourceControlService = resourcecontrolService
-
-	settingsService, err := settings.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.SettingsService = settingsService
-
-	stackService, err := stack.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.StackService = stackService
-
-	tagService, err := tag.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TagService = tagService
-
-	teammembershipService, err := teammembership.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TeamMembershipService = teammembershipService
-
-	teamService, err := team.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TeamService = teamService
-
-	tunnelServerService, err := tunnelserver.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.TunnelServerService = tunnelServerService
-
-	userService, err := user.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.UserService = userService
-
-	versionService, err := version.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.VersionService = versionService
-
-	webhookService, err := webhook.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.WebhookService = webhookService
-
-	scheduleService, err := schedule.NewService(store.db)
-	if err != nil {
-		return err
-	}
-	store.ScheduleService = scheduleService
-
-	return nil
-}
-
-// DockerHub gives access to the DockerHub data management layer
-func (store *Store) DockerHub() portainer.DockerHubService {
-	return store.DockerHubService
-}
-
-// EdgeGroup gives access to the EdgeGroup data management layer
-func (store *Store) EdgeGroup() portainer.EdgeGroupService {
-	return store.EdgeGroupService
-}
-
-// EdgeStack gives access to the EdgeStack data management layer
-func (store *Store) EdgeStack() portainer.EdgeStackService {
-	return store.EdgeStackService
-}
-
-// Endpoint gives access to the Endpoint data management layer
-func (store *Store) Endpoint() portainer.EndpointService {
-	return store.EndpointService
-}
-
-// EndpointGroup gives access to the EndpointGroup data management layer
-func (store *Store) EndpointGroup() portainer.EndpointGroupService {
-	return store.EndpointGroupService
-}
-
-// EndpointRelation gives access to the EndpointRelation data management layer
-func (store *Store) EndpointRelation() portainer.EndpointRelationService {
-	return store.EndpointRelationService
-}
-
-// Extension gives access to the Extension data management layer
-func (store *Store) Extension() portainer.ExtensionService {
-	return store.ExtensionService
-}
-
-// Registry gives access to the Registry data management layer
-func (store *Store) Registry() portainer.RegistryService {
-	return store.RegistryService
-}
-
-// ResourceControl gives access to the ResourceControl data management layer
-func (store *Store) ResourceControl() portainer.ResourceControlService {
-	return store.ResourceControlService
-}
-
-// Role gives access to the Role data management layer
-func (store *Store) Role() portainer.RoleService {
-	return store.RoleService
-}
-
-// Schedule gives access to the Schedule data management layer
-func (store *Store) Schedule() portainer.ScheduleService {
-	return store.ScheduleService
-}
-
-// Settings gives access to the Settings data management layer
-func (store *Store) Settings() portainer.SettingsService {
-	return store.SettingsService
-}
-
-// Stack gives access to the Stack data management layer
-func (store *Store) Stack() portainer.StackService {
-	return store.StackService
-}
-
-// Tag gives access to the Tag data management layer
-func (store *Store) Tag() portainer.TagService {
-	return store.TagService
-}
-
-// TeamMembership gives access to the TeamMembership data management layer
-func (store *Store) TeamMembership() portainer.TeamMembershipService {
-	return store.TeamMembershipService
-}
-
-// Team gives access to the Team data management layer
-func (store *Store) Team() portainer.TeamService {
-	return store.TeamService
-}
-
-// TunnelServer gives access to the TunnelServer data management layer
-func (store *Store) TunnelServer() portainer.TunnelServerService {
-	return store.TunnelServerService
-}
-
-// User gives access to the User data management layer
-func (store *Store) User() portainer.UserService {
-	return store.UserService
-}
-
-// Version gives access to the Version data management layer
-func (store *Store) Version() portainer.VersionService {
-	return store.VersionService
-}
-
-// Webhook gives access to the Webhook data management layer
-func (store *Store) Webhook() portainer.WebhookService {
-	return store.WebhookService
+	})
 }

api/bolt/dockerhub/dockerhub.go

@@ -1,10 +1,8 @@
 package dockerhub
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing Dockerhub data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
-	err := internal.GetObject(service.db, BucketName, []byte(dockerHubKey), &dockerhub)
+	err := internal.GetObject(service.connection, BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(dockerHubKey), dockerhub)
+	return internal.UpdateObject(service.connection, BucketName, []byte(dockerHubKey), dockerhub)
 }

api/bolt/edgegroup/edgegroup.go

@@ -2,7 +2,7 @@ package edgegroup
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 	var groups = make([]portainer.EdgeGroup, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 	var group portainer.EdgeGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &group)
+	err := internal.GetObject(service.connection, BucketName, identifier, &group)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 // UpdateEdgeGroup updates an Edge group.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, group)
+	return internal.UpdateObject(service.connection, BucketName, identifier, group)
 }
 
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) CreateEdgeGroup(group *portainer.EdgeGroup) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/edgejob/edgejob.go

@@ -0,0 +1,101 @@
+package edgejob
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edgejobs"
+)
+
+// Service represents a service for managing edge jobs data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EdgeJobs returns a list of Edge jobs
+func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var edgeJob portainer.EdgeJob
+			err := internal.UnmarshalObject(v, &edgeJob)
+			if err != nil {
+				return err
+			}
+			edgeJobs = append(edgeJobs, edgeJob)
+		}
+
+		return nil
+	})
+
+	return edgeJobs, err
+}
+
+// EdgeJob returns an Edge job by ID
+func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
+}
+
+// CreateEdgeJob creates a new Edge job
+func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		if edgeJob.ID == 0 {
+			id, _ := bucket.NextSequence()
+			edgeJob.ID = portainer.EdgeJobID(id)
+		}
+
+		data, err := internal.MarshalObject(edgeJob)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(edgeJob.ID)), data)
+	})
+}
+
+// UpdateEdgeJob updates an Edge job by ID
+func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeJob)
+}
+
+// DeleteEdgeJob deletes an Edge job
+func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for an endpoint.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/edgestack/edgestack.go

@@ -2,7 +2,7 @@ package edgestack
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 	var stack portainer.EdgeStack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &stack)
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 
 // CreateEdgeStack assign an ID to a new Edge stack and saves it.
 func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeStack.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
 // UpdateEdgeStack updates an Edge stack.
 func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, edgeStack)
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/endpoint/endpoint.go

@@ -2,7 +2,7 @@ package endpoint
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 	var endpoint portainer.Endpoint
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpoint)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -44,20 +44,20 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 // UpdateEndpoint updates an endpoint.
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpoint)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpoint)
 }
 
 // DeleteEndpoint deletes an endpoint.
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // Endpoints return an array containing all the endpoints.
 func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -78,7 +78,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 
 // CreateEndpoint assign an ID to a new endpoint and saves it.
 func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for endpoints
@@ -98,12 +98,12 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 
 // GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }
 
 // Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		for _, endpoint := range toCreate {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,7 +1,7 @@
 package endpointgroup
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 	var endpointGroup portainer.EndpointGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpointGroup)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointGroup)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 // UpdateEndpointGroup updates an endpoint group.
 func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpointGroup)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointGroup)
 }
 
 // DeleteEndpointGroup deletes an endpoint group.
 func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // EndpointGroups return an array containing all the endpoint groups.
 func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -79,7 +79,7 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 
 // CreateEndpointGroup assign an ID to a new endpoint group and saves it.
 func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/endpointrelation/endpointrelation.go

@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint relation data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 	var endpointRelation portainer.EndpointRelation
 	identifier := internal.Itob(int(endpointID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &endpointRelation)
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointRelation)
 	if err != nil {
 		return nil, err
 	}
@@ -43,7 +43,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 
 // CreateEndpointRelation saves endpointRelation
 func (service *Service) CreateEndpointRelation(endpointRelation *portainer.EndpointRelation) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(endpointRelation)
@@ -58,11 +58,11 @@ func (service *Service) CreateEndpointRelation(endpointRelation *portainer.Endpo
 // UpdateEndpointRelation updates an Endpoint relation object
 func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.UpdateObject(service.db, BucketName, identifier, endpointRelation)
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointRelation)
 }
 
 // DeleteEndpointRelation deletes an Endpoint relation object
 func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/errors/errors.go

@@ -0,0 +1,8 @@
+package errors
+
+import "errors"
+
+var (
+	ErrObjectNotFound = errors.New("Object not found inside the database")
+	ErrMigrationToCE  = errors.New("DB is already on CE edition")
+)

api/bolt/extension/extension.go

@@ -1,7 +1,7 @@
 package extension
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 	var extension portainer.Extension
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &extension)
+	err := internal.GetObject(service.connection, BucketName, identifier, &extension)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(extension)
@@ -82,5 +82,5 @@ func (service *Service) Persist(extension *portainer.Extension) error {
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/helpers_test.go

@@ -0,0 +1,115 @@
+package bolt
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+var (
+	dataStorePath  string
+	testBackupPath string
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+	databaseFileName = fmt.Sprintf("portainer-%08d.db", rand.Intn(100000000))
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		log.Println(err)
+	}
+
+	dataStorePath = path.Join(pwd, "tmp")
+	testBackupPath = path.Join(dataStorePath, "backups")
+
+	teardown()
+}
+
+func NewTestStore(edition portainer.SoftwareEdition, version int, init bool) *Store {
+	fileService, err := filesystem.NewService(dataStorePath, "")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	store, err := NewStore(dataStorePath, fileService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = store.Open()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if init {
+		err = store.Init()
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	err = store.VersionService.StoreEdition(edition)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = store.VersionService.StoreDBVersion(version)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return store
+}
+
+func teardown() {
+	err := os.RemoveAll(testBackupPath)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	files, err := filepath.Glob(path.Join(dataStorePath, "portainer-*.*"))
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for _, f := range files {
+		if err := os.Remove(f); err != nil {
+			log.Fatalln(err)
+		}
+	}
+}
+
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
+func updateVersion(store *Store, v int) {
+	err := store.VersionService.StoreDBVersion(v)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func testVersion(store *Store, versionWant int, t *testing.T) {
+	if v, _ := store.version(); v != versionWant {
+		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
+	}
+}
+
+func testEdition(store *Store, editionWant portainer.SoftwareEdition, t *testing.T) {
+	if e := store.edition(); e != editionWant {
+		t.Errorf("Expect store edition to be %s but was %s", editionWant.GetEditionLabel(), e.GetEditionLabel())
+	}
+}

api/bolt/init.go

@@ -1,11 +1,31 @@
 package bolt
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	"github.com/gofrs/uuid"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+)
 
 // Init creates the default data set.
 func (store *Store) Init() error {
-	_, err := store.SettingsService.Settings()
-	if err == portainer.ErrObjectNotFound {
+	instanceID, err := store.VersionService.InstanceID()
+	if err == errors.ErrObjectNotFound {
+		uid, err := uuid.NewV4()
+		if err != nil {
+			return err
+		}
+
+		instanceID = uid.String()
+		err = store.VersionService.StoreInstanceID(instanceID)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	_, err = store.SettingsService.Settings()
+	if err == errors.ErrObjectNotFound {
 		defaultSettings := &portainer.Settings{
 			AuthenticationMethod: portainer.AuthenticationInternal,
 			BlackListedLabels:    make([]portainer.Pair, 0),
@@ -13,6 +33,7 @@ func (store *Store) Init() error {
 				AnonymousMode:   true,
 				AutoCreateUsers: true,
 				TLSConfig:       portainer.TLSConfiguration{},
+				URLs:            []string{},
 				SearchSettings: []portainer.LDAPSearchSettings{
 					portainer.LDAPSearchSettings{},
 				},
@@ -20,13 +41,14 @@ func (store *Store) Init() error {
 					portainer.LDAPGroupSearchSettings{},
 				},
 			},
-			OAuthSettings:                      portainer.OAuthSettings{},
-			AllowBindMountsForRegularUsers:     true,
-			AllowPrivilegedModeForRegularUsers: true,
-			AllowVolumeBrowserForRegularUsers:  false,
-			EnableHostManagementFeatures:       false,
-			EdgeAgentCheckinInterval:           portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-			TemplatesURL:                       portainer.DefaultTemplatesURL,
+			OAuthSettings: portainer.OAuthSettings{
+				TeamMemberships: portainer.TeamMemberships{
+					OAuthClaimMappings: make([]portainer.OAuthClaimMappings, 0),
+				},
+			},
+			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
+			TemplatesURL:             portainer.DefaultTemplatesURL,
+			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)
@@ -38,7 +60,7 @@ func (store *Store) Init() error {
 	}
 
 	_, err = store.DockerHubService.DockerHub()
-	if err == portainer.ErrObjectNotFound {
+	if err == errors.ErrObjectNotFound {
 		defaultDockerHub := &portainer.DockerHub{
 			Authentication: false,
 			Username:       "",
@@ -80,50 +102,7 @@ func (store *Store) Init() error {
 	}
 
 	if len(roles) == 0 {
-		environmentAdministratorRole := &portainer.Role{
-			Name:           "Endpoint administrator",
-			Description:    "Full control of all resources in an endpoint",
-			Priority:       1,
-			Authorizations: portainer.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
-		}
-
-		err = store.RoleService.CreateRole(environmentAdministratorRole)
-		if err != nil {
-			return err
-		}
-
-		environmentReadOnlyUserRole := &portainer.Role{
-			Name:           "Helpdesk",
-			Description:    "Read-only access of all resources in an endpoint",
-			Priority:       2,
-			Authorizations: portainer.DefaultEndpointAuthorizationsForHelpDeskRole(false),
-		}
-
-		err = store.RoleService.CreateRole(environmentReadOnlyUserRole)
-		if err != nil {
-			return err
-		}
-
-		standardUserRole := &portainer.Role{
-			Name:           "Standard user",
-			Description:    "Full control of assigned resources in an endpoint",
-			Priority:       3,
-			Authorizations: portainer.DefaultEndpointAuthorizationsForStandardUserRole(false),
-		}
-
-		err = store.RoleService.CreateRole(standardUserRole)
-		if err != nil {
-			return err
-		}
-
-		readOnlyUserRole := &portainer.Role{
-			Name:           "Read-only user",
-			Description:    "Read-only access of assigned resources in an endpoint",
-			Priority:       4,
-			Authorizations: portainer.DefaultEndpointAuthorizationsForReadOnlyUserRole(false),
-		}
-
-		err = store.RoleService.CreateRole(readOnlyUserRole)
+		err := store.RoleService.CreateOrUpdatePredefinedRoles()
 		if err != nil {
 			return err
 		}

api/bolt/internal/db.go

@@ -4,9 +4,13 @@ import (
 	"encoding/binary"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 )
 
+type DbConnection struct {
+	*bolt.DB
+}
+
 // Itob returns an 8-byte big endian representation of v.
 // This function is typically used for encoding integer IDs to byte slices
 // so that they can be used as BoltDB keys.
@@ -17,8 +21,8 @@ func Itob(v int) []byte {
 }
 
 // CreateBucket is a generic function used to create a bucket inside a bolt database.
-func CreateBucket(db *bolt.DB, bucketName string) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func CreateBucket(connection *DbConnection, bucketName string) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
 		if err != nil {
 			return err
@@ -28,15 +32,15 @@ func CreateBucket(db *bolt.DB, bucketName string) error {
 }
 
 // GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
-func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+func GetObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
 	var data []byte
 
-	err := db.View(func(tx *bolt.Tx) error {
+	err := connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		value := bucket.Get(key)
 		if value == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		data = make([]byte, len(value))
@@ -52,8 +56,8 @@ func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) e
 }
 
 // UpdateObject is a generic function used to update an object inside a bolt database.
-func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func UpdateObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		data, err := MarshalObject(object)
@@ -71,18 +75,18 @@ func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}
 }
 
 // DeleteObject is a generic function used to delete an object inside a bolt database.
-func DeleteObject(db *bolt.DB, bucketName string, key []byte) error {
-	return db.Update(func(tx *bolt.Tx) error {
+func DeleteObject(connection *DbConnection, bucketName string, key []byte) error {
+	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
 }
 
 // GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
-func GetNextIdentifier(db *bolt.DB, bucketName string) int {
+func GetNextIdentifier(connection *DbConnection, bucketName string) int {
 	var identifier int
 
-	db.Update(func(tx *bolt.Tx) error {
+	connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {

api/bolt/license/license.go

@@ -0,0 +1,92 @@
+package license
+
+import (
+	"github.com/portainer/liblicense"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "license"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// License returns a license by licenseKey
+func (service *Service) License(licenseKey string) (*liblicense.PortainerLicense, error) {
+	var license liblicense.PortainerLicense
+	identifier := []byte(licenseKey)
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &license)
+	if err != nil {
+		return nil, err
+	}
+
+	return &license, nil
+}
+
+// Licenses return an array containing all the licenses.
+func (service *Service) Licenses() ([]liblicense.PortainerLicense, error) {
+	var licenses = make([]liblicense.PortainerLicense, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var license liblicense.PortainerLicense
+			err := internal.UnmarshalObject(v, &license)
+			if err != nil {
+				return err
+			}
+			licenses = append(licenses, license)
+		}
+
+		return nil
+	})
+
+	return licenses, err
+}
+
+// AddLicense persists a license inside the database.
+func (service *Service) AddLicense(licenseKey string, license *liblicense.PortainerLicense) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(license)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put([]byte(licenseKey), data)
+	})
+}
+
+// UpdateLicense updates a license.
+func (service *Service) UpdateLicense(licenseKey string, license *liblicense.PortainerLicense) error {
+	identifier := []byte(licenseKey)
+	return internal.UpdateObject(service.connection, BucketName, identifier, license)
+}
+
+// DeleteLicense deletes a License.
+func (service *Service) DeleteLicense(licenseKey string) error {
+	identifier := []byte(licenseKey)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}

api/bolt/log/log.go

@@ -0,0 +1,41 @@
+package log
+
+import (
+	"fmt"
+	"log"
+)
+
+const (
+	INFO  = "INFO"
+	ERROR = "ERROR"
+	DEBUG = "DEBUG"
+	FATAL = "FATAL"
+)
+
+type ScopedLog struct {
+	scope string
+}
+
+func NewScopedLog(scope string) *ScopedLog {
+	return &ScopedLog{scope: scope}
+}
+
+func (slog *ScopedLog) print(kind string, message string) {
+	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
+}
+
+func (slog *ScopedLog) Debug(message string) {
+	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Info(message string) {
+	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
+}
+
+func (slog *ScopedLog) Error(message string, err error) {
+	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
+}
+
+func (slog *ScopedLog) NotImplemented(method string) {
+	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
+}

api/bolt/log/log.test.go

@@ -0,0 +1 @@
+package log

api/bolt/migrate_data.go

@@ -0,0 +1,207 @@
+package bolt
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	errors "github.com/portainer/portainer/api/bolt/errors"
+	plog "github.com/portainer/portainer/api/bolt/log"
+	"github.com/portainer/portainer/api/bolt/migrator"
+	"github.com/portainer/portainer/api/cli"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+const beforePortainerUpgradeToEEBackup = "portainer.db.before-EE-upgrade"
+
+var migrateLog = plog.NewScopedLog("bolt, migrate")
+
+// FailSafeMigrate backup and restore DB if migration fail
+func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version int) error {
+	defer func() {
+		if err := recover(); err != nil {
+			migrateLog.Info(fmt.Sprintf("Error during migration, recovering [%v]", err))
+			store.Restore()
+		}
+	}()
+	return migrator.Migrate(version)
+}
+
+// MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
+// if force is true, then migrate regardless.
+func (store *Store) MigrateData(force bool) error {
+	// 0 – if DB is new then we don't need to migrate any data and just set version and edition to latest EE
+	if store.isNew && !force {
+		err := store.VersionService.StoreDBVersion(portainer.DBVersionEE)
+		if err != nil {
+			return err
+		}
+
+		err = store.VersionService.StoreEdition(portainer.PortainerEE)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	migrator, err := store.newMigrator()
+	if err != nil {
+		return err
+	}
+
+	if migrator.Edition() == portainer.PortainerCE {
+		// backup before migrating
+		store.BackupWithOptions(&BackupOptions{
+			BackupFileName: beforePortainerUpgradeToEEBackup,
+			Edition:        portainer.PortainerCE,
+		})
+
+		store.VersionService.StorePreviousDBVersion(migrator.Version())
+
+		// 1 – We need to migrate DB to latest CE version
+
+		if migrator.Version() < portainer.DBVersion {
+			store.Backup()
+			err = store.FailSafeMigrate(migrator, portainer.DBVersion)
+			if err != nil {
+				store.Restore()
+				migrateLog.Error("An error occurred while migrating CE database to latest version", err)
+				return err
+			}
+		}
+
+	}
+
+	if portainer.Edition == portainer.PortainerEE {
+		// 2 – if DB is CE Edition we need to upgrade settings to EE
+		if migrator.Edition() < portainer.PortainerEE {
+			err = migrator.UpgradeToEE()
+			if err != nil {
+				migrateLog.Error("An error occurred while upgrading database to EE", err)
+				store.RollbackFailedUpgradeToEE()
+				return err
+			}
+
+		}
+
+		// 3 – if DB is EE Edition we need to migrate to latest version of EE
+		if migrator.Edition() == portainer.PortainerEE && migrator.Version() < portainer.DBVersionEE {
+			store.Backup()
+			err = store.FailSafeMigrate(migrator, portainer.DBVersionEE)
+			if err != nil {
+				migrateLog.Error("An error occurred while migrating EE database to latest version", err)
+				store.Restore()
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// RollbackFailedUpgradeToEE down migrate to previous version
+func (store *Store) RollbackFailedUpgradeToEE() error {
+	return store.RestoreWithOptions(&BackupOptions{
+		BackupFileName: beforePortainerUpgradeToEEBackup,
+		Edition:        portainer.PortainerCE,
+	})
+}
+
+// RollbackToCE rollbacks the store to the current ce version
+func (store *Store) RollbackToCE() error {
+
+	migrator, err := store.newMigrator()
+	if err != nil {
+		return err
+	}
+
+	migrateLog.Info(fmt.Sprintf("Current Software Edition: %s", migrator.Edition().GetEditionLabel()))
+	migrateLog.Info(fmt.Sprintf("Current DB Version: %d", migrator.Version()))
+
+	if migrator.Edition() == portainer.PortainerCE {
+		return errors.ErrMigrationToCE
+	}
+
+	previousVersion, err := store.VersionService.PreviousDBVersion()
+	if err != nil {
+		migrateLog.Error("An Error occurred with retrieving previous DB version", err)
+		return err
+	}
+
+	confirmed, err := cli.Confirm(fmt.Sprintf("Are you sure you want to rollback your database to %d?", previousVersion))
+	if err != nil || !confirmed {
+		return err
+	}
+
+	if previousVersion < 25 {
+		migrator.DowngradeSettingsFrom25()
+	}
+
+	err = store.VersionService.StoreDBVersion(previousVersion)
+	if err != nil {
+		migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err)
+		return err
+	}
+
+	err = store.VersionService.StoreEdition(portainer.PortainerCE)
+	if err != nil {
+		migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err)
+		return err
+	}
+
+	migrateLog.Info(fmt.Sprintf("Rolled back to CE Edition, DB Version %d", previousVersion))
+	return nil
+}
+
+func (store *Store) newMigrator() (*migrator.Migrator, error) {
+
+	version, err := store.version()
+	if err != nil {
+		return nil, err
+	}
+
+	edition := store.edition()
+
+	params := &migrator.Parameters{
+		DB:              store.connection.DB,
+		DatabaseVersion: version,
+		CurrentEdition:  edition,
+
+		EndpointGroupService:    store.EndpointGroupService,
+		EndpointService:         store.EndpointService,
+		EndpointRelationService: store.EndpointRelationService,
+		ExtensionService:        store.ExtensionService,
+		RegistryService:         store.RegistryService,
+		ResourceControlService:  store.ResourceControlService,
+		RoleService:             store.RoleService,
+		ScheduleService:         store.ScheduleService,
+		SettingsService:         store.SettingsService,
+		StackService:            store.StackService,
+		TagService:              store.TagService,
+		TeamMembershipService:   store.TeamMembershipService,
+		UserService:             store.UserService,
+		VersionService:          store.VersionService,
+		FileService:             store.fileService,
+		AuthorizationService:    authorization.NewService(store),
+	}
+
+	return migrator.NewMigrator(params), nil
+}
+
+// RollbackVersion down migrate to previous version
+func (store *Store) RollbackVersion(version int) error {
+	// TODO
+	backupLog.NotImplemented("RollbackVersion")
+	return nil
+}
+
+// RollbackEdition downgrade to previous edition
+func (store *Store) RollbackEdition(edition portainer.SoftwareEdition) error {
+	// TODO
+	backupLog.NotImplemented("RollbackEdition")
+	// Change Edition
+	// Migrate Services
+	// Restore Latest
+	return nil
+}

api/bolt/migrate_data_test.go

@@ -0,0 +1,99 @@
+package bolt
+
+import (
+	"fmt"
+	"log"
+	"testing"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+// New Database should be EE and DBVersion
+//
+
+func TestMigrateData(t *testing.T) {
+	var store *Store
+
+	t.Run("MigrateData for New Store", func(t *testing.T) {
+		fileService, err := filesystem.NewService(dataStorePath, "")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		store, err := NewStore(dataStorePath, fileService)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = store.Open()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = store.Init()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		store.MigrateData(false)
+
+		testVersion(store, portainer.DBVersionEE, t)
+		testEdition(store, portainer.PortainerEE, t)
+
+		store.Close()
+	})
+
+	tests := []struct {
+		edition         portainer.SoftwareEdition
+		version         int
+		expectedVersion int
+	}{
+		{edition: portainer.PortainerCE, version: 5, expectedVersion: portainer.DBVersionEE},
+		{edition: portainer.PortainerCE, version: 21, expectedVersion: portainer.DBVersionEE},
+	}
+
+	for _, tc := range tests {
+		store = NewTestStore(tc.edition, tc.version, true)
+		t.Run(fmt.Sprintf("MigrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) {
+			store.MigrateData(false)
+			testVersion(store, tc.expectedVersion, t)
+			testEdition(store, portainer.PortainerEE, t)
+		})
+
+		t.Run(fmt.Sprintf("Restoring DB after migrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) {
+			store.RollbackToCE()
+			testVersion(store, tc.version, t)
+			testEdition(store, tc.edition, t)
+		})
+
+		store.Close()
+	}
+
+	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
+		version := 21
+		store = NewTestStore(portainer.PortainerCE, version, true)
+
+		deleteBucket(store.connection.DB, "settings")
+		store.MigrateData(false)
+
+		testVersion(store, version, t)
+		testEdition(store, portainer.PortainerCE, t)
+
+		store.Close()
+	})
+
+	teardown()
+}
+
+func deleteBucket(db *bolt.DB, bucketName string) {
+	db.Update(func(tx *bolt.Tx) error {
+		log.Printf("Delete bucket %s\n", bucketName)
+		err := tx.DeleteBucket([]byte(bucketName))
+		if err != nil {
+			log.Println(err)
+		}
+		return err
+	})
+}

api/bolt/migrator/downgrade_dbversion25.go

@@ -0,0 +1,15 @@
+package migrator
+
+// DowngradeSettingsFrom25 downgrade template settings for portainer v1.2
+func (migrator *Migrator) DowngradeSettingsFrom25() error {
+	legacySettings, err := migrator.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.TemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates-1.20.0.json"
+
+	err = migrator.settingsService.UpdateSettings(legacySettings)
+
+	return err
+}

api/bolt/migrator/migrate_ce.go

@@ -0,0 +1,308 @@
+package migrator
+
+import (
+	"log"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+// MigrateCE checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) MigrateCE() error {
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return err
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.23.0
+	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+	if m.currentDBVersion < 22 {
+		err := m.updateResourceControlsToDBVersion22()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateUsersAndRolesToDBVersion22()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.0
+	if m.currentDBVersion < 23 {
+		err := m.updateTagsToDBVersion23()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.1
+	if m.currentDBVersion < 24 {
+		err := m.updateSettingsToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.0.0
+	if m.currentDBVersion < 25 {
+		err := m.updateSettingsToDB25()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.1.0
+	if m.currentDBVersion < 26 {
+		err := m.updateEndpointSettingsToDB26()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRbacRolesToDB26()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.2.0
+	if m.currentDBVersion < 27 {
+		err := m.updateStackResourceControlToDB27()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer EE-2.4.0
+	if m.currentDBVersion < 28 {
+		err := m.updateUsersAndRolesToDBVersion28()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer EE-2.4.0
+	if m.currentDBVersion < 29 {
+		err := m.updateRbacRolesToDB29()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer EE-2.7.0
+	if m.currentDBVersion < 31 {
+		err := m.updateSettingsToDB31()
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Println("Update DB version to ", portainer.DBVersion)
+	return m.versionService.StoreDBVersion(portainer.DBVersion)
+}

api/bolt/migrator/migrate_dbversion0.go

@@ -3,6 +3,7 @@ package migrator
 import (
 	"github.com/boltdb/bolt"
 	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/user"
 )
 
@@ -22,7 +23,7 @@ func (m *Migrator) updateAdminUserToDBVersion1() error {
 		if err != nil {
 			return err
 		}
-	} else if err != nil && err != portainer.ErrObjectNotFound {
+	} else if err != nil && err != errors.ErrObjectNotFound {
 		return err
 	}
 	return nil

api/bolt/migrator/migrate_dbversion19.go

@@ -2,10 +2,10 @@ package migrator
 
 import (
 	"strings"
-
-	portainer "github.com/portainer/portainer/api"
 )
 
+const scheduleScriptExecutionJobType = 1
+
 func (m *Migrator) updateUsersToDBVersion20() error {
 	return m.authorizationService.UpdateUsersAuthorizations()
 }
@@ -28,7 +28,7 @@ func (m *Migrator) updateSchedulesToDBVersion20() error {
 	}
 
 	for _, schedule := range legacySchedules {
-		if schedule.JobType == portainer.ScriptExecutionJobType {
+		if schedule.JobType == scheduleScriptExecutionJobType {
 			if schedule.CronExpression == "0 0 * * *" {
 				schedule.CronExpression = "0 * * * *"
 			} else if schedule.CronExpression == "0 0 0/2 * *" {

api/bolt/migrator/migrate_dbversion20.go

@@ -1,6 +1,9 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
 
 func (m *Migrator) updateResourceControlsToDBVersion22() error {
 	legacyResourceControls, err := m.resourceControlService.ResourceControls()
@@ -26,13 +29,8 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
 	for _, user := range legacyUsers {
-		user.PortainerAuthorizations = portainer.DefaultPortainerAuthorizations()
+		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
 		err = m.userService.UpdateUser(user.ID, &user)
 		if err != nil {
 			return err
@@ -44,7 +42,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	endpointAdministratorRole.Priority = 1
-	endpointAdministratorRole.Authorizations = portainer.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
+	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
 
 	err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
 
@@ -53,7 +51,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	helpDeskRole.Priority = 2
-	helpDeskRole.Authorizations = portainer.DefaultEndpointAuthorizationsForHelpDeskRole(settings.AllowVolumeBrowserForRegularUsers)
+	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole()
 
 	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
 
@@ -62,7 +60,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	standardUserRole.Priority = 3
-	standardUserRole.Authorizations = portainer.DefaultEndpointAuthorizationsForStandardUserRole(settings.AllowVolumeBrowserForRegularUsers)
+	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole()
 
 	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
 
@@ -71,7 +69,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		return err
 	}
 	readOnlyUserRole.Priority = 4
-	readOnlyUserRole.Authorizations = portainer.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
+	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole()
 
 	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
 	if err != nil {

api/bolt/migrator/migrate_dbversion23.go

@@ -8,10 +8,26 @@ func (m *Migrator) updateSettingsToDB24() error {
 		return err
 	}
 
-	if legacySettings.TemplatesURL == "" {
-		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
+	legacySettings.AllowHostNamespaceForRegularUsers = true
+	legacySettings.AllowDeviceMappingForRegularUsers = true
+	legacySettings.AllowStackManagementForRegularUsers = true
 
-		return m.settingsService.UpdateSettings(legacySettings)
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+func (m *Migrator) updateStacksToDB24() error {
+	stacks, err := m.stackService.Stacks()
+	if err != nil {
+		return err
+	}
+
+	for idx := range stacks {
+		stack := &stacks[idx]
+		stack.Status = portainer.StackStatusActive
+		err := m.stackService.UpdateStack(stack.ID, stack)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil

api/bolt/migrator/migrate_dbversion24.go

@@ -0,0 +1,23 @@
+package migrator
+
+import (
+	"github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateSettingsToDB25() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.TemplatesURL == "" {
+		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
+	}
+
+	legacySettings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
+	legacySettings.EnableTelemetry = true
+
+	legacySettings.AllowContainerCapabilitiesForRegularUsers = true
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion25.go

@@ -0,0 +1,76 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+func (m *Migrator) updateEndpointSettingsToDB26() error {
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for i := range endpoints {
+		endpoint := endpoints[i]
+
+		securitySettings := portainer.EndpointSecuritySettings{}
+
+		if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment ||
+			endpoint.Type == portainer.AgentOnDockerEnvironment ||
+			endpoint.Type == portainer.DockerEnvironment {
+
+			securitySettings = portainer.EndpointSecuritySettings{
+				AllowBindMountsForRegularUsers:            settings.AllowBindMountsForRegularUsers,
+				AllowContainerCapabilitiesForRegularUsers: settings.AllowContainerCapabilitiesForRegularUsers,
+				AllowDeviceMappingForRegularUsers:         settings.AllowDeviceMappingForRegularUsers,
+				AllowHostNamespaceForRegularUsers:         settings.AllowHostNamespaceForRegularUsers,
+				AllowPrivilegedModeForRegularUsers:        settings.AllowPrivilegedModeForRegularUsers,
+				AllowStackManagementForRegularUsers:       settings.AllowStackManagementForRegularUsers,
+			}
+
+			if endpoint.Type == portainer.AgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+				securitySettings.AllowVolumeBrowserForRegularUsers = settings.AllowVolumeBrowserForRegularUsers
+				securitySettings.EnableHostManagementFeatures = settings.EnableHostManagementFeatures
+			}
+		}
+
+		endpoint.SecuritySettings = securitySettings
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateRbacRolesToDB26() error {
+	defaultAuthorizationsOfRoles := map[portainer.RoleID]portainer.Authorizations{
+		portainer.RoleIDEndpointAdmin: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
+		portainer.RoleIDHelpdesk:      authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
+		portainer.RoleIDStandardUser:  authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
+		portainer.RoleIDReadonly:      authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
+	}
+
+	for roleID, defaultAuthorizations := range defaultAuthorizationsOfRoles {
+		role, err := m.roleService.Role(roleID)
+		if err != nil {
+			return err
+		}
+		role.Authorizations = defaultAuthorizations
+
+		err = m.roleService.UpdateRole(role.ID, role)
+		if err != nil {
+			return err
+		}
+	}
+
+	return m.authorizationService.UpdateUsersAuthorizations()
+}

api/bolt/migrator/migrate_dbversion26.go

@@ -0,0 +1,43 @@
+package migrator
+
+import (
+	"fmt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+)
+
+func (m *Migrator) updateStackResourceControlToDB27() error {
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resource := range resourceControls {
+		if resource.Type != portainer.StackResourceControl {
+			continue
+		}
+
+		stackName := resource.ResourceID
+		if err != nil {
+			return err
+		}
+
+		stack, err := m.stackService.StackByName(stackName)
+		if err != nil {
+			if err == errors.ErrObjectNotFound {
+				continue
+			}
+
+			return err
+		}
+
+		resource.ResourceID = fmt.Sprintf("%d_%s", stack.EndpointID, stack.Name)
+
+		err = m.resourceControlService.UpdateResourceControl(resource.ID, &resource)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion27.go

@@ -0,0 +1,10 @@
+package migrator
+
+func (m *Migrator) updateUsersAndRolesToDBVersion28() error {
+	err := m.roleService.CreateOrUpdatePredefinedRoles()
+	if err != nil {
+		return err
+	}
+
+	return m.authorizationService.UpdateUsersAuthorizations()
+}

api/bolt/migrator/migrate_dbversion28.go

@@ -0,0 +1,31 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+func (m *Migrator) updateRbacRolesToDB29() error {
+	defaultAuthorizationsOfRoles := map[portainer.RoleID]portainer.Authorizations{
+		portainer.RoleIDEndpointAdmin: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
+		portainer.RoleIDHelpdesk:      authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
+		portainer.RoleIDOperator:      authorization.DefaultEndpointAuthorizationsForOperatorRole(),
+		portainer.RoleIDStandardUser:  authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
+		portainer.RoleIDReadonly:      authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
+	}
+
+	for roleID, defaultAuthorizations := range defaultAuthorizationsOfRoles {
+		role, err := m.roleService.Role(roleID)
+		if err != nil {
+			return err
+		}
+		role.Authorizations = defaultAuthorizations
+
+		err = m.roleService.UpdateRole(role.ID, role)
+		if err != nil {
+			return err
+		}
+	}
+
+	return m.authorizationService.UpdateUsersAuthorizations()
+}

api/bolt/migrator/migrate_dbversion30.go

@@ -0,0 +1,12 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDB31() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	legacySettings.OAuthSettings.SSO = false
+	legacySettings.OAuthSettings.HideInternalAuth = false
+	legacySettings.OAuthSettings.LogoutURI = ""
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion30_test.go

@@ -0,0 +1,67 @@
+package migrator
+
+import (
+	"os"
+	"testing"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+var (
+	testingDBStorePath string
+	testingDBFileName  string
+	dummyLogoURL       string
+	dbConn             *bolt.DB
+	settingsService    *settings.Service
+)
+
+func setup() error {
+	testingDBStorePath, _ = os.Getwd()
+	testingDBFileName = "portainer-ee-mig-30.db"
+	dummyLogoURL = "example.com"
+	var err error
+	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
+	if err != nil {
+		return err
+	}
+	dummySettingsObj := map[string]interface{}{
+		"LogoURL": dummyLogoURL,
+	}
+	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestUpdateSettingsToDB31(t *testing.T) {
+	if err := setup(); err != nil {
+		t.Errorf("failed to complete testing setups, err: %v", err)
+	}
+	defer dbConn.Close()
+	defer os.Remove(testingDBFileName)
+	m := &Migrator{
+		db:              dbConn,
+		settingsService: settingsService,
+	}
+	if err := m.updateSettingsToDB31(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+	updatedSettings, err := m.settingsService.Settings()
+	if err != nil {
+		t.Errorf("failed to retrieve the updated settings: %v", err)
+	}
+	if updatedSettings.LogoURL != dummyLogoURL {
+		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
+	}
+	if updatedSettings.OAuthSettings.SSO != false {
+		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
+	}
+	if updatedSettings.OAuthSettings.HideInternalAuth != false {
+		t.Errorf("unexpected default OAuth HideInternalAuth setting, want: false, got: %t", updatedSettings.OAuthSettings.HideInternalAuth)
+	}
+	if updatedSettings.OAuthSettings.LogoutURI != "" {
+		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
+	}
+}

api/bolt/migrator/migrate_test_helper.go

@@ -0,0 +1,38 @@
+package migrator
+
+import (
+	"path"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+// initTestingDBConn creates a raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
+	databasePath := path.Join(storePath, fileName)
+	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+	return dbConn, nil
+}
+
+// initTestingDBConn creates a settings service with raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
+	internalDBConn := &internal.DbConnection{
+		DB: dbConn,
+	}
+	settingsService, err := settings.NewService(internalDBConn)
+	if err != nil {
+		return nil, err
+	}
+	//insert a obj
+	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
+		return nil, err
+	}
+	return settingsService, nil
+}

api/bolt/migrator/migrator.go

@@ -1,12 +1,15 @@
 package migrator
 
 import (
+	"fmt"
+
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
+	plog "github.com/portainer/portainer/api/bolt/log"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -17,13 +20,18 @@ import (
 	"github.com/portainer/portainer/api/bolt/teammembership"
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/internal/authorization"
 )
 
+var migrateLog = plog.NewScopedLog("bolt, migrate")
+
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		currentDBVersion        int
-		db                      *bolt.DB
+		db               *bolt.DB
+		currentDBVersion int
+		currentEdition   portainer.SoftwareEdition
+
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -39,13 +47,15 @@ type (
 		userService             *user.Service
 		versionService          *version.Service
 		fileService             portainer.FileService
-		authorizationService    *portainer.AuthorizationService
+		authorizationService    *authorization.Service
 	}
 
 	// Parameters represents the required parameters to create a new Migrator instance.
 	Parameters struct {
-		DB                      *bolt.DB
-		DatabaseVersion         int
+		DB              *bolt.DB
+		DatabaseVersion int
+		CurrentEdition  portainer.SoftwareEdition
+
 		EndpointGroupService    *endpointgroup.Service
 		EndpointService         *endpoint.Service
 		EndpointRelationService *endpointrelation.Service
@@ -61,7 +71,7 @@ type (
 		UserService             *user.Service
 		VersionService          *version.Service
 		FileService             portainer.FileService
-		AuthorizationService    *portainer.AuthorizationService
+		AuthorizationService    *authorization.Service
 	}
 )
 
@@ -70,6 +80,7 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	return &Migrator{
 		db:                      parameters.DB,
 		currentDBVersion:        parameters.DatabaseVersion,
+		currentEdition:          parameters.CurrentEdition,
 		endpointGroupService:    parameters.EndpointGroupService,
 		endpointService:         parameters.EndpointService,
 		endpointRelationService: parameters.EndpointRelationService,
@@ -89,244 +100,43 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	}
 }
 
-// Migrate checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) Migrate() error {
-	// Portainer < 1.12
-	if m.currentDBVersion < 1 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.currentDBVersion < 2 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return err
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.13.x
-	if m.currentDBVersion < 3 {
-		err := m.updateSettingsToDBVersion3()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.14.0
-	if m.currentDBVersion < 4 {
-		err := m.updateEndpointsToDBVersion4()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1235
-	if m.currentDBVersion < 5 {
-		err := m.updateSettingsToVersion5()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1236
-	if m.currentDBVersion < 6 {
-		err := m.updateSettingsToVersion6()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1449
-	if m.currentDBVersion < 7 {
-		err := m.updateSettingsToVersion7()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 8 {
-		err := m.updateEndpointsToVersion8()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https: //github.com/portainer/portainer/issues/1396
-	if m.currentDBVersion < 9 {
-		err := m.updateEndpointsToVersion9()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/461
-	if m.currentDBVersion < 10 {
-		err := m.updateEndpointsToVersion10()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1906
-	if m.currentDBVersion < 11 {
-		err := m.updateEndpointsToVersion11()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.18.0
-	if m.currentDBVersion < 12 {
-		err := m.updateEndpointsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateStacksToVersion12()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.0
-	if m.currentDBVersion < 13 {
-		err := m.updateSettingsToVersion13()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.2
-	if m.currentDBVersion < 14 {
-		err := m.updateResourceControlsToDBVersion14()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.0
-	if m.currentDBVersion < 15 {
-		err := m.updateSettingsToDBVersion15()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateTemplatesToVersion15()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 16 {
-		err := m.updateSettingsToDBVersion16()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.1
-	if m.currentDBVersion < 17 {
-		err := m.updateExtensionsToDBVersion17()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.0
-	if m.currentDBVersion < 24 {
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return err
-		}
-	}
-
-	return m.versionService.StoreDBVersion(portainer.DBVersion)
+// Version exposes version of database
+func (migrator *Migrator) Version() int {
+	return migrator.currentDBVersion
+}
+
+// Edition exposes edition of portainer
+func (migrator *Migrator) Edition() portainer.SoftwareEdition {
+	return migrator.currentEdition
+}
+
+// Migrate helper to upgrade DB
+func (migrator *Migrator) Migrate(version int) error {
+
+	migrateLog.Info(fmt.Sprintf("Migrating %s database from version %d to %d.", migrator.Edition().GetEditionLabel(), migrator.currentDBVersion, version))
+	// TODO : run backup before migration and restore if failed
+	err := migrator.MigrateCE() //CE
+	if err != nil {
+		migrateLog.Error("An error occurred during database migration", err)
+		return err
+	}
+
+	migrator.versionService.StoreDBVersion(version)
+	migrator.currentDBVersion = version
+
+	return nil
+}
+
+// RollbackVersion rolls back the db to version
+func (migrator *Migrator) RollbackVersion(version int) error {
+
+	err := migrator.versionService.StoreDBVersion(version) // portainer.DBVersion
+	return err
+}
+
+// RollbackEdition rolls back the db to portainer CE
+func (migrator *Migrator) RollbackEdition(edition portainer.SoftwareEdition) error {
+
+	err := migrator.versionService.StoreEdition(portainer.PortainerCE)
+	return err
 }

api/bolt/migrator/migrator_test.go

@@ -0,0 +1,4 @@
+package migrator
+
+// test CE version is always upgraded to latest version of CE
+// test EE version is always upgraded to latest version of EE

api/bolt/migrator/upgrade_ee.go

@@ -0,0 +1,228 @@
+package migrator
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+// UpgradeToEE will migrate the db from latest ce version to latest ee version
+// Latest version is v25 on 06/11/2020
+func (m *Migrator) UpgradeToEE() error {
+
+	migrateLog.Info(fmt.Sprintf("Migrating CE database version %d to EE database version %d.", m.Version(), portainer.DBVersion))
+
+	migrateLog.Info("Updating LDAP settings to EE")
+	err := m.updateSettingsToEE()
+	if err != nil {
+		return err
+	}
+
+	migrateLog.Info("Updating user roles to EE")
+	err = m.updateUserRolesToEE()
+	if err != nil {
+		return err
+	}
+	migrateLog.Info("Updating role authorizations to EE")
+	err = m.updateRoleAuthorizationsToEE()
+	if err != nil {
+		return err
+	}
+	migrateLog.Info("Updating user authorizations")
+	err = m.authorizationService.UpdateUsersAuthorizations()
+	if err != nil {
+		return err
+	}
+
+	migrateLog.Info(fmt.Sprintf("Setting db version to %d", portainer.DBVersionEE))
+	err = m.versionService.StoreDBVersion(portainer.DBVersionEE)
+	if err != nil {
+		return err
+	}
+
+	migrateLog.Info(fmt.Sprintf("Setting edition to %s", portainer.PortainerEE.GetEditionLabel()))
+	err = m.versionService.StoreEdition(portainer.PortainerEE)
+	if err != nil {
+		return err
+	}
+
+	m.currentDBVersion = portainer.DBVersionEE
+	m.currentEdition = portainer.PortainerEE
+
+	return nil
+}
+
+func (m *Migrator) updateSettingsToEE() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.LDAPSettings.URLs = []string{}
+	url := legacySettings.LDAPSettings.URL
+	if url != "" {
+		legacySettings.LDAPSettings.URLs = append(legacySettings.LDAPSettings.URLs, url)
+	}
+
+	legacySettings.LDAPSettings.ServerType = portainer.LDAPServerCustom
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+// Updating role authorizations because of the new policies in Kube RBAC
+func (m *Migrator) updateRoleAuthorizationsToEE() error {
+	migrateLog.Debug("Retriving settings")
+
+	migrateLog.Debug("Updating Endpoint Admin Role")
+	endpointAdministratorRole, err := m.roleService.Role(portainer.RoleID(1))
+	if err != nil {
+		return err
+	}
+	endpointAdministratorRole.Priority = 1
+	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
+
+	err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
+
+	migrateLog.Debug("Updating Help Desk Role")
+	helpDeskRole, err := m.roleService.Role(portainer.RoleID(2))
+	if err != nil {
+		return err
+	}
+	helpDeskRole.Priority = 2
+	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole()
+
+	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
+
+	migrateLog.Debug("Updating Standard User Role")
+	standardUserRole, err := m.roleService.Role(portainer.RoleID(3))
+	if err != nil {
+		return err
+	}
+	standardUserRole.Priority = 3
+	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole()
+
+	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
+
+	migrateLog.Debug("Updating Read Only User Role")
+	readOnlyUserRole, err := m.roleService.Role(portainer.RoleID(4))
+	if err != nil {
+		return err
+	}
+	readOnlyUserRole.Priority = 4
+	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole()
+
+	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// If RBAC extension wasn't installed before, update all users in endpoints and
+// endpoint groups to have read only access.
+func (m *Migrator) updateUserRolesToEE() error {
+	err := m.updateUserAuthorizationToEE()
+	if err != nil {
+		return err
+	}
+
+	migrateLog.Debug("Retriving extension info")
+	extensions, err := m.extensionService.Extensions()
+	for _, extension := range extensions {
+		if extension.ID == 3 && extension.Enabled {
+			migrateLog.Info("RBAC extensions were enabled before; Skip updating User Roles")
+			return nil
+		}
+	}
+
+	migrateLog.Debug("Retriving endpoint groups")
+	endpointGroups, err := m.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range endpointGroups {
+		migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint group %v", endpointGroup.ID))
+		for key := range endpointGroup.UserAccessPolicies {
+			updateUserAccessPolicyToReadOnlyRole(endpointGroup.UserAccessPolicies, key)
+		}
+
+		for key := range endpointGroup.TeamAccessPolicies {
+			updateTeamAccessPolicyToReadOnlyRole(endpointGroup.TeamAccessPolicies, key)
+		}
+
+		err := m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	migrateLog.Debug("Retriving endpoints")
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint %v", endpoint.ID))
+		for key := range endpoint.UserAccessPolicies {
+			updateUserAccessPolicyToReadOnlyRole(endpoint.UserAccessPolicies, key)
+		}
+
+		for key := range endpoint.TeamAccessPolicies {
+			updateTeamAccessPolicyToReadOnlyRole(endpoint.TeamAccessPolicies, key)
+		}
+
+		err := m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateUserAuthorizationToEE() error {
+	legacyUsers, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
+
+		err = m.userService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func updateUserAccessPolicyToNoRole(policies portainer.UserAccessPolicies, key portainer.UserID) {
+	tmp := policies[key]
+	tmp.RoleID = 0
+	policies[key] = tmp
+}
+
+func updateTeamAccessPolicyToNoRole(policies portainer.TeamAccessPolicies, key portainer.TeamID) {
+	tmp := policies[key]
+	tmp.RoleID = 0
+	policies[key] = tmp
+}
+
+func updateUserAccessPolicyToReadOnlyRole(policies portainer.UserAccessPolicies, key portainer.UserID) {
+	tmp := policies[key]
+	tmp.RoleID = 4
+	policies[key] = tmp
+}
+
+func updateTeamAccessPolicyToReadOnlyRole(policies portainer.TeamAccessPolicies, key portainer.TeamID) {
+	tmp := policies[key]
+	tmp.RoleID = 4
+	policies[key] = tmp
+}

api/bolt/registry/registry.go

@@ -1,7 +1,7 @@
 package registry
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 	var registry portainer.Registry
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &registry)
+	err := internal.GetObject(service.connection, BucketName, identifier, &registry)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 func (service *Service) Registries() ([]portainer.Registry, error) {
 	var registries = make([]portainer.Registry, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 
 // CreateRegistry creates a new registry.
 func (service *Service) CreateRegistry(registry *portainer.Registry) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateRegistry(registry *portainer.Registry) error {
 // UpdateRegistry updates an registry.
 func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, registry)
+	return internal.UpdateObject(service.connection, BucketName, identifier, registry)
 }
 
 // DeleteRegistry deletes an registry.
 func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,7 +1,7 @@
 package resourcecontrol
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 	var resourceControl portainer.ResourceControl
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &resourceControl)
+	err := internal.GetObject(service.connection, BucketName, identifier, &resourceControl)
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +48,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -82,7 +82,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -121,11 +121,11 @@ func (service *Service) CreateResourceControl(resourceControl *portainer.Resourc
 // UpdateResourceControl saves a ResourceControl object.
 func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, resourceControl)
+	return internal.UpdateObject(service.connection, BucketName, identifier, resourceControl)
 }
 
 // DeleteResourceControl deletes a ResourceControl object by ID
 func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/role/predefined_roles.go

@@ -0,0 +1,68 @@
+package role
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+// CreateOrUpdatePredefinedRoles update the predefined roles. Create one if it does not exist yet.
+func (service *Service) CreateOrUpdatePredefinedRoles() error {
+	predefinedRoles := map[portainer.RoleID]*portainer.Role{
+		portainer.RoleIDEndpointAdmin: &portainer.Role{
+			Name:           "Endpoint administrator",
+			Description:    "Full control of all resources in an endpoint",
+			ID:             portainer.RoleIDEndpointAdmin,
+			Priority:       1,
+			Authorizations: authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole(),
+		},
+		portainer.RoleIDOperator: &portainer.Role{
+			Name:           "Operator",
+			Description:    "Operational control of all existing resources in an endpoint",
+			ID:             portainer.RoleIDOperator,
+			Priority:       2,
+			Authorizations: authorization.DefaultEndpointAuthorizationsForOperatorRole(),
+		},
+		portainer.RoleIDHelpdesk: &portainer.Role{
+			Name:           "Helpdesk",
+			Description:    "Read-only access of all resources in an endpoint",
+			ID:             portainer.RoleIDHelpdesk,
+			Priority:       3,
+			Authorizations: authorization.DefaultEndpointAuthorizationsForHelpDeskRole(),
+		},
+		portainer.RoleIDStandardUser: &portainer.Role{
+			Name:           "Standard user",
+			Description:    "Full control of assigned resources in an endpoint",
+			ID:             portainer.RoleIDStandardUser,
+			Priority:       4,
+			Authorizations: authorization.DefaultEndpointAuthorizationsForStandardUserRole(),
+		},
+		portainer.RoleIDReadonly: &portainer.Role{
+			Name:           "Read-only user",
+			Description:    "Read-only access of assigned resources in an endpoint",
+			ID:             portainer.RoleIDReadonly,
+			Priority:       5,
+			Authorizations: authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(),
+		},
+	}
+
+	for roleID, predefinedRole := range predefinedRoles {
+		_, err := service.Role(roleID)
+
+		if err == errors.ErrObjectNotFound {
+			err := service.CreateRole(predefinedRole)
+			if err != nil {
+				return err
+			}
+		} else if err != nil {
+			return err
+		} else {
+			err = service.UpdateRole(predefinedRole.ID, predefinedRole)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/bolt/role/role.go

@@ -1,7 +1,7 @@
 package role
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &set)
+	err := internal.GetObject(service.connection, BucketName, identifier, &set)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 func (service *Service) Roles() ([]portainer.Role, error) {
 	var sets = make([]portainer.Role, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,11 +67,13 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 
 // CreateRole creates a new Role.
 func (service *Service) CreateRole(role *portainer.Role) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
-		role.ID = portainer.RoleID(id)
+		if role.ID == 0 {
+			role.ID = portainer.RoleID(id)
+		}
 
 		data, err := internal.MarshalObject(role)
 		if err != nil {
@@ -85,5 +87,5 @@ func (service *Service) CreateRole(role *portainer.Role) error {
 // UpdateRole updates a role.
 func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, role)
+	return internal.UpdateObject(service.connection, BucketName, identifier, role)
 }

api/bolt/s3backup/s3backup.go

@@ -0,0 +1,66 @@
+package s3backup
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	bucketName  = "s3backup"
+	statusKey   = "lastRunStatus"
+	settingsKey = "settings"
+)
+
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new service and ensures corresponding bucket exist
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, bucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// GetStatus returns the status of the last scheduled backup run
+func (s *Service) GetStatus() (portainer.S3BackupStatus, error) {
+	var status portainer.S3BackupStatus
+	err := internal.GetObject(s.connection, bucketName, []byte(statusKey), &status)
+	if err == errors.ErrObjectNotFound {
+		return status, nil
+	}
+
+	return status, err
+}
+
+// DropStatus deletes the status of the last sheduled backup run
+func (s *Service) DropStatus() error {
+	return internal.DeleteObject(s.connection, bucketName, []byte(statusKey))
+}
+
+// UpdateStatus upserts a status of the last scheduled backup run
+func (s *Service) UpdateStatus(status portainer.S3BackupStatus) error {
+	return internal.UpdateObject(s.connection, bucketName, []byte(statusKey), status)
+}
+
+// UpdateSettings updates stored s3 backup settings
+func (s *Service) UpdateSettings(settings portainer.S3BackupSettings) error {
+	return internal.UpdateObject(s.connection, bucketName, []byte(settingsKey), settings)
+}
+
+// GetSettings returns stored s3 backup settings
+func (s *Service) GetSettings() (portainer.S3BackupSettings, error) {
+	var settings portainer.S3BackupSettings
+	err := internal.GetObject(s.connection, bucketName, []byte(settingsKey), &settings)
+	if err == errors.ErrObjectNotFound {
+		return settings, nil
+	}
+
+	return settings, err
+}

api/bolt/schedule/schedule.go

@@ -1,7 +1,7 @@
 package schedule
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing schedule data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 	var schedule portainer.Schedule
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
+	err := internal.GetObject(service.connection, BucketName, identifier, &schedule)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 // UpdateSchedule updates a schedule.
 func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
+	return internal.UpdateObject(service.connection, BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
 func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -82,7 +82,7 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -105,7 +105,7 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 
 // CreateSchedule assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for schedules
@@ -125,5 +125,5 @@ func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
 
 // GetNextIdentifier returns the next identifier for a schedule.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }

api/bolt/services.go

@@ -0,0 +1,287 @@
+package bolt
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/customtemplate"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
+	"github.com/portainer/portainer/api/bolt/edgestack"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/license"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/s3backup"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
+)
+
+func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
+	customTemplateService, err := customtemplate.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.CustomTemplateService = customTemplateService
+
+	dockerhubService, err := dockerhub.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	edgeStackService, err := edgestack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackService = edgeStackService
+
+	edgeGroupService, err := edgegroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeGroupService = edgeGroupService
+
+	edgeJobService, err := edgejob.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
+	endpointgroupService, err := endpointgroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	endpointRelationService, err := endpointrelation.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	extensionService, err := extension.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
+	licenseService, err := license.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.LicenseService = licenseService
+
+	registryService, err := registry.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	s3backupService, err := s3backup.NewService(store.connection)
+	if err != nil {
+		return nil
+	}
+	store.S3BackupService = s3backupService
+
+	settingsService, err := settings.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	stackService, err := stack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	tunnelServerService, err := tunnelserver.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
+	userService, err := user.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	webhookService, err := webhook.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
+	return nil
+}
+
+// CustomTemplate gives access to the CustomTemplate data management layer
+func (store *Store) CustomTemplate() portainer.CustomTemplateService {
+	return store.CustomTemplateService
+}
+
+// DockerHub gives access to the DockerHub data management layer
+func (store *Store) DockerHub() portainer.DockerHubService {
+	return store.DockerHubService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Endpoint gives access to the Endpoint data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// License provides access to the License data management layer
+func (store *Store) License() portainer.LicenseRepository {
+	return store.LicenseService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// S3Backup gives access to S3 backup settings and status
+func (store *Store) S3Backup() portainer.S3BackupService {
+	return store.S3BackupService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
+}

api/bolt/settings/settings.go

@@ -1,10 +1,8 @@
 package settings
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings
 
-	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &settings)
+	err := internal.GetObject(service.connection, BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), settings)
+	return internal.UpdateObject(service.connection, BucketName, []byte(settingsKey), settings)
 }

api/bolt/stack/stack.go

@@ -1,7 +1,8 @@
 package stack
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +35,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 	var stack portainer.Stack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &stack)
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +47,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 	var stack *portainer.Stack
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -64,7 +65,7 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 		}
 
 		if stack == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		return nil
@@ -77,7 +78,7 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) Stacks() ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -98,12 +99,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.db, BucketName)
+	return internal.GetNextIdentifier(service.connection, BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) CreateStack(stack *portainer.Stack) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for stacks
@@ -124,11 +125,11 @@ func (service *Service) CreateStack(stack *portainer.Stack) error {
 // UpdateStack updates a stack.
 func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, stack)
+	return internal.UpdateObject(service.connection, BucketName, identifier, stack)
 }
 
 // DeleteStack deletes a stack.
 func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/tag/tag.go

@@ -1,7 +1,7 @@
 package tag
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,7 +57,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 	var tag portainer.Tag
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &tag)
+	err := internal.GetObject(service.connection, BucketName, identifier, &tag)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +67,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 
 // CreateTag creates a new tag.
 func (service *Service) CreateTag(tag *portainer.Tag) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateTag(tag *portainer.Tag) error {
 // UpdateTag updates a tag.
 func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, tag)
+	return internal.UpdateObject(service.connection, BucketName, identifier, tag)
 }
 
 // DeleteTag deletes a tag.
 func (service *Service) DeleteTag(ID portainer.TagID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/team/team.go

@@ -1,10 +1,11 @@
 package team
 
 import (
-	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-
 	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"strings"
 )
 
 const (
@@ -14,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +35,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 	var team portainer.Team
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &team)
+	err := internal.GetObject(service.connection, BucketName, identifier, &team)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +47,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	var team *portainer.Team
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,14 +58,14 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 				return err
 			}
 
-			if t.Name == name {
+			if strings.EqualFold(t.Name, name) {
 				team = &t
 				break
 			}
 		}
 
 		if team == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		return nil
@@ -77,7 +78,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 func (service *Service) Teams() ([]portainer.Team, error) {
 	var teams = make([]portainer.Team, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -99,12 +100,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 // UpdateTeam saves a Team.
 func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, team)
+	return internal.UpdateObject(service.connection, BucketName, identifier, team)
 }
 
 // CreateTeam creates a new Team.
 func (service *Service) CreateTeam(team *portainer.Team) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -122,5 +123,5 @@ func (service *Service) CreateTeam(team *portainer.Team) error {
 // DeleteTeam deletes a Team.
 func (service *Service) DeleteTeam(ID portainer.TeamID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/teammembership/teammembership.go

@@ -1,7 +1,7 @@
 package teammembership
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 	var membership portainer.TeamMembership
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &membership)
+	err := internal.GetObject(service.connection, BucketName, identifier, &membership)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -69,7 +69,7 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -95,7 +95,7 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -120,12 +120,12 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 // UpdateTeamMembership saves a TeamMembership object.
 func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, membership)
+	return internal.UpdateObject(service.connection, BucketName, identifier, membership)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) CreateTeamMembership(membership *portainer.TeamMembership) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -143,12 +143,12 @@ func (service *Service) CreateTeamMembership(membership *portainer.TeamMembershi
 // DeleteTeamMembership deletes a TeamMembership object.
 func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -173,7 +173,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()

api/bolt/tunnelserver/tunnelserver.go

@@ -1,10 +1,8 @@
 package tunnelserver
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
 )
 
 const (
@@ -15,18 +13,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +32,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 	var info portainer.TunnelServerInfo
 
-	err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
+	err := internal.GetObject(service.connection, BucketName, []byte(infoKey), &info)
 	if err != nil {
 		return nil, err
 	}
@@ -44,5 +42,5 @@ func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 
 // UpdateInfo persists a TunnelServerInfo object.
 func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
-	return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
+	return internal.UpdateObject(service.connection, BucketName, []byte(infoKey), settings)
 }

api/bolt/user/user.go

@@ -1,10 +1,11 @@
 package user
 
 import (
-	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-
 	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"strings"
 )
 
 const (
@@ -14,18 +15,18 @@ const (
 
 // Service represents a service for managing endpoint data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -34,7 +35,7 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 	var user portainer.User
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &user)
+	err := internal.GetObject(service.connection, BucketName, identifier, &user)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +47,9 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 func (service *Service) UserByUsername(username string) (*portainer.User, error) {
 	var user *portainer.User
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	username = strings.ToLower(username)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -57,14 +60,14 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 				return err
 			}
 
-			if u.Username == username {
+			if strings.EqualFold(u.Username, username) {
 				user = &u
 				break
 			}
 		}
 
 		if user == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 		return nil
 	})
@@ -76,7 +79,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 func (service *Service) Users() ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -98,7 +101,7 @@ func (service *Service) Users() ([]portainer.User, error) {
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -122,16 +125,18 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 // UpdateUser saves a user.
 func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, user)
+	user.Username = strings.ToLower(user.Username)
+	return internal.UpdateObject(service.connection, BucketName, identifier, user)
 }
 
 // CreateUser creates a new user.
 func (service *Service) CreateUser(user *portainer.User) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
 		user.ID = portainer.UserID(id)
+		user.Username = strings.ToLower(user.Username)
 
 		data, err := internal.MarshalObject(user)
 		if err != nil {
@@ -145,5 +150,5 @@ func (service *Service) CreateUser(user *portainer.User) error {
 // DeleteUser deletes a user.
 func (service *Service) DeleteUser(ID portainer.UserID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/version/version.go

@@ -4,43 +4,111 @@ import (
 	"strconv"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
 	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "version"
-	versionKey = "DB_VERSION"
+	BucketName         = "version"
+	versionKey         = "DB_VERSION"
+	previousVersionKey = "PREVIOUS_DB_VERSION"
+	instanceKey        = "INSTANCE_ID"
+	editionKey         = "EDITION"
 )
 
 // Service represents a service to manage stored versions.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
+// Edition retrieves the stored portainer edition.
+func (service *Service) Edition() (portainer.SoftwareEdition, error) {
+	editionData, err := service.getKey(editionKey)
+	if err != nil {
+		return 0, err
+	}
+
+	edition, err := strconv.Atoi(string(editionData))
+	if err != nil {
+		return 0, err
+	}
+
+	return portainer.SoftwareEdition(edition), nil
+}
+
+// StoreEdition store the portainer edition.
+func (service *Service) StoreEdition(edition portainer.SoftwareEdition) error {
+	return service.setKey(editionKey, strconv.Itoa(int(edition)))
+}
+
+// PreviousDBVersion retrieves the stored database version.
+func (service *Service) PreviousDBVersion() (int, error) {
+	version, err := service.getKey(previousVersionKey)
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.Atoi(string(version))
+}
+
 // DBVersion retrieves the stored database version.
 func (service *Service) DBVersion() (int, error) {
+	version, err := service.getKey(versionKey)
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.Atoi(string(version))
+}
+
+// StorePreviousDBVersion store the database version.
+func (service *Service) StorePreviousDBVersion(version int) error {
+	return service.setKey(previousVersionKey, strconv.Itoa(version))
+}
+
+// StoreDBVersion store the database version.
+func (service *Service) StoreDBVersion(version int) error {
+	return service.setKey(versionKey, strconv.Itoa(version))
+}
+
+// InstanceID retrieves the stored instance ID.
+func (service *Service) InstanceID() (string, error) {
+	instanceID, err := service.getKey(instanceKey)
+	if err != nil {
+		return "", err
+	}
+
+	return string(instanceID), nil
+}
+
+// StoreInstanceID store the instance ID.
+func (service *Service) StoreInstanceID(ID string) error {
+	return service.setKey(instanceKey, ID)
+}
+
+func (service *Service) getKey(key string) ([]byte, error) {
 	var data []byte
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		value := bucket.Get([]byte(versionKey))
+		value := bucket.Get([]byte(key))
 		if value == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		data = make([]byte, len(value))
@@ -48,19 +116,19 @@ func (service *Service) DBVersion() (int, error) {
 
 		return nil
 	})
+
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 
-	return strconv.Atoi(string(data))
+	return data, nil
 }
 
-// StoreDBVersion store the database version.
-func (service *Service) StoreDBVersion(version int) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+func (service *Service) setKey(key string, value string) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		data := []byte(strconv.Itoa(version))
-		return bucket.Put([]byte(versionKey), data)
+		data := []byte(value)
+		return bucket.Put([]byte(key), data)
 	})
 }

api/bolt/webhook/webhook.go

@@ -1,7 +1,8 @@
 package webhook
 
 import (
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +15,18 @@ const (
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	db *bolt.DB
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		db: db,
+		connection: connection,
 	}, nil
 }
 
@@ -33,7 +34,7 @@ func NewService(db *bolt.DB) (*Service, error) {
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,7 +58,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 	var webhook portainer.Webhook
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
+	err := internal.GetObject(service.connection, BucketName, identifier, &webhook)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +70,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -87,7 +88,7 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 		}
 
 		if webhook == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		return nil
@@ -100,7 +101,7 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.db.View(func(tx *bolt.Tx) error {
+	err := service.connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -118,7 +119,7 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 		}
 
 		if webhook == nil {
-			return portainer.ErrObjectNotFound
+			return errors.ErrObjectNotFound
 		}
 
 		return nil
@@ -130,12 +131,12 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 // DeleteWebhook deletes a webhook.
 func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
 
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/chisel/schedules.go

@@ -6,42 +6,42 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-// AddSchedule register a schedule inside the tunnel details associated to an endpoint.
-func (service *Service) AddSchedule(endpointID portainer.EndpointID, schedule *portainer.EdgeSchedule) {
+// AddEdgeJob register an EdgeJob inside the tunnel details associated to an endpoint.
+func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *portainer.EdgeJob) {
 	tunnel := service.GetTunnelDetails(endpointID)
 
-	existingScheduleIndex := -1
-	for idx, existingSchedule := range tunnel.Schedules {
-		if existingSchedule.ID == schedule.ID {
-			existingScheduleIndex = idx
+	existingJobIndex := -1
+	for idx, existingJob := range tunnel.Jobs {
+		if existingJob.ID == edgeJob.ID {
+			existingJobIndex = idx
 			break
 		}
 	}
 
-	if existingScheduleIndex == -1 {
-		tunnel.Schedules = append(tunnel.Schedules, *schedule)
+	if existingJobIndex == -1 {
+		tunnel.Jobs = append(tunnel.Jobs, *edgeJob)
 	} else {
-		tunnel.Schedules[existingScheduleIndex] = *schedule
+		tunnel.Jobs[existingJobIndex] = *edgeJob
 	}
 
 	key := strconv.Itoa(int(endpointID))
 	service.tunnelDetailsMap.Set(key, tunnel)
 }
 
-// RemoveSchedule will remove the specified schedule from each tunnel it was registered with.
-func (service *Service) RemoveSchedule(scheduleID portainer.ScheduleID) {
+// RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
+func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
 	for item := range service.tunnelDetailsMap.IterBuffered() {
 		tunnelDetails := item.Val.(*portainer.TunnelDetails)
 
-		updatedSchedules := make([]portainer.EdgeSchedule, 0)
-		for _, schedule := range tunnelDetails.Schedules {
-			if schedule.ID == scheduleID {
+		updatedJobs := make([]portainer.EdgeJob, 0)
+		for _, edgeJob := range tunnelDetails.Jobs {
+			if edgeJob.ID == edgeJobID {
 				continue
 			}
-			updatedSchedules = append(updatedSchedules, schedule)
+			updatedJobs = append(updatedJobs, edgeJob)
 		}
 
-		tunnelDetails.Schedules = updatedSchedules
+		tunnelDetails.Jobs = updatedJobs
 		service.tunnelDetailsMap.Set(item.Key, tunnelDetails)
 	}
 }

api/chisel/service.go

@@ -1,17 +1,17 @@
 package chisel
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"strconv"
 	"time"
 
 	"github.com/dchest/uniuri"
-
-	cmap "github.com/orcaman/concurrent-map"
-
 	chserver "github.com/jpillora/chisel/server"
+	cmap "github.com/orcaman/concurrent-map"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 )
 
 const (
@@ -28,15 +28,17 @@ type Service struct {
 	serverPort        string
 	tunnelDetailsMap  cmap.ConcurrentMap
 	dataStore         portainer.DataStore
-	snapshotter       portainer.Snapshotter
+	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
+	shutdownCtx       context.Context
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore portainer.DataStore) *Service {
+func NewService(dataStore portainer.DataStore, shutdownCtx context.Context) *Service {
 	return &Service{
 		tunnelDetailsMap: cmap.New(),
 		dataStore:        dataStore,
+		shutdownCtx:      shutdownCtx,
 	}
 }
 
@@ -45,7 +47,7 @@ func NewService(dataStore portainer.DataStore) *Service {
 // be found inside the database, it will generate a new one randomly and persist it.
 // It starts the tunnel status verification process in the background.
 // The snapshotter is used in the tunnel status verification process.
-func (service *Service) StartTunnelServer(addr, port string, snapshotter portainer.Snapshotter) error {
+func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
 	keySeed, err := service.retrievePrivateKeySeed()
 	if err != nil {
 		return err
@@ -78,17 +80,22 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotter portain
 		return err
 	}
 
-	service.snapshotter = snapshotter
+	service.snapshotService = snapshotService
 	go service.startTunnelVerificationLoop()
 
 	return nil
 }
 
+// StopTunnelServer stops tunnel http server
+func (service *Service) StopTunnelServer() error {
+	return service.chiselServer.Close()
+}
+
 func (service *Service) retrievePrivateKeySeed() (string, error) {
 	var serverInfo *portainer.TunnelServerInfo
 
 	serverInfo, err := service.dataStore.TunnelServer().Info()
-	if err == portainer.ErrObjectNotFound {
+	if err == errors.ErrObjectNotFound {
 		keySeed := uniuri.NewLen(16)
 
 		serverInfo = &portainer.TunnelServerInfo{
@@ -109,13 +116,16 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 func (service *Service) startTunnelVerificationLoop() {
 	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
 	ticker := time.NewTicker(tunnelCleanupInterval)
-	stopSignal := make(chan struct{})
 
 	for {
 		select {
 		case <-ticker.C:
 			service.checkTunnels()
-		case <-stopSignal:
+		case <-service.shutdownCtx.Done():
+			log.Println("[DEBUG] Shutting down tunnel service")
+			if err := service.StopTunnelServer(); err != nil {
+				log.Printf("Stopped tunnel service: %s", err)
+			}
 			ticker.Stop()
 			return
 		}
@@ -155,7 +165,7 @@ func (service *Service) checkTunnels() {
 			}
 		}
 
-		if len(tunnel.Schedules) > 0 {
+		if len(tunnel.Jobs) > 0 {
 			endpointID, err := strconv.Atoi(item.Key)
 			if err != nil {
 				log.Printf("[ERROR] [chisel,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
@@ -177,13 +187,13 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 	}
 
 	endpointURL := endpoint.URL
+
 	endpoint.URL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnelPort)
-	snapshot, err := service.snapshotter.CreateSnapshot(endpoint)
+	err = service.snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
 		return err
 	}
 
-	endpoint.Snapshots = []portainer.Snapshot{*snapshot}
 	endpoint.URL = endpointURL
 	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 }

api/chisel/tunnel.go

@@ -47,11 +47,11 @@ func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *porta
 		return tunnelDetails
 	}
 
-	schedules := make([]portainer.EdgeSchedule, 0)
+	jobs := make([]portainer.EdgeJob, 0)
 	return &portainer.TunnelDetails{
 		Status:      portainer.EdgeAgentIdle,
 		Port:        0,
-		Schedules:   schedules,
+		Jobs:        jobs,
 		Credentials: "",
 	}
 }

api/cli/cli.go

@@ -1,10 +1,11 @@
 package cli
 
 import (
+	"errors"
 	"log"
 	"time"
 
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -16,12 +17,11 @@ import (
 // Service implements the CLIService interface
 type Service struct{}
 
-const (
-	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
-	errSocketOrNamedPipeNotFound     = portainer.Error("Unable to locate Unix socket or named pipe")
-	errInvalidSnapshotInterval       = portainer.Error("Invalid snapshot interval")
-	errNoAuthExcludeAdminPassword    = portainer.Error("Cannot use --no-auth with --admin-password or --admin-password-file")
-	errAdminPassExcludeAdminPassFile = portainer.Error("Cannot use --admin-password with --admin-password-file")
+var (
+	errInvalidEndpointProtocol       = errors.New("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errSocketOrNamedPipeNotFound     = errors.New("Unable to locate Unix socket or named pipe")
+	errInvalidSnapshotInterval       = errors.New("Invalid snapshot interval")
+	errAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
 )
 
 // ParseFlags parse the CLI flags and return a portainer.Flags struct
@@ -29,28 +29,29 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 	kingpin.Version(version)
 
 	flags := &portainer.CLIFlags{
-		Addr:              kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
-		TunnelAddr:        kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
-		TunnelPort:        kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
-		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
-		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
-		NoAuth:            kingpin.Flag("no-auth", "Disable authentication (deprecated)").Default(defaultNoAuth).Bool(),
-		NoAnalytics:       kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAnalytics).Bool(),
-		TLS:               kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
-		TLSSkipVerify:     kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
-		TLSCacert:         kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
-		TLSCert:           kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
-		TLSKey:            kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
-		SSL:               kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
-		SSLCert:           kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
-		SSLKey:            kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
-		SnapshotInterval:  kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
-		AdminPassword:     kingpin.Flag("admin-password", "Hashed admin password").String(),
-		AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
-		Labels:            pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
-		Logo:              kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
-		Templates:         kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
+		Addr:                      kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
+		TunnelAddr:                kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
+		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
+		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
+		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
+		EndpointURL:               kingpin.Flag("host", "Endpoint URL").Short('H').String(),
+		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
+		NoAnalytics:               kingpin.Flag("no-analytics", "Disable Analytics in app (deprecated)").Bool(),
+		TLS:                       kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
+		TLSSkipVerify:             kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
+		TLSCacert:                 kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
+		TLSCert:                   kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
+		TLSKey:                    kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
+		RollbackToCE:              kingpin.Flag("rollback-to-ce", "Rollback the database store to CE").Bool(),
+		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
+		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
+		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
+		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
+		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
+		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
+		Labels:                    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
+		Logo:                      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
+		Templates:                 kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
 	}
 
 	kingpin.Parse()
@@ -81,10 +82,6 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return err
 	}
 
-	if *flags.NoAuth && (*flags.AdminPassword != "" || *flags.AdminPasswordFile != "") {
-		return errNoAuthExcludeAdminPassword
-	}
-
 	if *flags.AdminPassword != "" && *flags.AdminPasswordFile != "" {
 		return errAdminPassExcludeAdminPassFile
 	}
@@ -93,8 +90,8 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 }
 
 func displayDeprecationWarnings(flags *portainer.CLIFlags) {
-	if *flags.NoAuth {
-		log.Println("Warning: the --no-auth flag is deprecated and will likely be removed in a future version of Portainer.")
+	if *flags.NoAnalytics {
+		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
 	}
 }
 

api/cli/confirm.go

@@ -0,0 +1,24 @@
+package cli
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"strings"
+)
+
+// Confirm starts a rollback db cli application
+func Confirm(message string) (bool, error) {
+	log.Printf("%s [y/N]", message)
+
+	reader := bufio.NewReader(os.Stdin)
+	answer, err := reader.ReadString('\n')
+	if err != nil {
+		return false, err
+	}
+	answer = strings.Replace(answer, "\n", "", -1)
+	answer = strings.ToLower(answer)
+
+	return answer == "y" || answer == "yes", nil
+
+}

api/cli/defaults.go

@@ -8,8 +8,6 @@ const (
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "/data"
 	defaultAssetsDirectory     = "./"
-	defaultNoAuth              = "false"
-	defaultNoAnalytics         = "false"
 	defaultTLS                 = "false"
 	defaultTLSSkipVerify       = "false"
 	defaultTLSCACertPath       = "/certs/ca.pem"

api/cli/defaults_windows.go

@@ -6,8 +6,6 @@ const (
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "C:\\data"
 	defaultAssetsDirectory     = "./"
-	defaultNoAuth              = "false"
-	defaultNoAnalytics         = "false"
 	defaultTLS                 = "false"
 	defaultTLSSkipVerify       = "false"
 	defaultTLSCACertPath       = "C:\\certs\\ca.pem"

api/cmd/portainer/main.go

@@ -1,91 +1,136 @@
 package main
 
 import (
+	"context"
 	"log"
 	"os"
 	"strings"
-	"time"
 
-	"github.com/portainer/portainer/api/chisel"
-
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/chisel"
 	"github.com/portainer/portainer/api/cli"
-	"github.com/portainer/portainer/api/cron"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/docker"
+
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
 	"github.com/portainer/portainer/api/http"
 	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/http/proxy"
+	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/kubernetes"
+	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/libcompose"
+	"github.com/portainer/portainer/api/license"
+	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/useractivity"
 )
 
 func initCLI() *portainer.CLIFlags {
 	var cliService portainer.CLIService = &cli.Service{}
 	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed parsing flags: %s", err)
 	}
 
 	err = cliService.ValidateFlags(flags)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed validating flags:%s", err)
 	}
 	return flags
 }
 
+func initUserActivityStore(dataStorePath string) portainer.UserActivityStore {
+	store, err := useractivity.NewUserActivityStore(dataStorePath)
+	if err != nil {
+		log.Fatalf("Failed initalizing user activity store: %s", err)
+	}
+
+	return store
+}
+
 func initFileService(dataStorePath string) portainer.FileService {
 	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed creating file service: %s", err)
 	}
 	return fileService
 }
 
-func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
+func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService) portainer.DataStore {
 	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed creating data store: %s", err)
 	}
 
 	err = store.Open()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed opening store: %s", err)
 	}
 
 	err = store.Init()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed initializing data store: %s", err)
 	}
 
-	err = store.MigrateData()
+	if rollback {
+		err := store.RollbackToCE()
+		if err != nil {
+			log.Fatalf("failed rolling back to CE: %s", err)
+		}
+
+		log.Println("Exiting rollback")
+		os.Exit(0)
+		return nil
+	}
+
+	err = store.MigrateData(false)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed migration: %s", err)
 	}
 	return store
 }
 
-func initComposeStackManager(dataStorePath string, reverseTunnelService portainer.ReverseTunnelService) portainer.ComposeStackManager {
-	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
+func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+	composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
+	if err != nil {
+		log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err)
+		return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
+	}
+
+	return composeWrapper
 }
 
 func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
 	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initJWTService(authenticationEnabled bool) portainer.JWTService {
-	if authenticationEnabled {
-		jwtService, err := jwt.NewService()
-		if err != nil {
-			log.Fatal(err)
-		}
-		return jwtService
+func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath)
+}
+
+func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
+	settings, err := dataStore.Settings().Settings()
+	if err != nil {
+		return nil, err
 	}
-	return nil
+
+	userSessionTimeout := settings.UserSessionTimeout
+	if userSessionTimeout == "" {
+		userSessionTimeout = portainer.DefaultUserSessionTimeout
+	}
+	jwtService, err := jwt.NewService(userSessionTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return jwtService, nil
 }
 
 func initDigitalSignatureService() portainer.DigitalSignatureService {
@@ -100,97 +145,37 @@ func initLDAPService() portainer.LDAPService {
 	return &ldap.Service{}
 }
 
+func initOAuthService() portainer.OAuthService {
+	return oauth.NewService()
+}
+
 func initGitService() portainer.GitService {
 	return git.NewService()
 }
 
-func initClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
+func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initSnapshotter(clientFactory *docker.ClientFactory) portainer.Snapshotter {
-	return docker.NewSnapshotter(clientFactory)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
 }
 
-func initJobScheduler() portainer.JobScheduler {
-	return cron.NewJobScheduler()
-}
+func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
+	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter portainer.Snapshotter, dataStore portainer.DataStore) error {
-	settings, err := dataStore.Settings().Settings()
+	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	schedules, err := dataStore.Schedule().SchedulesByJobType(portainer.SnapshotJobType)
-	if err != nil {
-		return err
-	}
-
-	var snapshotSchedule *portainer.Schedule
-	if len(schedules) == 0 {
-		snapshotJob := &portainer.SnapshotJob{}
-		snapshotSchedule = &portainer.Schedule{
-			ID:             portainer.ScheduleID(dataStore.Schedule().GetNextIdentifier()),
-			Name:           "system_snapshot",
-			CronExpression: "@every " + settings.SnapshotInterval,
-			Recurring:      true,
-			JobType:        portainer.SnapshotJobType,
-			SnapshotJob:    snapshotJob,
-			Created:        time.Now().Unix(),
-		}
-	} else {
-		snapshotSchedule = &schedules[0]
-	}
-
-	snapshotJobContext := cron.NewSnapshotJobContext(dataStore, snapshotter)
-	snapshotJobRunner := cron.NewSnapshotJobRunner(snapshotSchedule, snapshotJobContext)
-
-	err = jobScheduler.ScheduleJob(snapshotJobRunner)
-	if err != nil {
-		return err
-	}
-
-	if len(schedules) == 0 {
-		return dataStore.Schedule().CreateSchedule(snapshotSchedule)
-	}
-	return nil
-}
-
-func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, dataStore portainer.DataStore, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) error {
-	schedules, err := dataStore.Schedule().Schedules()
-	if err != nil {
-		return err
-	}
-
-	for _, schedule := range schedules {
-
-		if schedule.JobType == portainer.ScriptExecutionJobType {
-			jobContext := cron.NewScriptExecutionJobContext(jobService, dataStore, fileService)
-			jobRunner := cron.NewScriptExecutionJobRunner(&schedule, jobContext)
-
-			err = jobScheduler.ScheduleJob(jobRunner)
-			if err != nil {
-				return err
-			}
-		}
-
-		if schedule.EdgeSchedule != nil {
-			for _, endpointID := range schedule.EdgeSchedule.Endpoints {
-				reverseTunnelService.AddSchedule(endpointID, schedule.EdgeSchedule)
-			}
-		}
-
-	}
-
-	return nil
+	return snapshotService, nil
 }
 
 func initStatus(flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
-		Analytics:      !*flags.NoAnalytics,
-		Authentication: !*flags.NoAuth,
-		Version:        portainer.APIVersion,
+		Version: portainer.APIVersion,
 	}
 }
 
@@ -202,6 +187,10 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 
 	settings.LogoURL = *flags.Logo
 	settings.SnapshotInterval = *flags.SnapshotInterval
+	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
+	settings.EnableTelemetry = true
+	settings.OAuthSettings.SSO = true
+	settings.OAuthSettings.HideInternalAuth = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -234,7 +223,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
 func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	existingKeyPair, err := fileService.KeyPairFilesExist()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed checking for existing key pair: %s", err)
 	}
 
 	if existingKeyPair {
@@ -243,7 +232,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotService portainer.SnapshotService) error {
 	tlsConfiguration := portainer.TLSConfiguration{
 		TLS:           *flags.TLS,
 		TLSSkipVerify: *flags.TLSSkipVerify,
@@ -270,7 +259,21 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 		Extensions:         []portainer.EndpointExtension{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.Snapshot{},
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
 	}
 
 	if strings.HasPrefix(endpoint.URL, "tcp://") {
@@ -289,10 +292,15 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 		}
 	}
 
-	return snapshotAndPersistEndpoint(endpoint, dataStore, snapshotter)
+	err := snapshotService.SnapshotEndpoint(endpoint)
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+	}
+
+	return dataStore.Endpoint().CreateEndpoint(endpoint)
 }
 
-func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
+func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore, snapshotService portainer.SnapshotService) error {
 	if strings.HasPrefix(endpointURL, "tcp://") {
 		_, err := client.ExecutePingOperation(endpointURL, nil)
 		if err != nil {
@@ -313,27 +321,32 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 		Extensions:         []portainer.EndpointExtension{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.Snapshot{},
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
 	}
 
-	return snapshotAndPersistEndpoint(endpoint, dataStore, snapshotter)
-}
-
-func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
-	snapshot, err := snapshotter.CreateSnapshot(endpoint)
-	endpoint.Status = portainer.EndpointStatusUp
+	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
 		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 	}
 
-	if snapshot != nil {
-		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
-	}
-
 	return dataStore.Endpoint().CreateEndpoint(endpoint)
 }
 
-func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
+func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotService portainer.SnapshotService) error {
 	if *flags.EndpointURL == "" {
 		return nil
 	}
@@ -349,121 +362,104 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	}
 
 	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, dataStore, snapshotter)
+		return createTLSSecuredEndpoint(flags, dataStore, snapshotService)
 	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotter)
+	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
-func initJobService(dockerClientFactory *docker.ClientFactory) portainer.JobService {
-	return docker.NewJobService(dockerClientFactory)
-}
-
-func initExtensionManager(fileService portainer.FileService, dataStore portainer.DataStore) (portainer.ExtensionManager, error) {
-	extensionManager := exec.NewExtensionManager(fileService, dataStore)
-
-	err := extensionManager.StartExtensions()
-	if err != nil {
-		return nil, err
-	}
-
-	return extensionManager, nil
-}
-
-func terminateIfNoAdminCreated(dataStore portainer.DataStore) {
-	timer1 := time.NewTimer(5 * time.Minute)
-	<-timer1.C
-
-	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if len(users) == 0 {
-		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
-		return
-	}
-}
-
-func main() {
-	flags := initCLI()
+func buildServer(flags *portainer.CLIFlags) portainer.Server {
+	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, fileService)
-	defer dataStore.Close()
+	dataStore := initDataStore(*flags.Data, *flags.RollbackToCE, fileService)
 
-	jwtService := initJWTService(!*flags.NoAuth)
+	jwtService, err := initJWTService(dataStore)
+	if err != nil {
+		log.Fatalf("failed initializing JWT service: %s", err)
+	}
+
+	licenseService := license.NewService(dataStore.License(), shutdownCtx)
+	if err = licenseService.Init(); err != nil {
+		log.Fatalf("failed initializing license service: %s", err)
+	}
 
 	ldapService := initLDAPService()
 
+	oauthService := initOAuthService()
+
 	gitService := initGitService()
 
 	cryptoService := initCryptoService()
 
 	digitalSignatureService := initDigitalSignatureService()
 
-	err := initKeyPair(fileService, digitalSignatureService)
+	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed initializing key pair: %s", err)
 	}
 
-	extensionManager, err := initExtensionManager(fileService, dataStore)
+	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+
+	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed to get datastore version: %s", err)
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore)
+	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
+	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
 
-	clientFactory := initClientFactory(digitalSignatureService, reverseTunnelService)
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
+	if err != nil {
+		log.Fatalf("failed initializing snapshot service: %s", err)
+	}
+	snapshotService.Start()
 
-	jobService := initJobService(clientFactory)
-
-	snapshotter := initSnapshotter(clientFactory)
+	authorizationService := authorization.NewService(dataStore)
+	authorizationService.K8sClientFactory = kubernetesClientFactory
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed initializing swarm stack manager: %s", err)
 	}
+	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
-	composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)
+	userActivityStore := initUserActivityStore(*flags.Data)
+
+	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, authorizationService, userActivityStore)
+
+	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
+
+	kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("failed updating settings from flags: %s", err)
 		}
 	}
 
-	jobScheduler := initJobScheduler()
-
-	err = loadSchedulesFromDatabase(jobScheduler, jobService, dataStore, fileService, reverseTunnelService)
+	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed loading edge jobs from database: %s", err)
 	}
 
-	err = loadSnapshotSystemSchedule(jobScheduler, snapshotter, dataStore)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	jobScheduler.Start()
-
 	applicationStatus := initStatus(flags)
 
-	err = initEndpoint(flags, dataStore, snapshotter)
+	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed initializing endpoint: %s", err)
 	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
 		content, err := fileService.GetFileContent(*flags.AdminPasswordFile)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("failed getting admin password file: %s", err)
 		}
 		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("failed hashing admin password: %s", err)
 		}
 	} else if *flags.AdminPassword != "" {
 		adminPasswordHash = *flags.AdminPassword
@@ -472,7 +468,7 @@ func main() {
 	if adminPasswordHash != "" {
 		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("failed getting admin user: %s", err)
 		}
 
 		if len(users) == 0 {
@@ -481,54 +477,66 @@ func main() {
 				Username:                "admin",
 				Role:                    portainer.AdministratorRole,
 				Password:                adminPasswordHash,
-				PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
+				PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
 			}
 			err := dataStore.User().CreateUser(user)
 			if err != nil {
-				log.Fatal(err)
+				log.Fatalf("failed creating admin user: %s", err)
 			}
 		} else {
 			log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
 		}
 	}
 
-	if !*flags.NoAuth {
-		go terminateIfNoAdminCreated(dataStore)
-	}
-
-	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotter)
+	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed starting tunnel server: %s", err)
 	}
 
-	var server portainer.Server = &http.Server{
-		ReverseTunnelService: reverseTunnelService,
-		Status:               applicationStatus,
-		BindAddress:          *flags.Addr,
-		AssetsPath:           *flags.Assets,
-		AuthDisabled:         *flags.NoAuth,
-		DataStore:            dataStore,
-		SwarmStackManager:    swarmStackManager,
-		ComposeStackManager:  composeStackManager,
-		ExtensionManager:     extensionManager,
-		CryptoService:        cryptoService,
-		JWTService:           jwtService,
-		FileService:          fileService,
-		LDAPService:          ldapService,
-		GitService:           gitService,
-		SignatureService:     digitalSignatureService,
-		JobScheduler:         jobScheduler,
-		Snapshotter:          snapshotter,
-		SSL:                  *flags.SSL,
-		SSLCert:              *flags.SSLCert,
-		SSLKey:               *flags.SSLKey,
-		DockerClientFactory:  clientFactory,
-		JobService:           jobService,
-	}
-
-	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)
-	err = server.Start()
+	err = licenseService.Start()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("failed starting license service: %s", err)
+	}
+
+	return &http.Server{
+		AuthorizationService:        authorizationService,
+		ReverseTunnelService:        reverseTunnelService,
+		Status:                      applicationStatus,
+		BindAddress:                 *flags.Addr,
+		AssetsPath:                  *flags.Assets,
+		DataStore:                   dataStore,
+		LicenseService:              licenseService,
+		SwarmStackManager:           swarmStackManager,
+		ComposeStackManager:         composeStackManager,
+		KubernetesDeployer:          kubernetesDeployer,
+		CryptoService:               cryptoService,
+		JWTService:                  jwtService,
+		FileService:                 fileService,
+		LDAPService:                 ldapService,
+		OAuthService:                oauthService,
+		GitService:                  gitService,
+		ProxyManager:                proxyManager,
+		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
+		SignatureService:            digitalSignatureService,
+		SnapshotService:             snapshotService,
+		SSL:                         *flags.SSL,
+		SSLCert:                     *flags.SSLCert,
+		SSLKey:                      *flags.SSLKey,
+		DockerClientFactory:         dockerClientFactory,
+		UserActivityStore:           userActivityStore,
+		KubernetesClientFactory:     kubernetesClientFactory,
+		ShutdownCtx:                 shutdownCtx,
+		ShutdownTrigger:             shutdownTrigger,
+	}
+}
+
+func main() {
+	flags := initCLI()
+
+	for {
+		server := buildServer(flags)
+		log.Printf("Starting Portainer %s on %s\n", portainer.APIVersion, *flags.Addr)
+		err := server.Start()
+		log.Printf("Http server exited: %s\n", err)
 	}
 }

api/cron/job_script_execution.go

@@ -1,96 +0,0 @@
-package cron
-
-import (
-	"log"
-	"time"
-
-	"github.com/portainer/portainer/api"
-)
-
-// ScriptExecutionJobRunner is used to run a ScriptExecutionJob
-type ScriptExecutionJobRunner struct {
-	schedule     *portainer.Schedule
-	context      *ScriptExecutionJobContext
-	executedOnce bool
-}
-
-// ScriptExecutionJobContext represents the context of execution of a ScriptExecutionJob
-type ScriptExecutionJobContext struct {
-	dataStore   portainer.DataStore
-	jobService  portainer.JobService
-	fileService portainer.FileService
-}
-
-// NewScriptExecutionJobContext returns a new context that can be used to execute a ScriptExecutionJob
-func NewScriptExecutionJobContext(jobService portainer.JobService, dataStore portainer.DataStore, fileService portainer.FileService) *ScriptExecutionJobContext {
-	return &ScriptExecutionJobContext{
-		jobService:  jobService,
-		dataStore:   dataStore,
-		fileService: fileService,
-	}
-}
-
-// NewScriptExecutionJobRunner returns a new runner that can be scheduled
-func NewScriptExecutionJobRunner(schedule *portainer.Schedule, context *ScriptExecutionJobContext) *ScriptExecutionJobRunner {
-	return &ScriptExecutionJobRunner{
-		schedule:     schedule,
-		context:      context,
-		executedOnce: false,
-	}
-}
-
-// Run triggers the execution of the job.
-// It will iterate through all the endpoints specified in the context to
-// execute the script associated to the job.
-func (runner *ScriptExecutionJobRunner) Run() {
-	if !runner.schedule.Recurring && runner.executedOnce {
-		return
-	}
-	runner.executedOnce = true
-
-	scriptFile, err := runner.context.fileService.GetFileContent(runner.schedule.ScriptExecutionJob.ScriptPath)
-	if err != nil {
-		log.Printf("scheduled job error (script execution). Unable to retrieve script file (err=%s)\n", err)
-		return
-	}
-
-	targets := make([]*portainer.Endpoint, 0)
-	for _, endpointID := range runner.schedule.ScriptExecutionJob.Endpoints {
-		endpoint, err := runner.context.dataStore.Endpoint().Endpoint(endpointID)
-		if err != nil {
-			log.Printf("scheduled job error (script execution). Unable to retrieve information about endpoint (id=%d) (err=%s)\n", endpointID, err)
-			return
-		}
-
-		targets = append(targets, endpoint)
-	}
-
-	runner.executeAndRetry(targets, scriptFile, 0)
-}
-
-func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) {
-	retryTargets := make([]*portainer.Endpoint, 0)
-
-	for _, endpoint := range endpoints {
-		err := runner.context.jobService.ExecuteScript(endpoint, "", runner.schedule.ScriptExecutionJob.Image, script, runner.schedule)
-		if err == portainer.ErrUnableToPingEndpoint {
-			retryTargets = append(retryTargets, endpoint)
-		} else if err != nil {
-			log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err)
-		}
-	}
-
-	retryCount++
-	if retryCount >= runner.schedule.ScriptExecutionJob.RetryCount {
-		return
-	}
-
-	time.Sleep(time.Duration(runner.schedule.ScriptExecutionJob.RetryInterval) * time.Second)
-
-	runner.executeAndRetry(retryTargets, script, retryCount)
-}
-
-// GetSchedule returns the schedule associated to the runner
-func (runner *ScriptExecutionJobRunner) GetSchedule() *portainer.Schedule {
-	return runner.schedule
-}

api/cron/job_snapshot.go

@@ -1,85 +0,0 @@
-package cron
-
-import (
-	"log"
-
-	"github.com/portainer/portainer/api"
-)
-
-// SnapshotJobRunner is used to run a SnapshotJob
-type SnapshotJobRunner struct {
-	schedule *portainer.Schedule
-	context  *SnapshotJobContext
-}
-
-// SnapshotJobContext represents the context of execution of a SnapshotJob
-type SnapshotJobContext struct {
-	dataStore   portainer.DataStore
-	snapshotter portainer.Snapshotter
-}
-
-// NewSnapshotJobContext returns a new context that can be used to execute a SnapshotJob
-func NewSnapshotJobContext(dataStore portainer.DataStore, snapshotter portainer.Snapshotter) *SnapshotJobContext {
-	return &SnapshotJobContext{
-		dataStore:   dataStore,
-		snapshotter: snapshotter,
-	}
-}
-
-// NewSnapshotJobRunner returns a new runner that can be scheduled
-func NewSnapshotJobRunner(schedule *portainer.Schedule, context *SnapshotJobContext) *SnapshotJobRunner {
-	return &SnapshotJobRunner{
-		schedule: schedule,
-		context:  context,
-	}
-}
-
-// GetSchedule returns the schedule associated to the runner
-func (runner *SnapshotJobRunner) GetSchedule() *portainer.Schedule {
-	return runner.schedule
-}
-
-// Run triggers the execution of the schedule.
-// It will iterate through all the endpoints available in the database to
-// create a snapshot of each one of them.
-// As a snapshot can be a long process, to avoid any concurrency issue we
-// retrieve the latest version of the endpoint right after a snapshot.
-func (runner *SnapshotJobRunner) Run() {
-	go func() {
-		endpoints, err := runner.context.dataStore.Endpoint().Endpoints()
-		if err != nil {
-			log.Printf("background schedule error (endpoint snapshot). Unable to retrieve endpoint list (err=%s)\n", err)
-			return
-		}
-
-		for _, endpoint := range endpoints {
-			if endpoint.Type == portainer.EdgeAgentEnvironment {
-				continue
-			}
-
-			snapshot, snapshotError := runner.context.snapshotter.CreateSnapshot(&endpoint)
-
-			latestEndpointReference, err := runner.context.dataStore.Endpoint().Endpoint(endpoint.ID)
-			if latestEndpointReference == nil {
-				log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
-				continue
-			}
-
-			latestEndpointReference.Status = portainer.EndpointStatusUp
-			if snapshotError != nil {
-				log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, snapshotError)
-				latestEndpointReference.Status = portainer.EndpointStatusDown
-			}
-
-			if snapshot != nil {
-				latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
-			}
-
-			err = runner.context.dataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
-			if err != nil {
-				log.Printf("background schedule error (endpoint snapshot). Unable to update endpoint (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
-				return
-			}
-		}
-	}()
-}

api/cron/scheduler.go

@@ -1,116 +0,0 @@
-package cron
-
-import (
-	"github.com/portainer/portainer/api"
-	"github.com/robfig/cron/v3"
-)
-
-// JobScheduler represents a service for managing crons
-type JobScheduler struct {
-	cron *cron.Cron
-}
-
-// NewJobScheduler initializes a new service
-func NewJobScheduler() *JobScheduler {
-	return &JobScheduler{
-		cron: cron.New(),
-	}
-}
-
-// ScheduleJob schedules the execution of a job via a runner
-func (scheduler *JobScheduler) ScheduleJob(runner portainer.JobRunner) error {
-	_, err := scheduler.cron.AddJob(runner.GetSchedule().CronExpression, runner)
-	return err
-}
-
-// UpdateSystemJobSchedule updates the first occurence of the specified
-// scheduled job based on the specified job type.
-// It does so by re-creating a new cron
-// and adding all the existing jobs. It will then re-schedule the new job
-// with the update cron expression passed in parameter.
-// NOTE: the cron library do not support updating schedules directly
-// hence the work-around
-func (scheduler *JobScheduler) UpdateSystemJobSchedule(jobType portainer.JobType, newCronExpression string) error {
-	cronEntries := scheduler.cron.Entries()
-	newCron := cron.New()
-
-	for _, entry := range cronEntries {
-		if entry.Job.(portainer.JobRunner).GetSchedule().JobType == jobType {
-			_, err := newCron.AddJob(newCronExpression, entry.Job)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		newCron.Schedule(entry.Schedule, entry.Job)
-	}
-
-	scheduler.cron.Stop()
-	scheduler.cron = newCron
-	scheduler.cron.Start()
-	return nil
-}
-
-// UpdateJobSchedule updates a specific scheduled job by re-creating a new cron
-// and adding all the existing jobs. It will then re-schedule the new job
-// via the specified JobRunner parameter.
-// NOTE: the cron library do not support updating schedules directly
-// hence the work-around
-func (scheduler *JobScheduler) UpdateJobSchedule(runner portainer.JobRunner) error {
-	cronEntries := scheduler.cron.Entries()
-	newCron := cron.New()
-
-	for _, entry := range cronEntries {
-
-		if entry.Job.(portainer.JobRunner).GetSchedule().ID == runner.GetSchedule().ID {
-
-			var jobRunner cron.Job = runner
-			if entry.Job.(portainer.JobRunner).GetSchedule().JobType == portainer.SnapshotJobType {
-				jobRunner = entry.Job
-			}
-
-			_, err := newCron.AddJob(runner.GetSchedule().CronExpression, jobRunner)
-			if err != nil {
-				return err
-			}
-			continue
-		}
-
-		newCron.Schedule(entry.Schedule, entry.Job)
-	}
-
-	scheduler.cron.Stop()
-	scheduler.cron = newCron
-	scheduler.cron.Start()
-	return nil
-}
-
-// UnscheduleJob remove a scheduled job by re-creating a new cron
-// and adding all the existing jobs except for the one specified via scheduleID.
-// NOTE: the cron library do not support removing schedules directly
-// hence the work-around
-func (scheduler *JobScheduler) UnscheduleJob(scheduleID portainer.ScheduleID) {
-	cronEntries := scheduler.cron.Entries()
-	newCron := cron.New()
-
-	for _, entry := range cronEntries {
-
-		if entry.Job.(portainer.JobRunner).GetSchedule().ID == scheduleID {
-			continue
-		}
-
-		newCron.Schedule(entry.Schedule, entry.Job)
-	}
-
-	scheduler.cron.Stop()
-	scheduler.cron = newCron
-	scheduler.cron.Start()
-}
-
-// Start starts the scheduled jobs
-func (scheduler *JobScheduler) Start() {
-	if len(scheduler.cron.Entries()) > 0 {
-		scheduler.cron.Start()
-	}
-}

api/crypto/aes.go

@@ -0,0 +1,70 @@
+package crypto
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"io"
+
+	"golang.org/x/crypto/scrypt"
+)
+
+// NOTE: has to go with what is considered to be a simplistic in that it omits any
+// authentication of the encrypted data.
+// Person with better knowledge is welcomed to improve it.
+// sourced from https://golang.org/src/crypto/cipher/example_test.go
+
+var emptySalt []byte = make([]byte, 0, 0)
+
+// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
+// passphrase is used to generate an encryption key.
+func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
+	// making a 32 bytes key that would correspond to AES-256
+	// don't necessarily need a salt, so just kept in empty
+	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	if err != nil {
+		return err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	// If the key is unique for each ciphertext, then it's ok to use a zero
+	// IV.
+	var iv [aes.BlockSize]byte
+	stream := cipher.NewOFB(block, iv[:])
+
+	writer := &cipher.StreamWriter{S: stream, W: output}
+	// Copy the input to the output, encrypting as we go.
+	if _, err := io.Copy(writer, input); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// making a 32 bytes key that would correspond to AES-256
+	// don't necessarily need a salt, so just kept in empty
+	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the key is unique for each ciphertext, then it's ok to use a zero
+	// IV.
+	var iv [aes.BlockSize]byte
+	stream := cipher.NewOFB(block, iv[:])
+
+	reader := &cipher.StreamReader{S: stream, R: input}
+
+	return reader, nil
+}

api/crypto/aes_test.go

@@ -0,0 +1,131 @@
+package crypto
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(""))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(""))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "encrypt")
+	defer os.RemoveAll(tmpdir)
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
+	)
+
+	content := []byte("content")
+	ioutil.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+	defer encryptedFileWriter.Close()
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
+	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+}

api/crypto/tls.go

@@ -6,6 +6,24 @@ import (
 	"io/ioutil"
 )
 
+// CreateServerTLSConfiguration creates a basic tls.Config to be used by servers with recommended TLS settings
+func CreateServerTLSConfiguration() *tls.Config {
+	return &tls.Config{
+		MinVersion: tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_AES_128_GCM_SHA256,
+			tls.TLS_AES_256_GCM_SHA384,
+			tls.TLS_CHACHA20_POLY1305_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+		},
+	}
+}
+
 // CreateTLSConfigurationFromBytes initializes a tls.Config using a CA certificate, a certificate and a key
 // loaded from memory.
 func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) {

api/dev.sh

@@ -0,0 +1,3 @@
+#! /bin/sh
+
+go run -v -ldflags="-X github.com/portainer/liblicense.LicenseServerBaseURL=http://localhost:8080" cmd/portainer/main.go --data=./tmp/data

api/docker/client.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
@@ -11,8 +12,9 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 )
 
+var errUnsupportedEnvironmentType = errors.New("Environment not supported")
+
 const (
-	unsupportedEnvironmentType  = portainer.Error("Environment not supported")
 	defaultDockerRequestTimeout = 60
 	dockerClientVersion         = "1.37"
 )
@@ -31,13 +33,15 @@ func NewClientFactory(signatureService portainer.DigitalSignatureService, revers
 	}
 }
 
-// CreateClient is a generic function to create a Docker client based on
+// createClient is a generic function to create a Docker client based on
 // a specific endpoint configuration. The nodeName parameter can be used
 // with an agent enabled endpoint to target a specific node in an agent cluster.
 func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string) (*client.Client, error) {
-	if endpoint.Type == portainer.AgentOnDockerEnvironment {
+	if endpoint.Type == portainer.AzureEnvironment {
+		return nil, errUnsupportedEnvironmentType
+	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
 		return createAgentClient(endpoint, factory.signatureService, nodeName)
-	} else if endpoint.Type == portainer.EdgeAgentEnvironment {
+	} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
 		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
 	}