feat(registries): merge back and front features

.github/ISSUE_TEMPLATE.md

@@ -28,15 +28,17 @@ Briefly describe the problem you are having in a few paragraphs.
 
 **Steps to reproduce the issue:**
 
-1. 2. 3.
+1.
+2.
+3.
 
 Any other info e.g. Why do you consider this to be a bug? What did you expect to happen instead?
 
 **Technical details:**
 
-- Portainer version:
-- Target Docker version (the host/cluster you manage):
-- Platform (windows/linux):
-- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
-- Target Swarm version (if applicable):
-- Browser:
+* Portainer version:
+* Target Docker version (the host/cluster you manage):
+* Platform (windows/linux):
+* Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
+* Target Swarm version (if applicable):
+* Browser:

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -4,6 +4,7 @@ about: Create a bug report
 title: ''
 labels: bug/need-confirmation, kind/bug
 assignees: ''
+
 ---
 
 <!--
@@ -30,7 +31,7 @@ A clear and concise description of what you expected to happen.
 
 **Portainer Logs**
 Provide the logs of your Portainer container or Service.
-You can see how [here](https://documentation.portainer.io/r/portainer-logs)
+You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#how-do-i-get-the-logs-from-portainer)
 
 **Steps to reproduce the issue:**
 
@@ -45,7 +46,7 @@ You can see how [here](https://documentation.portainer.io/r/portainer-logs)
 - Docker version (managed by Portainer):
 - Kubernetes version (managed by Portainer):
 - Platform (windows/linux):
-- Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
+- Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
 - Browser:
 - Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
 - Have you reviewed our technical documentation and knowledge base? Yes/No

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Portainer Business
-    url: https://www.portainer.io/portainerbusiness 
-    about: Would you and your co-workers benefit from our enterprise edition which provides functionality to deploy Portainer at scale? 

.github/stale.yml

@@ -0,0 +1,54 @@
+# Config for Stalebot, limited to only `issues`
+only: issues
+
+# Issues config
+issues:
+  daysUntilStale: 60
+  daysUntilClose: 7
+
+  # Limit the number of actions per hour, from 1-30. Default is 30
+  limitPerRun: 30
+
+  # Issues with these labels will never be considered stale
+  exemptLabels:
+    - kind/enhancement
+    - kind/question
+    - kind/style
+    - kind/workaround
+    - kind/refactor
+    - bug/need-confirmation
+    - bug/confirmed
+    - status/discuss
+
+  # Only issues with all of these labels are checked if stale. Defaults to `[]` (disabled)
+  onlyLabels: []
+
+  # Set to true to ignore issues in a project (defaults to false)
+  exemptProjects: true
+  # Set to true to ignore issues in a milestone (defaults to false)
+  exemptMilestones: true
+  # Set to true to ignore issues with an assignee (defaults to false)
+  exemptAssignees: true
+
+  # Label to use when marking an issue as stale
+  staleLabel: status/stale
+
+  # Comment to post when marking an issue as stale. Set to `false` to disable
+  markComment: >
+    This issue has been marked as stale as it has not had recent activity,
+    it will be closed if no further activity occurs in the next 7 days.
+    If you believe that it has been incorrectly labelled as stale,
+    leave a comment and the label will be removed.
+
+  # Comment to post when removing the stale label.
+  # unmarkComment: >
+  #   Your comment here.
+
+  # Comment to post when closing a stale issue. Set to `false` to disable
+  closeComment: >
+    Since no further activity has appeared on this issue it will be closed.
+    If you believe that it has been incorrectly closed, leave a comment
+    mentioning `ametdoohan`, `balasu` or `keverv` and one of our staff will then review the issue.
+
+    Note - If it is an old bug report, make sure that it is reproduceable in the
+    latest version of Portainer as it may have already been fixed.

.github/workflows/label-conflcts.yaml

@@ -1,15 +0,0 @@
-on:
-  push:
-    branches:
-      - develop
-      - 'release/**'
-jobs:
-  triage:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: mschilde/auto-label-merge-conflicts@master
-        with:
-          CONFLICT_LABEL_NAME: 'has conflicts'
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MAX_RETRIES: 5
-          WAIT_MS: 5000

.github/workflows/stale.yml

@@ -1,27 +0,0 @@
-name: Close Stale Issues
-on:
-  schedule:
-  - cron: '0 12 * * *'
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
-    steps:
-    - uses: actions/stale@v4.0.0
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-        # Issue Config
-        days-before-issue-stale: 60
-        days-before-issue-close: 7
-        stale-issue-label: 'status/stale'
-        exempt-all-issue-milestones: true # Do not stale issues in a milestone
-        exempt-issue-labels: kind/enhancement, kind/style, kind/workaround, kind/refactor, bug/need-confirmation, bug/confirmed, status/discuss
-        stale-issue-message: 'This issue has been marked as stale as it has not had recent activity, it will be closed if no further activity occurs in the next 7 days. If you believe that it has been incorrectly labelled as stale, leave a comment and the label will be removed.'
-        close-issue-message: 'Since no further activity has appeared on this issue it will be closed. If you believe that it has been incorrectly closed, leave a comment mentioning `portainer/support` and one of our staff will then review the issue. Note - If it is an old bug report, make sure that it is reproduceable in the latest version of Portainer as it may have already been fixed.'
-        
-        # Pull Request Config
-        days-before-pr-stale: -1 # Do not stale pull request
-        days-before-pr-close: -1 # Do not close pull request

.gitignore

@@ -11,5 +11,3 @@ api/cmd/portainer/portainer*
 __debug_bin
 
 api/docs
-.idea
-.env

.vscode.example/portainer.code-snippets

@@ -163,19 +163,5 @@
       "// @failure 500 \"Server error\"",
       "// @router /{id} [get]"
     ]
-  },
-  "analytics": {
-    "prefix": "nlt",
-    "body": ["analytics-on", "analytics-category=\"$1\"", "analytics-event=\"$2\""],
-    "description": "analytics"
-  },
-  "analytics-if": {
-    "prefix": "nltf",
-    "body": ["analytics-if=\"$1\""],
-    "description": "analytics"
-  },
-  "analytics-metadata": {
-    "prefix": "nltm",
-    "body": "analytics-properties=\"{ metadata: { $1 } }\""
   }
 }

.vscode.example/settings.json

@@ -1,4 +0,0 @@
-{
-  "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast", "-E", "exportloopref"]
-}

ATTRIBUTIONS.md

@@ -1,32 +0,0 @@
-# Open Source License Attribution
-
-This application uses Open Source components. You can find the source
-code of their open source projects along with license information below.
-We acknowledge and are grateful to these developers for their contributions
-to open source.
-
-### [angular-json-tree](https://github.com/awendland/angular-json-tree)
-
-by [Alex Wendland](https://github.com/awendland) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
-
-### [caniuse-db](https://github.com/Fyrd/caniuse)
-
-by [caniuse.com](caniuse.com) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
-
-### [caniuse-lite](https://github.com/ben-eb/caniuse-lite)
-
-by [caniuse.com](caniuse.com) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
-
-### [spdx-exceptions](https://github.com/jslicense/spdx-exceptions.json)
-
-by Kyle Mitchell using [SPDX](https://spdx.dev/) from Linux Foundation licensed under [CC BY 3.0 License](https://creativecommons.org/licenses/by/3.0/)
-
-### [fontawesome-free](https://github.com/FortAwesome/Font-Awesome) Icons
-
-by [Fort Awesome](https://fortawesome.com/) is licensed under [CC BY 4.0 License](https://creativecommons.org/licenses/by/4.0/)
-
-Portainer also contains the following code, which is licensed under the [MIT license](https://opensource.org/licenses/MIT):
-
-UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
-
-rdash-angular: Copyright (c) [2014][elliot hesp]

CONTRIBUTING.md

@@ -91,7 +91,7 @@ Then build and run the project:
 $ yarn start
 ```
 
-Portainer can now be accessed at <https://localhost:9443>.
+Portainer can now be accessed at <http://localhost:9000>.
 
 Find more detailed steps at <https://documentation.portainer.io/contributing/instructions/>.
 
@@ -127,9 +127,3 @@ When adding a new route to an existing handler use the following as a template (
 ```
 
 explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]
-
-## Licensing
-
-See the [LICENSE](https://github.com/portainer/portainer/blob/develop/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
-
-We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.

README.md

@@ -1,14 +1,16 @@
 <p align="center">
-  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/portainer-github-banner.png?raw=true' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/logo_alt.png?raw=true' />
 </p>
 
-**Portainer CE** is a lightweight ‘universal’ management GUI that can be used to **easily** manage Docker, Swarm, Kubernetes and ACI environments. It is designed to be as **simple** to deploy as it is to use.
+[![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
+[![Microbadger](https://images.microbadger.com/badges/image/portainer/portainer.svg)](http://microbadger.com/images/portainer/portainer 'Image size')
+[![Build Status](https://portainer.visualstudio.com/Portainer%20CI/_apis/build/status/Portainer%20CI?branchName=develop)](https://portainer.visualstudio.com/Portainer%20CI/_build/latest?definitionId=3&branchName=develop)
+[![Code Climate](https://codeclimate.com/github/portainer/portainer/badges/gpa.svg)](https://codeclimate.com/github/portainer/portainer)
+[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YHXZJQNJQ36H6)
 
-Portainer consists of a single container that can run on any cluster. It can be deployed as a Linux container or a Windows native container.
-
-**Portainer** allows you to manage all your orchestrator resources (containers, images, volumes, networks and more) through a super-simple graphical interface.
-
-A fully supported version of Portainer is available for business use. Visit http://www.portainer.io to learn more
+**_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
+**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container, supports other platforms too).
+**_Portainer_** allows you to manage all your Docker resources (containers, images, volumes, networks and more!) It is compatible with the _standalone Docker_ engine and with _Docker Swarm mode_.
 
 ## Demo
 
@@ -16,37 +18,29 @@ You can try out the public demo instance: http://demo.portainer.io/ (login with
 
 Please note that the public demo cluster is **reset every 15min**.
 
-## Latest Version
+Alternatively, you can deploy a copy of the demo stack inside a [play-with-docker (PWD)](https://labs.play-with-docker.com) playground:
 
-Portainer CE is updated regularly. We aim to do an update release every couple of months.
+- Browse [PWD/?stack=portainer-demo/play-with-docker/docker-stack.yml](http://play-with-docker.com/?stack=https://raw.githubusercontent.com/portainer/portainer-demo/master/play-with-docker/docker-stack.yml)
+- Sign in with your [Docker ID](https://docs.docker.com/docker-id)
+- Follow [these](https://github.com/portainer/portainer-demo/blob/master/play-with-docker/docker-stack.yml#L5-L8) steps.
 
-**The latest version of Portainer is 2.6.x** And you can find the release notes [here.](https://www.portainer.io/blog/new-portainer-ce-2.6.0-release)
-Portainer is on version 2, the second number denotes the month of release.
+Unlike the public demo, the playground sessions are deleted after 4 hours. Apart from that, all the settings are the same, including default credentials.
 
 ## Getting started
 
 - [Deploy Portainer](https://documentation.portainer.io/quickstart/)
 - [Documentation](https://documentation.portainer.io)
-- [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
-
-## Features & Functions
-
-View [this](https://www.portainer.io/products) table to see all of the Portainer CE functionality and compare to Portainer Business.
-
-- [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
-- [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
-- [Portainer CE for Azure ACI](https://www.portainer.io/solutions/serverless-containers)
+- [Building Portainer](https://documentation.portainer.io/contributing/instructions/)
 
 ## Getting help
 
-Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
+For FORMAL Support, please purchase a support subscription from here: https://www.portainer.io/products/portainer-business
 
-Learn more about Portainers community support channels [here.](https://www.portainer.io/help_about)
+For community support: You can find more information about Portainer's community support framework policy here: https://www.portainer.io/products/community-edition/customer-success
 
 - Issues: https://github.com/portainer/portainer/issues
-- Slack (chat): [https://portainer.slack.com/](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA)
-
-You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
+- FAQ: https://documentation.portainer.io
+- Slack (chat): https://portainer.io/slack/
 
 ## Reporting bugs and contributing
 
@@ -57,10 +51,6 @@ You can join the Portainer Community by visiting community.portainer.io. This wi
 
 - Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
 
-## WORK FOR US
-
-If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and we will be in touch.
-
 ## Privacy
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
@@ -75,4 +65,8 @@ Portainer supports "Current - 2 docker versions only. Prior versions may operate
 
 Portainer is licensed under the zlib license. See [LICENSE](./LICENSE) for reference.
 
-Portainer also contains code from open source projects. See [ATTRIBUTIONS.md](./ATTRIBUTIONS.md) for a list.
+Portainer also contains the following code, which is licensed under the [MIT license](https://opensource.org/licenses/MIT):
+
+UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
+
+rdash-angular: Copyright (c) [2014][elliot hesp]

api/adminmonitor/admin_monitor.go

@@ -1,69 +0,0 @@
-package adminmonitor
-
-import (
-	"context"
-	"log"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-var logFatalf = log.Fatalf
-
-type Monitor struct {
-	timeout          time.Duration
-	datastore        portainer.DataStore
-	shutdownCtx      context.Context
-	cancellationFunc context.CancelFunc
-}
-
-// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
-func New(timeout time.Duration, datastore portainer.DataStore, shutdownCtx context.Context) *Monitor {
-	return &Monitor{
-		timeout:     timeout,
-		datastore:   datastore,
-		shutdownCtx: shutdownCtx,
-	}
-}
-
-// Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
-func (m *Monitor) Start() {
-	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
-	m.cancellationFunc = cancellationFunc
-
-	go func() {
-		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
-		select {
-		case <-time.After(m.timeout):
-			initialized, err := m.WasInitialized()
-			if err != nil {
-				logFatalf("%s", err)
-			}
-			if !initialized {
-				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
-			}
-		case <-cancellationCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
-		case <-m.shutdownCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
-		}
-	}()
-}
-
-// Stop stops monitor. Safe to call even if monitor wasn't started.
-func (m *Monitor) Stop() {
-	if m.cancellationFunc == nil {
-		return
-	}
-	m.cancellationFunc()
-	m.cancellationFunc = nil
-}
-
-// WasInitialized is a system initialization check
-func (m *Monitor) WasInitialized() (bool, error) {
-	users, err := m.datastore.User().UsersByRole(portainer.AdministratorRole)
-	if err != nil {
-		return false, err
-	}
-	return len(users) > 0, nil
-}

api/adminmonitor/admin_monitor_test.go

@@ -1,50 +0,0 @@
-package adminmonitor
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	i "github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_stopWithoutStarting(t *testing.T) {
-	monitor := New(1*time.Minute, nil, nil)
-	monitor.Stop()
-}
-
-func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
-	monitor := New(1*time.Minute, nil, nil)
-	monitor.Stop()
-	monitor.Stop()
-}
-
-func Test_canStopStartedMonitor(t *testing.T) {
-	monitor := New(1*time.Minute, nil, context.Background())
-	monitor.Start()
-	assert.NotNil(t, monitor.cancellationFunc, "cancellation function is missing in started monitor")
-
-	monitor.Stop()
-	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
-}
-
-func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
-	timeout := 10 * time.Millisecond
-
-	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
-
-	var fataled bool
-	origLogFatalf := logFatalf
-	logFatalf = func(s string, v ...interface{}) { fataled = true }
-	defer func() {
-		logFatalf = origLogFatalf
-	}()
-
-	monitor := New(timeout, datastore, context.Background())
-	monitor.Start()
-	<-time.After(2 * timeout)
-
-	assert.True(t, fataled, "monitor should been timeout and fatal")
-}

api/api-description.md

@@ -1,10 +1,10 @@
 Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
-Examples are available at https://documentation.portainer.io/api/api-examples/
+Examples are available at https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8
 You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
 
 # Authentication
 
-Most of the API environments(endpoints) require to be authenticated as well as some level of authorization to be used.
+Most of the API endpoints require to be authenticated as well as some level of authorization to be used.
 Portainer API uses JSON Web Token to manage authentication and thus requires you to provide a token in the **Authorization** header of each request
 with the **Bearer** authentication mechanism.
 
@@ -16,7 +16,7 @@ Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIs
 
 # Security
 
-Each API environment(endpoint) has an associated access policy, it is documented in the description of each environment(endpoint).
+Each API endpoint has an associated access policy, it is documented in the description of each endpoint.
 
 Different access policies are available:
 
@@ -27,27 +27,27 @@ Different access policies are available:
 
 ### Public access
 
-No authentication is required to access the environments(endpoints) with this access policy.
+No authentication is required to access the endpoints with this access policy.
 
 ### Authenticated access
 
-Authentication is required to access the environments(endpoints) with this access policy.
+Authentication is required to access the endpoints with this access policy.
 
 ### Restricted access
 
-Authentication is required to access the environments(endpoints) with this access policy.
+Authentication is required to access the endpoints with this access policy.
 Extra-checks might be added to ensure access to the resource is granted. Returned data might also be filtered.
 
 ### Administrator access
 
-Authentication as well as an administrator role are required to access the environments(endpoints) with this access policy.
+Authentication as well as an administrator role are required to access the endpoints with this access policy.
 
 # Execute Docker requests
 
-Portainer **DO NOT** expose specific environments(endpoints) to manage your Docker resources (create a container, remove a volume, etc...).
+Portainer **DO NOT** expose specific endpoints to manage your Docker resources (create a container, remove a volume, etc...).
 
 Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you can execute Docker requests **via** the Portainer HTTP API.
 
-To do so, you can use the `/endpoints/{id}/docker` Portainer API environment(endpoint) (which is not documented below due to Swagger limitations). This environment(endpoint) has a restricted access policy so you still need to be authenticated to be able to query this environment(endpoint). Any query on this environment(endpoint) will be proxied to the Docker API of the associated environment(endpoint) (requests and responses objects are the same as documented in the Docker API).
+To do so, you can use the `/endpoints/{id}/docker` Portainer API endpoint (which is not documented below due to Swagger limitations). This endpoint has a restricted access policy so you still need to be authenticated to be able to query this endpoint. Any query on this endpoint will be proxied to the Docker API of the associated endpoint (requests and responses objects are the same as documented in the Docker API).
 
-**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).
+**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://gist.github.com/deviantony/77026d402366b4b43fa5918d41bc42f8).

api/archive/targz.go

@@ -1,119 +0,0 @@
-package archive
-
-import (
-	"archive/tar"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// TarGzDir creates a tar.gz archive and returns it's path.
-// abosolutePath should be an absolute path to a directory.
-// Archive name will be <directoryName>.tar.gz and will be placed next to the directory.
-func TarGzDir(absolutePath string) (string, error) {
-	targzPath := filepath.Join(absolutePath, fmt.Sprintf("%s.tar.gz", filepath.Base(absolutePath)))
-	outFile, err := os.Create(targzPath)
-	if err != nil {
-		return "", err
-	}
-	defer outFile.Close()
-
-	zipWriter := gzip.NewWriter(outFile)
-	defer zipWriter.Close()
-	tarWriter := tar.NewWriter(zipWriter)
-	defer tarWriter.Close()
-
-	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if path == targzPath {
-			return nil // skip archive file
-		}
-
-		pathInArchive := filepath.Clean(strings.TrimPrefix(path, absolutePath))
-		if pathInArchive == "" {
-			return nil // skip root dir
-		}
-
-		return addToArchive(tarWriter, pathInArchive, path, info)
-	})
-
-	return targzPath, err
-}
-
-func addToArchive(tarWriter *tar.Writer, pathInArchive string, path string, info os.FileInfo) error {
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-
-	header.Name = pathInArchive // use relative paths in archive
-
-	err = tarWriter.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
-	if info.IsDir() {
-		return nil
-	}
-
-	file, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(tarWriter, file)
-	return err
-}
-
-// ExtractTarGz reads a .tar.gz archive from the reader and extracts it into outputDirPath directory
-func ExtractTarGz(r io.Reader, outputDirPath string) error {
-	zipReader, err := gzip.NewReader(r)
-	if err != nil {
-		return err
-	}
-	defer zipReader.Close()
-
-	tarReader := tar.NewReader(zipReader)
-
-	for {
-		header, err := tarReader.Next()
-
-		if err == io.EOF {
-			break
-		}
-
-		if err != nil {
-			return err
-		}
-
-		switch header.Typeflag {
-		case tar.TypeDir:
-			// skip, dir will be created with a file
-		case tar.TypeReg:
-			p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
-			if err := os.MkdirAll(filepath.Dir(p), 0744); err != nil {
-				return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
-			}
-			outFile, err := os.Create(p)
-			if err != nil {
-				return fmt.Errorf("Failed to create file %s", header.Name)
-			}
-			if _, err := io.Copy(outFile, tarReader); err != nil {
-				return fmt.Errorf("Failed to extract file %s", header.Name)
-			}
-			outFile.Close()
-		default:
-			return fmt.Errorf("Tar: uknown type: %v in %s",
-				header.Typeflag,
-				header.Name)
-		}
-	}
-
-	return nil
-}

api/archive/targz_test.go

@@ -1,99 +0,0 @@
-package archive
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func Test_shouldCreateArhive(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
-	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
-
-	gzPath, err := TarGzDir(tmpdir)
-	assert.Nil(t, err)
-	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
-
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
-	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
-	err = cmd.Run()
-	if err != nil {
-		t.Fatal("Failed to extract archive: ", err)
-	}
-	extractedFiles := listFiles(extractionDir)
-
-	wasExtracted := func(p string) {
-		fullpath := path.Join(extractionDir, p)
-		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
-		assert.Equal(t, content, copyContent)
-	}
-
-	wasExtracted("outer")
-	wasExtracted("dir/inner")
-	wasExtracted("dir/.dotfile")
-}
-
-func Test_shouldCreateArhiveXXXXX(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
-	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
-	ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
-
-	gzPath, err := TarGzDir(tmpdir)
-	assert.Nil(t, err)
-	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
-
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
-	r, _ := os.Open(gzPath)
-	ExtractTarGz(r, extractionDir)
-	if err != nil {
-		t.Fatal("Failed to extract archive: ", err)
-	}
-	extractedFiles := listFiles(extractionDir)
-
-	wasExtracted := func(p string) {
-		fullpath := path.Join(extractionDir, p)
-		assert.Contains(t, extractedFiles, fullpath)
-		copyContent, _ := ioutil.ReadFile(fullpath)
-		assert.Equal(t, content, copyContent)
-	}
-
-	wasExtracted("outer")
-	wasExtracted("dir/inner")
-	wasExtracted("dir/.dotfile")
-}

api/archive/zip.go

@@ -3,13 +3,10 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
-	"fmt"
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
@@ -55,60 +52,3 @@ func extractFileFromArchive(file *zip.File, dest string) error {
 
 	return outFile.Close()
 }
-
-// UnzipFile will decompress a zip archive, moving all files and folders
-// within the zip file (parameter 1) to an output directory (parameter 2).
-func UnzipFile(src string, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer r.Close()
-
-	for _, f := range r.File {
-		p := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
-		if !strings.HasPrefix(p, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("%s: illegal file path", p)
-		}
-
-		if f.FileInfo().IsDir() {
-			// Make Folder
-			os.MkdirAll(p, os.ModePerm)
-			continue
-		}
-
-		err = unzipFile(f, p)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func unzipFile(f *zip.File, p string) error {
-	// Make File
-	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
-		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
-	}
-	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
-	}
-	defer outFile.Close()
-	rc, err := f.Open()
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
-	}
-	defer rc.Close()
-
-	_, err = io.Copy(outFile, rc)
-
-	if err != nil {
-		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
-	}
-
-	return nil
-}

api/archive/zip_test.go

@@ -1,32 +0,0 @@
-package archive
-
-import (
-	"github.com/stretchr/testify/assert"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestUnzipFile(t *testing.T) {
-	dir, err := ioutil.TempDir("", "unzip-test-")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dir)
-	/*
-		Archive structure.
-		├── 0
-		│	├── 1
-		│	│	└── 2.txt
-		│	└── 1.txt
-		└── 0.txt
-	*/
-
-	err = UnzipFile("./testdata/sample_archive.zip", dir)
-
-	assert.NoError(t, err)
-	archiveDir := dir + "/sample_archive"
-	assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
-	assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
-	assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
-
-}

api/backup/backup.go

@@ -1,95 +0,0 @@
-package backup
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/filesystem"
-	"github.com/portainer/portainer/api/http/offlinegate"
-)
-
-const rwxr__r__ os.FileMode = 0744
-
-var filesToBackup = []string{
-	"certs",
-	"compose",
-	"config.json",
-	"custom_templates",
-	"edge_jobs",
-	"edge_stacks",
-	"extensions",
-	"portainer.key",
-	"portainer.pub",
-	"tls",
-}
-
-// Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
-func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
-	unlock := gate.Lock()
-	defer unlock()
-
-	backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
-	if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
-		return "", errors.Wrap(err, "Failed to create backup dir")
-	}
-
-	if err := backupDb(backupDirPath, datastore); err != nil {
-		return "", errors.Wrap(err, "Failed to backup database")
-	}
-
-	for _, filename := range filesToBackup {
-		err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath)
-		if err != nil {
-			return "", errors.Wrap(err, "Failed to create backup file")
-		}
-	}
-
-	archivePath, err := archive.TarGzDir(backupDirPath)
-	if err != nil {
-		return "", errors.Wrap(err, "Failed to make an archive")
-	}
-
-	if password != "" {
-		archivePath, err = encrypt(archivePath, password)
-		if err != nil {
-			return "", errors.Wrap(err, "Failed to encrypt backup with the password")
-		}
-	}
-
-	return archivePath, nil
-}
-
-func backupDb(backupDirPath string, datastore portainer.DataStore) error {
-	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
-	if err != nil {
-		return err
-	}
-	if err = datastore.BackupTo(backupWriter); err != nil {
-		return err
-	}
-	return backupWriter.Close()
-}
-
-func encrypt(path string, passphrase string) (string, error) {
-	in, err := os.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer in.Close()
-
-	outFileName := fmt.Sprintf("%s.encrypted", path)
-	out, err := os.Create(outFileName)
-	if err != nil {
-		return "", err
-	}
-
-	err = crypto.AesEncrypt(in, out, []byte(passphrase))
-
-	return outFileName, err
-}

api/backup/restore.go

@@ -1,69 +0,0 @@
-package backup
-
-import (
-	"context"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/archive"
-	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/filesystem"
-	"github.com/portainer/portainer/api/http/offlinegate"
-)
-
-var filesToRestore = append(filesToBackup, "portainer.db")
-
-// Restores system state from backup archive, will trigger system shutdown, when finished.
-func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, shutdownTrigger context.CancelFunc) error {
-	var err error
-	if password != "" {
-		archive, err = decrypt(archive, password)
-		if err != nil {
-			return errors.Wrap(err, "failed to decrypt the archive")
-		}
-	}
-
-	restorePath := filepath.Join(filestorePath, "restore", time.Now().Format("20060102150405"))
-	defer os.RemoveAll(filepath.Dir(restorePath))
-
-	err = extractArchive(archive, restorePath)
-	if err != nil {
-		return errors.Wrap(err, "cannot extract files from the archive. Please ensure the password is correct and try again")
-	}
-
-	unlock := gate.Lock()
-	defer unlock()
-
-	if err = datastore.Close(); err != nil {
-		return errors.Wrap(err, "Failed to stop db")
-	}
-
-	if err = restoreFiles(restorePath, filestorePath); err != nil {
-		return errors.Wrap(err, "failed to restore the system state")
-	}
-
-	shutdownTrigger()
-	return nil
-}
-
-func decrypt(r io.Reader, password string) (io.Reader, error) {
-	return crypto.AesDecrypt(r, []byte(password))
-}
-
-func extractArchive(r io.Reader, destinationDirPath string) error {
-	return archive.ExtractTarGz(r, destinationDirPath)
-}
-
-func restoreFiles(srcDir string, destinationDir string) error {
-	for _, filename := range filesToRestore {
-		err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

api/bolt/backup.go

@@ -1,142 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"time"
-
-	plog "github.com/portainer/portainer/api/bolt/log"
-)
-
-var backupDefaults = struct {
-	backupDir        string
-	commonDir        string
-	databaseFileName string
-}{
-	"backups",
-	"common",
-	databaseFileName,
-}
-
-var backupLog = plog.NewScopedLog("bolt, backup")
-
-//
-// Backup Helpers
-//
-
-// createBackupFolders create initial folders for backups
-func (store *Store) createBackupFolders() {
-	// create common dir
-	commonDir := store.commonBackupDir()
-	if exists, _ := store.fileService.FileExists(commonDir); !exists {
-		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			backupLog.Error("Error while creating common backup folder", err)
-		}
-	}
-}
-
-func (store *Store) databasePath() string {
-	return path.Join(store.path, databaseFileName)
-}
-
-func (store *Store) commonBackupDir() string {
-	return path.Join(store.path, backupDefaults.backupDir, backupDefaults.commonDir)
-}
-
-func (store *Store) copyDBFile(from string, to string) error {
-	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
-	err := store.fileService.Copy(from, to, true)
-	if err != nil {
-		backupLog.Error("Failed", err)
-	}
-	return err
-}
-
-// BackupOptions provide a helper to inject backup options
-type BackupOptions struct {
-	Version        int
-	BackupDir      string
-	BackupFileName string
-	BackupPath     string
-}
-
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-	if options == nil {
-		options = &BackupOptions{}
-	}
-	if options.Version == 0 {
-		options.Version, _ = store.version()
-	}
-	if options.BackupDir == "" {
-		options.BackupDir = store.commonBackupDir()
-	}
-	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
-	}
-	if options.BackupPath == "" {
-		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-	}
-	return options
-}
-
-// BackupWithOptions backup current database with options
-func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
-	backupLog.Info("creating db backup")
-	store.createBackupFolders()
-
-	options = store.setupOptions(options)
-
-	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
-}
-
-// RestoreWithOptions previously saved backup for the current Edition  with options
-// Restore strategies:
-// - default: restore latest from current edition
-// - restore a specific
-func (store *Store) RestoreWithOptions(options *BackupOptions) error {
-	options = store.setupOptions(options)
-
-	// Check if backup file exist before restoring
-	_, err := os.Stat(options.BackupPath)
-	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
-		return err
-	}
-
-	err = store.Close()
-	if err != nil {
-		backupLog.Error("Error while closing store before restore", err)
-		return err
-	}
-
-	backupLog.Info("Restoring db backup")
-	err = store.copyDBFile(options.BackupPath, store.databasePath())
-	if err != nil {
-		return err
-	}
-
-	return store.Open()
-}
-
-// RemoveWithOptions removes backup database based on supplied options
-func (store *Store) RemoveWithOptions(options *BackupOptions) error {
-	backupLog.Info("Removing db backup")
-
-	options = store.setupOptions(options)
-	_, err := os.Stat(options.BackupPath)
-
-	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
-		return err
-	}
-
-	backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
-	err = os.Remove(options.BackupPath)
-	if err != nil {
-		backupLog.Error("Failed", err)
-		return err
-	}
-
-	return nil
-}

api/bolt/backup_test.go

@@ -1,116 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// isFileExist is helper function to check for file existence
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}
-
-func TestCreateBackupFolders(t *testing.T) {
-	store, teardown := MustNewTestStore(false)
-	defer teardown()
-
-	backupPath := path.Join(store.path, backupDefaults.backupDir)
-
-	if isFileExist(backupPath) {
-		t.Error("Expect backups folder to not exist")
-	}
-
-	store.createBackupFolders()
-	if !isFileExist(backupPath) {
-		t.Error("Expect backups folder to exist")
-	}
-}
-
-func TestStoreCreation(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	if store == nil {
-		t.Error("Expect to create a store")
-	}
-
-	if store.edition() != portainer.PortainerCE {
-		t.Error("Expect to get CE Edition")
-	}
-}
-
-func TestBackup(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	t.Run("Backup should create default db backup", func(t *testing.T) {
-		store.VersionService.StoreDBVersion(portainer.DBVersion)
-		store.BackupWithOptions(nil)
-
-		backupFileName := path.Join(store.path, "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-
-	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-		store.BackupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerVersionUpgradeBackup,
-			BackupDir:      store.commonBackupDir(),
-		})
-		backupFileName := path.Join(store.path, "backups", "common", beforePortainerVersionUpgradeBackup)
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-}
-
-func TestRemoveWithOptions(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	t.Run("successfully removes file if existent", func(t *testing.T) {
-		store.createBackupFolders()
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
-
-		filePath := path.Join(options.BackupDir, options.BackupFileName)
-		f, err := os.Create(filePath)
-		if err != nil {
-			t.Fatalf("file should be created; err=%s", err)
-		}
-		f.Close()
-
-		err = store.RemoveWithOptions(options)
-		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
-		}
-
-		if isFileExist(f.Name()) {
-			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
-		}
-	})
-
-	t.Run("fails to removes file if non-existent", func(t *testing.T) {
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
-
-		err := store.RemoveWithOptions(options)
-		if err == nil {
-			t.Error("RemoveWithOptions should fail for non-existent file")
-		}
-	})
-}

api/bolt/customtemplate/customtemplate.go

@@ -2,7 +2,7 @@ package customtemplate
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 	var customTemplates = make([]portainer.CustomTemplate, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 	var customTemplate portainer.CustomTemplate
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &customTemplate)
+	err := internal.GetObject(service.db, BucketName, identifier, &customTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portaine
 // UpdateCustomTemplate updates an custom template.
 func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, customTemplate)
+	return internal.UpdateObject(service.db, BucketName, identifier, customTemplate)
 }
 
 // DeleteCustomTemplate deletes an custom template.
 func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateCustomTemplate assign an ID to a new custom template and saves it.
 func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTemplate) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(customTemplate)
@@ -92,5 +92,5 @@ func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTem
 
 // GetNextIdentifier returns the next identifier for a custom template.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/datastore.go

@@ -1,12 +1,10 @@
 package bolt
 
 import (
-	"io"
+	"log"
 	"path"
 	"time"
 
-	"github.com/portainer/portainer/api/bolt/helmuserrepository"
-
 	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/customtemplate"
@@ -19,13 +17,12 @@ import (
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/extension"
-	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
 	"github.com/portainer/portainer/api/bolt/schedule"
 	"github.com/portainer/portainer/api/bolt/settings"
-	"github.com/portainer/portainer/api/bolt/ssl"
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
@@ -34,6 +31,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/bolt/webhook"
+	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 const (
@@ -43,42 +41,32 @@ const (
 // Store defines the implementation of portainer.DataStore using
 // BoltDB as the storage system.
 type Store struct {
-	path                      string
-	connection                *internal.DbConnection
-	isNew                     bool
-	fileService               portainer.FileService
-	CustomTemplateService     *customtemplate.Service
-	DockerHubService          *dockerhub.Service
-	EdgeGroupService          *edgegroup.Service
-	EdgeJobService            *edgejob.Service
-	EdgeStackService          *edgestack.Service
-	EndpointGroupService      *endpointgroup.Service
-	EndpointService           *endpoint.Service
-	EndpointRelationService   *endpointrelation.Service
-	ExtensionService          *extension.Service
-	HelmUserRepositoryService *helmuserrepository.Service
-	RegistryService           *registry.Service
-	ResourceControlService    *resourcecontrol.Service
-	RoleService               *role.Service
-	ScheduleService           *schedule.Service
-	SettingsService           *settings.Service
-	SSLSettingsService        *ssl.Service
-	StackService              *stack.Service
-	TagService                *tag.Service
-	TeamMembershipService     *teammembership.Service
-	TeamService               *team.Service
-	TunnelServerService       *tunnelserver.Service
-	UserService               *user.Service
-	VersionService            *version.Service
-	WebhookService            *webhook.Service
-}
-
-func (store *Store) version() (int, error) {
-	version, err := store.VersionService.DBVersion()
-	if err == errors.ErrObjectNotFound {
-		version = 0
-	}
-	return version, err
+	path                    string
+	db                      *bolt.DB
+	isNew                   bool
+	fileService             portainer.FileService
+	CustomTemplateService   *customtemplate.Service
+	DockerHubService        *dockerhub.Service
+	EdgeGroupService        *edgegroup.Service
+	EdgeJobService          *edgejob.Service
+	EdgeStackService        *edgestack.Service
+	EndpointGroupService    *endpointgroup.Service
+	EndpointService         *endpoint.Service
+	EndpointRelationService *endpointrelation.Service
+	ExtensionService        *extension.Service
+	RegistryService         *registry.Service
+	ResourceControlService  *resourcecontrol.Service
+	RoleService             *role.Service
+	ScheduleService         *schedule.Service
+	SettingsService         *settings.Service
+	StackService            *stack.Service
+	TagService              *tag.Service
+	TeamMembershipService   *teammembership.Service
+	TeamService             *team.Service
+	TunnelServerService     *tunnelserver.Service
+	UserService             *user.Service
+	VersionService          *version.Service
+	WebhookService          *webhook.Service
 }
 
 func (store *Store) edition() portainer.SoftwareEdition {
@@ -90,13 +78,24 @@ func (store *Store) edition() portainer.SoftwareEdition {
 }
 
 // NewStore initializes a new Store and the associated services
-func NewStore(storePath string, fileService portainer.FileService) *Store {
-	return &Store{
+func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
+	store := &Store{
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
-		connection:  &internal.DbConnection{},
 	}
+
+	databasePath := path.Join(storePath, databaseFileName)
+	databaseFileExists, err := fileService.FileExists(databasePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if databaseFileExists {
+		store.isNew = false
+	}
+
+	return store, nil
 }
 
 // Open opens and initializes the BoltDB database.
@@ -106,26 +105,15 @@ func (store *Store) Open() error {
 	if err != nil {
 		return err
 	}
-	store.connection.DB = db
+	store.db = db
 
-	err = store.initServices()
-	if err != nil {
-		return err
-	}
-
-	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
-		store.isNew = false
-	}
-
-	return nil
+	return store.initServices()
 }
 
 // Close closes the BoltDB database.
-// Safe to being called multiple times.
 func (store *Store) Close() error {
-	if store.connection.DB != nil {
-		return store.connection.Close()
+	if store.db != nil {
+		return store.db.Close()
 	}
 	return nil
 }
@@ -136,15 +124,6 @@ func (store *Store) IsNew() bool {
 	return store.isNew
 }
 
-// BackupTo backs up db to a provided writer.
-// It does hot backup and doesn't block other database reads and writes
-func (store *Store) BackupTo(w io.Writer) error {
-	return store.connection.View(func(tx *bolt.Tx) error {
-		_, err := tx.WriteTo(w)
-		return err
-	})
-}
-
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
 	if store.edition() != portainer.PortainerCE {
@@ -152,3 +131,287 @@ func (store *Store) CheckCurrentEdition() error {
 	}
 	return nil
 }
+
+// MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
+func (store *Store) MigrateData() error {
+	if store.isNew {
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
+	}
+
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	} else if err != nil {
+		return err
+	}
+
+	if version < portainer.DBVersion {
+		migratorParams := &migrator.Parameters{
+			DB:                      store.db,
+			DatabaseVersion:         version,
+			EndpointGroupService:    store.EndpointGroupService,
+			EndpointService:         store.EndpointService,
+			EndpointRelationService: store.EndpointRelationService,
+			ExtensionService:        store.ExtensionService,
+			RegistryService:         store.RegistryService,
+			ResourceControlService:  store.ResourceControlService,
+			RoleService:             store.RoleService,
+			ScheduleService:         store.ScheduleService,
+			SettingsService:         store.SettingsService,
+			StackService:            store.StackService,
+			TagService:              store.TagService,
+			TeamMembershipService:   store.TeamMembershipService,
+			UserService:             store.UserService,
+			VersionService:          store.VersionService,
+			FileService:             store.fileService,
+			AuthorizationService:    authorization.NewService(store),
+		}
+		migrator := migrator.NewMigrator(migratorParams)
+
+		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
+		err = migrator.Migrate()
+		if err != nil {
+			log.Printf("An error occurred during database migration: %s\n", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
+	customTemplateService, err := customtemplate.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.CustomTemplateService = customTemplateService
+
+	dockerhubService, err := dockerhub.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	edgeStackService, err := edgestack.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackService = edgeStackService
+
+	edgeGroupService, err := edgegroup.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeGroupService = edgeGroupService
+
+	edgeJobService, err := edgejob.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
+	endpointgroupService, err := endpointgroup.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	endpointRelationService, err := endpointrelation.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	extensionService, err := extension.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
+	registryService, err := registry.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	settingsService, err := settings.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	stackService, err := stack.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	tunnelServerService, err := tunnelserver.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
+	userService, err := user.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	webhookService, err := webhook.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
+	return nil
+}
+
+// CustomTemplate gives access to the CustomTemplate data management layer
+func (store *Store) CustomTemplate() portainer.CustomTemplateService {
+	return store.CustomTemplateService
+}
+
+// DockerHub gives access to the DockerHub data management layer
+func (store *Store) DockerHub() portainer.DockerHubService {
+	return store.DockerHubService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Endpoint gives access to the Endpoint data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
+}

api/bolt/dockerhub/dockerhub.go

@@ -1,8 +1,10 @@
 package dockerhub
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -13,18 +15,18 @@ const (
 
 // Service represents a service for managing Dockerhub data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
-	err := internal.GetObject(service.connection, BucketName, []byte(dockerHubKey), &dockerhub)
+	err := internal.GetObject(service.db, BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(dockerHubKey), dockerhub)
+	return internal.UpdateObject(service.db, BucketName, []byte(dockerHubKey), dockerhub)
 }

api/bolt/edgegroup/edgegroup.go

@@ -2,7 +2,7 @@ package edgegroup
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 	var groups = make([]portainer.EdgeGroup, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 	var group portainer.EdgeGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &group)
+	err := internal.GetObject(service.db, BucketName, identifier, &group)
 	if err != nil {
 		return nil, err
 	}
@@ -67,18 +67,18 @@ func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGrou
 // UpdateEdgeGroup updates an Edge group.
 func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, group)
+	return internal.UpdateObject(service.db, BucketName, identifier, group)
 }
 
 // DeleteEdgeGroup deletes an Edge group.
 func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateEdgeGroup assign an ID to a new Edge group and saves it.
 func (service *Service) CreateEdgeGroup(group *portainer.EdgeGroup) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/edgejob/edgejob.go

@@ -2,7 +2,7 @@ package edgejob
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 	var edgeJobs = make([]portainer.EdgeJob, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 	var edgeJob portainer.EdgeJob
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &edgeJob)
+	err := internal.GetObject(service.db, BucketName, identifier, &edgeJob)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 
 // CreateEdgeJob creates a new Edge job
 func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeJob.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
 // UpdateEdgeJob updates an Edge job by ID
 func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, edgeJob)
+	return internal.UpdateObject(service.db, BucketName, identifier, edgeJob)
 }
 
 // DeleteEdgeJob deletes an Edge job
 func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
-// GetNextIdentifier returns the next identifier for an environment(endpoint).
+// GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/edgestack/edgestack.go

@@ -2,7 +2,7 @@ package edgestack
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -13,18 +13,18 @@ const (
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +32,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 	var stacks = make([]portainer.EdgeStack, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -56,7 +56,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 	var stack portainer.EdgeStack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	err := internal.GetObject(service.db, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 
 // CreateEdgeStack assign an ID to a new Edge stack and saves it.
 func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		if edgeStack.ID == 0 {
@@ -86,16 +86,16 @@ func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
 // UpdateEdgeStack updates an Edge stack.
 func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, edgeStack)
+	return internal.UpdateObject(service.db, BucketName, identifier, edgeStack)
 }
 
 // DeleteEdgeStack deletes an Edge stack.
 func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
-// GetNextIdentifier returns the next identifier for an environment(endpoint).
+// GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/endpoint/endpoint.go

@@ -2,7 +2,7 @@ package endpoint
 
 import (
 	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 )
 
@@ -11,29 +11,29 @@ const (
 	BucketName = "endpoints"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
-// Endpoint returns an environment(endpoint) by ID.
+// Endpoint returns an endpoint by ID.
 func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
 	var endpoint portainer.Endpoint
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpoint)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -41,23 +41,23 @@ func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
 	return &endpoint, nil
 }
 
-// UpdateEndpoint updates an environment(endpoint).
+// UpdateEndpoint updates an endpoint.
 func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpoint)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpoint)
 }
 
-// DeleteEndpoint deletes an environment(endpoint).
+// DeleteEndpoint deletes an endpoint.
 func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
-// Endpoints return an array containing all the environments(endpoints).
+// Endpoints return an array containing all the endpoints.
 func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	var endpoints = make([]portainer.Endpoint, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -76,12 +76,12 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 	return endpoints, err
 }
 
-// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
+// CreateEndpoint assign an ID to a new endpoint and saves it.
 func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		// We manually manage sequences for environments(endpoints)
+		// We manually manage sequences for endpoints
 		err := bucket.SetSequence(uint64(endpoint.ID))
 		if err != nil {
 			return err
@@ -96,14 +96,14 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 	})
 }
 
-// GetNextIdentifier returns the next identifier for an environment(endpoint).
+// GetNextIdentifier returns the next identifier for an endpoint.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }
 
-// Synchronize creates, updates and deletes environments(endpoints) inside a single transaction.
+// Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		for _, endpoint := range toCreate {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,7 +1,7 @@
 package endpointgroup
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,29 +12,29 @@ const (
 	BucketName = "endpoint_groups"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
-// EndpointGroup returns an environment(endpoint) group by ID.
+// EndpointGroup returns an endpoint group by ID.
 func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
 	var endpointGroup portainer.EndpointGroup
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpointGroup)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpointGroup)
 	if err != nil {
 		return nil, err
 	}
@@ -42,23 +42,23 @@ func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.
 	return &endpointGroup, nil
 }
 
-// UpdateEndpointGroup updates an environment(endpoint) group.
+// UpdateEndpointGroup updates an endpoint group.
 func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpointGroup)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpointGroup)
 }
 
-// DeleteEndpointGroup deletes an environment(endpoint) group.
+// DeleteEndpointGroup deletes an endpoint group.
 func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
-// EndpointGroups return an array containing all the environment(endpoint) groups.
+// EndpointGroups return an array containing all the endpoint groups.
 func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	var endpointGroups = make([]portainer.EndpointGroup, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -77,9 +77,9 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 	return endpointGroups, err
 }
 
-// CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
+// CreateEndpointGroup assign an ID to a new endpoint group and saves it.
 func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/bolt/endpointrelation/endpointrelation.go

@@ -11,29 +11,29 @@ const (
 	BucketName = "endpoint_relations"
 )
 
-// Service represents a service for managing environment(endpoint) relation data.
+// Service represents a service for managing endpoint relation data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
-// EndpointRelation returns a Environment(Endpoint) relation object by EndpointID
+// EndpointRelation returns a Endpoint relation object by EndpointID
 func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
 	var endpointRelation portainer.EndpointRelation
 	identifier := internal.Itob(int(endpointID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &endpointRelation)
+	err := internal.GetObject(service.db, BucketName, identifier, &endpointRelation)
 	if err != nil {
 		return nil, err
 	}
@@ -43,7 +43,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
 
 // CreateEndpointRelation saves endpointRelation
 func (service *Service) CreateEndpointRelation(endpointRelation *portainer.EndpointRelation) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(endpointRelation)
@@ -55,14 +55,14 @@ func (service *Service) CreateEndpointRelation(endpointRelation *portainer.Endpo
 	})
 }
 
-// UpdateEndpointRelation updates an Environment(Endpoint) relation object
+// UpdateEndpointRelation updates an Endpoint relation object
 func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, endpointRelation)
+	return internal.UpdateObject(service.db, BucketName, identifier, endpointRelation)
 }
 
-// DeleteEndpointRelation deletes an Environment(Endpoint) relation object
+// DeleteEndpointRelation deletes an Endpoint relation object
 func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
 	identifier := internal.Itob(int(EndpointID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/errors/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 var (
 	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documention to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
 )

api/bolt/extension/extension.go

@@ -1,7 +1,7 @@
 package extension
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "extension"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 	var extension portainer.Extension
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &extension)
+	err := internal.GetObject(service.db, BucketName, identifier, &extension)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data, err := internal.MarshalObject(extension)
@@ -82,5 +82,5 @@ func (service *Service) Persist(extension *portainer.Extension) error {
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/helmuserrepository/helmuserrepository.go

@@ -1,73 +0,0 @@
-package helmuserrepository
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "helm_user_repository"
-)
-
-// Service represents a service for managing environment(endpoint) data.
-type Service struct {
-	connection *internal.DbConnection
-}
-
-// NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
-func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
-	var result = make([]portainer.HelmUserRepository, 0)
-
-	err := service.connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var record portainer.HelmUserRepository
-			err := internal.UnmarshalObject(v, &record)
-			if err != nil {
-				return err
-			}
-
-			if record.UserID == userID {
-				result = append(result, record)
-			}
-		}
-
-		return nil
-	})
-
-	return result, err
-}
-
-// CreateHelmUserRepository creates a new HelmUserRepository object.
-func (service *Service) CreateHelmUserRepository(record *portainer.HelmUserRepository) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		id, _ := bucket.NextSequence()
-		record.ID = portainer.HelmUserRepositoryID(id)
-
-		data, err := internal.MarshalObject(record)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(internal.Itob(int(record.ID)), data)
-	})
-}

api/bolt/init.go

@@ -44,10 +44,7 @@ func (store *Store) Init() error {
 
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
 			TemplatesURL:             portainer.DefaultTemplatesURL,
-			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
-			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,
-			KubectlShellImage:        portainer.DefaultKubectlShellImage,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)
@@ -58,20 +55,20 @@ func (store *Store) Init() error {
 		return err
 	}
 
-	_, err = store.SSLSettings().Settings()
-	if err != nil {
-		if err != errors.ErrObjectNotFound {
-			return err
+	_, err = store.DockerHubService.DockerHub()
+	if err == errors.ErrObjectNotFound {
+		defaultDockerHub := &portainer.DockerHub{
+			Authentication: false,
+			Username:       "",
+			Password:       "",
 		}
 
-		defaultSSLSettings := &portainer.SSLSettings{
-			HTTPEnabled: true,
-		}
-
-		err = store.SSLSettings().UpdateSettings(defaultSSLSettings)
+		err := store.DockerHubService.UpdateDockerHub(defaultDockerHub)
 		if err != nil {
 			return err
 		}
+	} else if err != nil {
+		return err
 	}
 
 	groups, err := store.EndpointGroupService.EndpointGroups()
@@ -82,7 +79,7 @@ func (store *Store) Init() error {
 	if len(groups) == 0 {
 		unassignedGroup := &portainer.EndpointGroup{
 			Name:               "Unassigned",
-			Description:        "Unassigned environments",
+			Description:        "Unassigned endpoints",
 			Labels:             []portainer.Pair{},
 			UserAccessPolicies: portainer.UserAccessPolicies{},
 			TeamAccessPolicies: portainer.TeamAccessPolicies{},

api/bolt/internal/db.go

@@ -7,10 +7,6 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
-type DbConnection struct {
-	*bolt.DB
-}
-
 // Itob returns an 8-byte big endian representation of v.
 // This function is typically used for encoding integer IDs to byte slices
 // so that they can be used as BoltDB keys.
@@ -21,8 +17,8 @@ func Itob(v int) []byte {
 }
 
 // CreateBucket is a generic function used to create a bucket inside a bolt database.
-func CreateBucket(connection *DbConnection, bucketName string) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func CreateBucket(db *bolt.DB, bucketName string) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
 		if err != nil {
 			return err
@@ -32,10 +28,10 @@ func CreateBucket(connection *DbConnection, bucketName string) error {
 }
 
 // GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
-func GetObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
 	var data []byte
 
-	err := connection.View(func(tx *bolt.Tx) error {
+	err := db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		value := bucket.Get(key)
@@ -56,8 +52,8 @@ func GetObject(connection *DbConnection, bucketName string, key []byte, object i
 }
 
 // UpdateObject is a generic function used to update an object inside a bolt database.
-func UpdateObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		data, err := MarshalObject(object)
@@ -75,18 +71,18 @@ func UpdateObject(connection *DbConnection, bucketName string, key []byte, objec
 }
 
 // DeleteObject is a generic function used to delete an object inside a bolt database.
-func DeleteObject(connection *DbConnection, bucketName string, key []byte) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+func DeleteObject(db *bolt.DB, bucketName string, key []byte) error {
+	return db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
 }
 
 // GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
-func GetNextIdentifier(connection *DbConnection, bucketName string) int {
+func GetNextIdentifier(db *bolt.DB, bucketName string) int {
 	var identifier int
 
-	connection.Update(func(tx *bolt.Tx) error {
+	db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {

api/bolt/internal/json.go

@@ -17,7 +17,7 @@ func UnmarshalObject(data []byte, object interface{}) error {
 }
 
 // UnmarshalObjectWithJsoniter decodes an object from binary data
-// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// using the jsoniter library. It is mainly used to accelerate endpoint
 // decoding at the moment.
 func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
 	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary

api/bolt/log/log.go

@@ -1,41 +0,0 @@
-package log
-
-import (
-	"fmt"
-	"log"
-)
-
-const (
-	INFO  = "INFO"
-	ERROR = "ERROR"
-	DEBUG = "DEBUG"
-	FATAL = "FATAL"
-)
-
-type ScopedLog struct {
-	scope string
-}
-
-func NewScopedLog(scope string) *ScopedLog {
-	return &ScopedLog{scope: scope}
-}
-
-func (slog *ScopedLog) print(kind string, message string) {
-	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
-}
-
-func (slog *ScopedLog) Debug(message string) {
-	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Info(message string) {
-	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Error(message string, err error) {
-	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
-}
-
-func (slog *ScopedLog) NotImplemented(method string) {
-	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
-}

api/bolt/log/log.test.go

@@ -1 +0,0 @@
-package log

api/bolt/migrate_data.go

@@ -1,149 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-
-	"github.com/portainer/portainer/api/cli"
-
-	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-	plog "github.com/portainer/portainer/api/bolt/log"
-	"github.com/portainer/portainer/api/bolt/migrator"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
-
-var migrateLog = plog.NewScopedLog("bolt, migrate")
-
-// FailSafeMigrate backup and restore DB if migration fail
-func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) (err error) {
-	defer func() {
-		if e := recover(); e != nil {
-			store.Rollback(true)
-			err = fmt.Errorf("%v", e)
-		}
-	}()
-
-	// !Important: we must use a named return value in the function definition and not a local
-	// !variable referenced from the closure or else the return value will be incorrectly set
-	return migrator.Migrate()
-}
-
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-// if force is true, then migrate regardless.
-func (store *Store) MigrateData(force bool) error {
-	if store.isNew && !force {
-		return store.VersionService.StoreDBVersion(portainer.DBVersion)
-	}
-
-	migrator, err := store.newMigrator()
-	if err != nil {
-		return err
-	}
-
-	// backup db file before upgrading DB to support rollback
-	isUpdating, err := store.VersionService.IsUpdating()
-	if err != nil && err != errors.ErrObjectNotFound {
-		return err
-	}
-
-	if !isUpdating && migrator.Version() != portainer.DBVersion {
-		err = store.backupVersion(migrator)
-		if err != nil {
-			return werrors.Wrapf(err, "failed to backup database")
-		}
-	}
-
-	if migrator.Version() < portainer.DBVersion {
-		migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
-		err = store.FailSafeMigrate(migrator)
-		if err != nil {
-			migrateLog.Error("An error occurred during database migration", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (store *Store) newMigrator() (*migrator.Migrator, error) {
-	version, err := store.version()
-	if err != nil {
-		return nil, err
-	}
-
-	migratorParams := &migrator.Parameters{
-		DB:                      store.connection.DB,
-		DatabaseVersion:         version,
-		EndpointGroupService:    store.EndpointGroupService,
-		EndpointService:         store.EndpointService,
-		EndpointRelationService: store.EndpointRelationService,
-		ExtensionService:        store.ExtensionService,
-		RegistryService:         store.RegistryService,
-		ResourceControlService:  store.ResourceControlService,
-		RoleService:             store.RoleService,
-		ScheduleService:         store.ScheduleService,
-		SettingsService:         store.SettingsService,
-		StackService:            store.StackService,
-		TagService:              store.TagService,
-		TeamMembershipService:   store.TeamMembershipService,
-		UserService:             store.UserService,
-		VersionService:          store.VersionService,
-		FileService:             store.fileService,
-		DockerhubService:        store.DockerHubService,
-		AuthorizationService:    authorization.NewService(store),
-	}
-	return migrator.NewMigrator(migratorParams), nil
-}
-
-// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// - db backup prior to version upgrade
-// - db rollback
-func getBackupRestoreOptions(store *Store) *BackupOptions {
-	return &BackupOptions{
-		BackupDir:      store.commonBackupDir(),
-		BackupFileName: beforePortainerVersionUpgradeBackup,
-	}
-}
-
-// backupVersion will backup the database or panic if any errors occur
-func (store *Store) backupVersion(migrator *migrator.Migrator) error {
-	migrateLog.Info("Backing up database prior to version upgrade...")
-
-	options := getBackupRestoreOptions(store)
-
-	_, err := store.BackupWithOptions(options)
-	if err != nil {
-		migrateLog.Error("An error occurred during database backup", err)
-		removalErr := store.RemoveWithOptions(options)
-		if removalErr != nil {
-			migrateLog.Error("An error occurred during store removal prior to backup", err)
-		}
-		return err
-	}
-
-	return nil
-}
-
-// Rollback to a pre-upgrade backup copy/snapshot of portainer.db
-func (store *Store) Rollback(force bool) error {
-
-	if !force {
-		confirmed, err := cli.Confirm("Are you sure you want to rollback your database to the previous backup?")
-		if err != nil || !confirmed {
-			return err
-		}
-	}
-
-	options := getBackupRestoreOptions(store)
-
-	err := store.RestoreWithOptions(options)
-	if err != nil {
-		return err
-	}
-
-	return store.Close()
-}

api/bolt/migrate_data_test.go

@@ -1,172 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"log"
-	"strings"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// testVersion is a helper which tests current store version against wanted version
-func testVersion(store *Store, versionWant int, t *testing.T) {
-	if v, _ := store.version(); v != versionWant {
-		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
-	}
-}
-
-func TestMigrateData(t *testing.T) {
-	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		if !store.IsNew() {
-			t.Error("Expect a new DB")
-		}
-
-		store.MigrateData(false)
-
-		testVersion(store, portainer.DBVersion, t)
-		store.Close()
-
-		store.Open()
-		if store.IsNew() {
-			t.Error("Expect store to NOT be new DB")
-		}
-	})
-
-	tests := []struct {
-		version         int
-		expectedVersion int
-	}{
-		{version: 2, expectedVersion: portainer.DBVersion},
-		{version: 21, expectedVersion: portainer.DBVersion},
-	}
-	for _, tc := range tests {
-		store, teardown := MustNewTestStore(true)
-		defer teardown()
-
-		// Setup data
-		store.VersionService.StoreDBVersion(tc.version)
-
-		// Required roles by migrations 22.2
-		store.RoleService.CreateRole(&portainer.Role{ID: 1})
-		store.RoleService.CreateRole(&portainer.Role{ID: 2})
-		store.RoleService.CreateRole(&portainer.Role{ID: 3})
-		store.RoleService.CreateRole(&portainer.Role{ID: 4})
-
-		t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
-			store.MigrateData(true)
-			testVersion(store, tc.expectedVersion, t)
-		})
-
-		t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) {
-			store.Rollback(true)
-			store.Open()
-			testVersion(store, tc.version, t)
-		})
-	}
-
-	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		version := 2
-		store.VersionService.StoreDBVersion(version)
-
-		store.MigrateData(true)
-
-		testVersion(store, version, t)
-	})
-
-	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-		store.VersionService.StoreDBVersion(0)
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if !isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
-		}
-	})
-
-	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		store.VersionService.StoreIsUpdating(true)
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
-
-	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
-
-}
-
-func Test_getBackupRestoreOptions(t *testing.T) {
-	store, teardown := MustNewTestStore(false)
-	defer teardown()
-
-	options := getBackupRestoreOptions(store)
-
-	wantDir := store.commonBackupDir()
-	if !strings.HasSuffix(options.BackupDir, wantDir) {
-		log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
-	}
-
-	wantFilename := "portainer.db.bak"
-	if options.BackupFileName != wantFilename {
-		log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
-	}
-}
-
-func TestRollback(t *testing.T) {
-	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := 21
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-		store.VersionService.StoreDBVersion(version)
-
-		_, err := store.BackupWithOptions(getBackupRestoreOptions(store))
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Change the current edition
-		err = store.VersionService.StoreDBVersion(version + 10)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		err = store.Rollback(true)
-		if err != nil {
-			t.Logf("Rollback failed: %s", err)
-			t.Fail()
-			return
-		}
-
-		store.Open()
-		testVersion(store, version, t)
-	})
-}

api/bolt/migrator/migrate_ce.go

@@ -1,327 +0,0 @@
-package migrator
-
-import (
-	"fmt"
-
-	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-)
-
-func migrationError(err error, context string) error {
-	return werrors.Wrap(err, "failed in "+context)
-}
-
-// Migrate checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) Migrate() error {
-	// set DB to updating status
-	err := m.versionService.StoreIsUpdating(true)
-	if err != nil {
-		return migrationError(err, "StoreIsUpdating")
-	}
-
-	// Portainer < 1.12
-	if m.currentDBVersion < 1 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return migrationError(err, "updateAdminUserToDBVersion1")
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.currentDBVersion < 2 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion2")
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion2")
-		}
-	}
-
-	// Portainer 1.13.x
-	if m.currentDBVersion < 3 {
-		err := m.updateSettingsToDBVersion3()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion3")
-		}
-	}
-
-	// Portainer 1.14.0
-	if m.currentDBVersion < 4 {
-		err := m.updateEndpointsToDBVersion4()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion4")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1235
-	if m.currentDBVersion < 5 {
-		err := m.updateSettingsToVersion5()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion5")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1236
-	if m.currentDBVersion < 6 {
-		err := m.updateSettingsToVersion6()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion6")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1449
-	if m.currentDBVersion < 7 {
-		err := m.updateSettingsToVersion7()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion7")
-		}
-	}
-
-	if m.currentDBVersion < 8 {
-		err := m.updateEndpointsToVersion8()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion8")
-		}
-	}
-
-	// https: //github.com/portainer/portainer/issues/1396
-	if m.currentDBVersion < 9 {
-		err := m.updateEndpointsToVersion9()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion9")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/461
-	if m.currentDBVersion < 10 {
-		err := m.updateEndpointsToVersion10()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion10")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1906
-	if m.currentDBVersion < 11 {
-		err := m.updateEndpointsToVersion11()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion11")
-		}
-	}
-
-	// Portainer 1.18.0
-	if m.currentDBVersion < 12 {
-		err := m.updateEndpointsToVersion12()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion12")
-		}
-
-		err = m.updateEndpointGroupsToVersion12()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToVersion12")
-		}
-
-		err = m.updateStacksToVersion12()
-		if err != nil {
-			return migrationError(err, "updateStacksToVersion12")
-		}
-	}
-
-	// Portainer 1.19.0
-	if m.currentDBVersion < 13 {
-		err := m.updateSettingsToVersion13()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion13")
-		}
-	}
-
-	// Portainer 1.19.2
-	if m.currentDBVersion < 14 {
-		err := m.updateResourceControlsToDBVersion14()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion14")
-		}
-	}
-
-	// Portainer 1.20.0
-	if m.currentDBVersion < 15 {
-		err := m.updateSettingsToDBVersion15()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion15")
-		}
-
-		err = m.updateTemplatesToVersion15()
-		if err != nil {
-			return migrationError(err, "updateTemplatesToVersion15")
-		}
-	}
-
-	if m.currentDBVersion < 16 {
-		err := m.updateSettingsToDBVersion16()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion16")
-		}
-	}
-
-	// Portainer 1.20.1
-	if m.currentDBVersion < 17 {
-		err := m.updateExtensionsToDBVersion17()
-		if err != nil {
-			return migrationError(err, "updateExtensionsToDBVersion17")
-		}
-	}
-
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion18")
-		}
-
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion18")
-		}
-
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToDBVersion18")
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateRegistriesToDBVersion18")
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion19")
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion20")
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion20")
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSchedulesToDBVersion20")
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion22")
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateUsersAndRolesToDBVersion22")
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateTagsToDBVersion23")
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB24")
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB25")
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return migrationError(err, "updateStacksToDB24")
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		err := m.updateEndpointSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateEndpointSettingsToDB25")
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return migrationError(err, "updateStackResourceControlToDB27")
-		}
-	}
-
-	// Portainer 2.6.0
-	if m.currentDBVersion < 30 {
-		err := m.migrateDBVersionToDB30()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB30")
-		}
-	}
-
-	// Portainer 2.9.0
-	if m.currentDBVersion < 32 {
-		err := m.migrateDBVersionToDB32()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB32")
-		}
-	}
-
-	// Portainer 2.9.1
-	if m.currentDBVersion < 33 {
-		err := m.migrateDBVersionToDB33()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB33")
-		}
-	}
-
-	// Portainer 2.10
-	if m.currentDBVersion < 34 {
-		if err := m.migrateDBVersionToDB34(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB34")
-		}
-	}
-
-	err = m.versionService.StoreDBVersion(portainer.DBVersion)
-	if err != nil {
-		return migrationError(err, "StoreDBVersion")
-	}
-	migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
-
-	// reset DB updating status
-	return m.versionService.StoreIsUpdating(false)
-}

api/bolt/migrator/migrate_dbversion26.go

@@ -1,8 +1,11 @@
 package migrator
 
 import (
+	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
@@ -38,3 +41,38 @@ func (m *Migrator) updateStackResourceControlToDB27() error {
 
 	return nil
 }
+
+func (m *Migrator) updateRegistriesToDB27() error {
+	registries, err := m.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		for _, registry := range registries {
+			userIDs := []portainer.UserID{}
+			for id := range registry.UserAccessPolicies {
+				userIDs = append(userIDs, portainer.UserID(id))
+			}
+
+			teamIDs := []portainer.TeamID{}
+			for id := range registry.TeamAccessPolicies {
+				teamIDs = append(teamIDs, portainer.TeamID(id))
+			}
+
+			resourceControl := authorization.NewRestrictedResourceControl(
+				fmt.Sprintf("%d-%d", int(registry.ID), int(endpoint.ID)),
+				portainer.RegistryResourceControl,
+				userIDs,
+				teamIDs,
+			)
+			m.resourceControlService.CreateResourceControl(resourceControl)
+		}
+	}
+	return nil
+}

api/bolt/migrator/migrate_dbversion29.go

@@ -1,19 +0,0 @@
-package migrator
-
-func (m *Migrator) migrateDBVersionToDB30() error {
-	if err := m.migrateSettingsToDB30(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *Migrator) migrateSettingsToDB30() error {
-	legacySettings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-	legacySettings.OAuthSettings.SSO = false
-	legacySettings.OAuthSettings.LogoutURI = ""
-	return m.settingsService.UpdateSettings(legacySettings)
-}

api/bolt/migrator/migrate_dbversion29_test.go

@@ -1,95 +0,0 @@
-package migrator
-
-import (
-	"os"
-	"path"
-	"testing"
-	"time"
-
-	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/settings"
-)
-
-var (
-	testingDBStorePath string
-	testingDBFileName  string
-	dummyLogoURL       string
-	dbConn             *bolt.DB
-	settingsService    *settings.Service
-)
-
-// initTestingDBConn creates a raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
-	databasePath := path.Join(storePath, fileName)
-	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return nil, err
-	}
-	return dbConn, nil
-}
-
-// initTestingDBConn creates a settings service with raw bolt DB connection
-// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
-func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
-	internalDBConn := &internal.DbConnection{
-		DB: dbConn,
-	}
-	settingsService, err := settings.NewService(internalDBConn)
-	if err != nil {
-		return nil, err
-	}
-	//insert a obj
-	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
-		return nil, err
-	}
-	return settingsService, nil
-}
-
-func setup() error {
-	testingDBStorePath, _ = os.Getwd()
-	testingDBFileName = "portainer-ee-mig-30.db"
-	dummyLogoURL = "example.com"
-	var err error
-	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
-	if err != nil {
-		return err
-	}
-	dummySettingsObj := map[string]interface{}{
-		"LogoURL": dummyLogoURL,
-	}
-	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func TestMigrateSettings(t *testing.T) {
-	if err := setup(); err != nil {
-		t.Errorf("failed to complete testing setups, err: %v", err)
-	}
-	defer dbConn.Close()
-	defer os.Remove(testingDBFileName)
-	m := &Migrator{
-		db:              dbConn,
-		settingsService: settingsService,
-	}
-	if err := m.migrateSettingsToDB30(); err != nil {
-		t.Errorf("failed to update settings: %v", err)
-	}
-	updatedSettings, err := m.settingsService.Settings()
-	if err != nil {
-		t.Errorf("failed to retrieve the updated settings: %v", err)
-	}
-	if updatedSettings.LogoURL != dummyLogoURL {
-		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
-	}
-	if updatedSettings.OAuthSettings.SSO != false {
-		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
-	}
-	if updatedSettings.OAuthSettings.LogoutURI != "" {
-		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
-	}
-}

api/bolt/migrator/migrate_dbversion31.go

@@ -1,251 +0,0 @@
-package migrator
-
-import (
-	"fmt"
-	"log"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/internal/endpointutils"
-	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
-)
-
-func (m *Migrator) migrateDBVersionToDB32() error {
-	err := m.updateRegistriesToDB32()
-	if err != nil {
-		return err
-	}
-
-	err = m.updateDockerhubToDB32()
-	if err != nil {
-		return err
-	}
-
-	if err := m.updateVolumeResourceControlToDB32(); err != nil {
-		return err
-	}
-
-	if err := m.kubeconfigExpiryToDB32(); err != nil {
-		return err
-	}
-
-	if err := m.helmRepositoryURLToDB32(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *Migrator) updateRegistriesToDB32() error {
-	registries, err := m.registryService.Registries()
-	if err != nil {
-		return err
-	}
-
-	endpoints, err := m.endpointService.Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for _, registry := range registries {
-
-		registry.RegistryAccesses = portainer.RegistryAccesses{}
-
-		for _, endpoint := range endpoints {
-
-			filteredUserAccessPolicies := portainer.UserAccessPolicies{}
-			for userId, registryPolicy := range registry.UserAccessPolicies {
-				if _, found := endpoint.UserAccessPolicies[userId]; found {
-					filteredUserAccessPolicies[userId] = registryPolicy
-				}
-			}
-
-			filteredTeamAccessPolicies := portainer.TeamAccessPolicies{}
-			for teamId, registryPolicy := range registry.TeamAccessPolicies {
-				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
-					filteredTeamAccessPolicies[teamId] = registryPolicy
-				}
-			}
-
-			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
-				UserAccessPolicies: filteredUserAccessPolicies,
-				TeamAccessPolicies: filteredTeamAccessPolicies,
-				Namespaces:         []string{},
-			}
-		}
-		m.registryService.UpdateRegistry(registry.ID, &registry)
-	}
-	return nil
-}
-
-func (m *Migrator) updateDockerhubToDB32() error {
-	dockerhub, err := m.dockerhubService.DockerHub()
-	if err == errors.ErrObjectNotFound {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	if !dockerhub.Authentication {
-		return nil
-	}
-
-	registry := &portainer.Registry{
-		Type:             portainer.DockerHubRegistry,
-		Name:             "Dockerhub (authenticated - migrated)",
-		URL:              "docker.io",
-		Authentication:   true,
-		Username:         dockerhub.Username,
-		Password:         dockerhub.Password,
-		RegistryAccesses: portainer.RegistryAccesses{},
-	}
-
-	endpoints, err := m.endpointService.Endpoints()
-	if err != nil {
-		return err
-	}
-
-	for _, endpoint := range endpoints {
-
-		if endpoint.Type != portainer.KubernetesLocalEnvironment &&
-			endpoint.Type != portainer.AgentOnKubernetesEnvironment &&
-			endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
-
-			userAccessPolicies := portainer.UserAccessPolicies{}
-			for userId := range endpoint.UserAccessPolicies {
-				if _, found := endpoint.UserAccessPolicies[userId]; found {
-					userAccessPolicies[userId] = portainer.AccessPolicy{
-						RoleID: 0,
-					}
-				}
-			}
-
-			teamAccessPolicies := portainer.TeamAccessPolicies{}
-			for teamId := range endpoint.TeamAccessPolicies {
-				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
-					teamAccessPolicies[teamId] = portainer.AccessPolicy{
-						RoleID: 0,
-					}
-				}
-			}
-
-			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
-				UserAccessPolicies: userAccessPolicies,
-				TeamAccessPolicies: teamAccessPolicies,
-				Namespaces:         []string{},
-			}
-		}
-	}
-
-	return m.registryService.CreateRegistry(registry)
-}
-
-func (m *Migrator) updateVolumeResourceControlToDB32() error {
-	endpoints, err := m.endpointService.Endpoints()
-	if err != nil {
-		return fmt.Errorf("failed fetching environments: %w", err)
-	}
-
-	resourceControls, err := m.resourceControlService.ResourceControls()
-	if err != nil {
-		return fmt.Errorf("failed fetching resource controls: %w", err)
-	}
-
-	toUpdate := map[portainer.ResourceControlID]string{}
-	volumeResourceControls := map[string]*portainer.ResourceControl{}
-
-	for i := range resourceControls {
-		resourceControl := resourceControls[i]
-		if resourceControl.Type == portainer.VolumeResourceControl {
-			volumeResourceControls[resourceControl.ResourceID] = &resourceControl
-		}
-	}
-
-	for _, endpoint := range endpoints {
-		if !endpointutils.IsDockerEndpoint(&endpoint) {
-			continue
-		}
-
-		totalSnapshots := len(endpoint.Snapshots)
-		if totalSnapshots == 0 {
-			log.Println("[DEBUG] [volume migration] [message: no snapshot found]")
-			continue
-		}
-
-		snapshot := endpoint.Snapshots[totalSnapshots-1]
-
-		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
-		if err != nil {
-			log.Printf("[WARN] [bolt,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
-			continue
-		}
-
-		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
-			if volumesData["Volumes"] == nil {
-				log.Println("[DEBUG] [volume migration] [message: no volume data found]")
-				continue
-			}
-
-			findResourcesToUpdateForDB32(endpointDockerID, volumesData, toUpdate, volumeResourceControls)
-		}
-	}
-
-	for _, resourceControl := range volumeResourceControls {
-		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
-			resourceControl.ResourceID = newResourceID
-			err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
-			if err != nil {
-				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
-			}
-
-		} else {
-			err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
-			if err != nil {
-				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
-			}
-			log.Printf("[DEBUG] [volume migration] [message: legacy resource control(%s) has been deleted]", resourceControl.ResourceID)
-		}
-	}
-
-	return nil
-}
-
-func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interface{}, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
-	volumes := volumesData["Volumes"].([]interface{})
-	for _, volumeMeta := range volumes {
-		volume := volumeMeta.(map[string]interface{})
-		volumeName, nameExist := volume["Name"].(string)
-		if !nameExist {
-			continue
-		}
-		createTime, createTimeExist := volume["CreatedAt"].(string)
-		if !createTimeExist {
-			continue
-		}
-
-		oldResourceID := fmt.Sprintf("%s%s", volumeName, createTime)
-		resourceControl, ok := volumeResourceControls[oldResourceID]
-
-		if ok {
-			toUpdate[resourceControl.ID] = fmt.Sprintf("%s_%s", volumeName, dockerID)
-		}
-	}
-}
-
-func (m *Migrator) kubeconfigExpiryToDB32() error {
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-	settings.KubeconfigExpiry = portainer.DefaultKubeconfigExpiry
-	return m.settingsService.UpdateSettings(settings)
-}
-
-func (m *Migrator) helmRepositoryURLToDB32() error {
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-	settings.HelmRepositoryURL = portainer.DefaultHelmRepositoryURL
-	return m.settingsService.UpdateSettings(settings)
-}

api/bolt/migrator/migrate_dbversion32.go

@@ -1,21 +0,0 @@
-package migrator
-
-import portainer "github.com/portainer/portainer/api"
-
-func (m *Migrator) migrateDBVersionToDB33() error {
-	if err := m.migrateSettingsToDB33(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *Migrator) migrateSettingsToDB33() error {
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
-	return m.settingsService.UpdateSettings(settings)
-}

api/bolt/migrator/migrate_dbversion33.go

@@ -1,32 +0,0 @@
-package migrator
-
-import (
-	portainer "github.com/portainer/portainer/api"
-)
-
-func (m *Migrator) migrateDBVersionToDB34() error {
-	err := migrateStackEntryPoint(m.stackService)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func migrateStackEntryPoint(stackService portainer.StackService) error {
-	stacks, err := stackService.Stacks()
-	if err != nil {
-		return err
-	}
-	for i := range stacks {
-		stack := &stacks[i]
-		if stack.GitConfig == nil {
-			continue
-		}
-		stack.GitConfig.ConfigFilePath = stack.EntryPoint
-		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
-			return err
-		}
-	}
-	return nil
-}

api/bolt/migrator/migrate_dbversion33_test.go

@@ -1,51 +0,0 @@
-package migrator
-
-import (
-	"path"
-	"testing"
-	"time"
-
-	"github.com/boltdb/bolt"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/stack"
-	gittypes "github.com/portainer/portainer/api/git/types"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMigrateStackEntryPoint(t *testing.T) {
-	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-34.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
-	assert.NoError(t, err, "failed to init testing DB connection")
-	defer dbConn.Close()
-
-	stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn})
-	assert.NoError(t, err, "failed to init testing Stack service")
-
-	stacks := []*portainer.Stack{
-		{
-			ID:         1,
-			EntryPoint: "dir/sub/compose.yml",
-		},
-		{
-			ID:         2,
-			EntryPoint: "dir/sub/compose.yml",
-			GitConfig:  &gittypes.RepoConfig{},
-		},
-	}
-
-	for _, s := range stacks {
-		err := stackService.CreateStack(s)
-		assert.NoError(t, err, "failed to create stack")
-	}
-
-	err = migrateStackEntryPoint(stackService)
-	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
-
-	s, err := stackService.Stack(1)
-	assert.NoError(t, err)
-	assert.Nil(t, s.GitConfig, "first stack should not have git config")
-
-	s, err = stackService.Stack(2)
-	assert.NoError(t, err)
-	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")
-}

api/bolt/migrator/migrator.go

@@ -3,12 +3,10 @@ package migrator
 import (
 	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
 	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
-	plog "github.com/portainer/portainer/api/bolt/log"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -22,14 +20,11 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
-var migrateLog = plog.NewScopedLog("bolt, migrate")
-
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		db               *bolt.DB
-		currentDBVersion int
-
+		currentDBVersion        int
+		db                      *bolt.DB
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -46,7 +41,6 @@ type (
 		versionService          *version.Service
 		fileService             portainer.FileService
 		authorizationService    *authorization.Service
-		dockerhubService        *dockerhub.Service
 	}
 
 	// Parameters represents the required parameters to create a new Migrator instance.
@@ -69,7 +63,6 @@ type (
 		VersionService          *version.Service
 		FileService             portainer.FileService
 		AuthorizationService    *authorization.Service
-		DockerhubService        *dockerhub.Service
 	}
 )
 
@@ -94,11 +87,281 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		versionService:          parameters.VersionService,
 		fileService:             parameters.FileService,
 		authorizationService:    parameters.AuthorizationService,
-		dockerhubService:        parameters.DockerhubService,
 	}
 }
 
-// Version exposes version of database
-func (migrator *Migrator) Version() int {
-	return migrator.currentDBVersion
+// Migrate checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) Migrate() error {
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return err
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.23.0
+	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+	if m.currentDBVersion < 22 {
+		err := m.updateResourceControlsToDBVersion22()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateUsersAndRolesToDBVersion22()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.0
+	if m.currentDBVersion < 23 {
+		err := m.updateTagsToDBVersion23()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.1
+	if m.currentDBVersion < 24 {
+		err := m.updateSettingsToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.0.0
+	if m.currentDBVersion < 25 {
+		err := m.updateSettingsToDB25()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.1.0
+	if m.currentDBVersion < 26 {
+		err := m.updateEndpointSettingsToDB25()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.2.0
+	if m.currentDBVersion < 27 {
+		err := m.updateStackResourceControlToDB27()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDB27()
+		if err != nil {
+			return err
+		}
+	}
+
+	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/registry/registry.go

@@ -1,7 +1,7 @@
 package registry
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "registries"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 	var registry portainer.Registry
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &registry)
+	err := internal.GetObject(service.db, BucketName, identifier, &registry)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry,
 func (service *Service) Registries() ([]portainer.Registry, error) {
 	var registries = make([]portainer.Registry, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 
 // CreateRegistry creates a new registry.
 func (service *Service) CreateRegistry(registry *portainer.Registry) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateRegistry(registry *portainer.Registry) error {
 // UpdateRegistry updates an registry.
 func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, registry)
+	return internal.UpdateObject(service.db, BucketName, identifier, registry)
 }
 
 // DeleteRegistry deletes an registry.
 func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,7 +1,7 @@
 package resourcecontrol
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "resource_control"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 	var resourceControl portainer.ResourceControl
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &resourceControl)
+	err := internal.GetObject(service.db, BucketName, identifier, &resourceControl)
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +48,7 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -82,7 +82,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 
 // CreateResourceControl creates a new ResourceControl object
 func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -121,11 +121,11 @@ func (service *Service) CreateResourceControl(resourceControl *portainer.Resourc
 // UpdateResourceControl saves a ResourceControl object.
 func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, resourceControl)
+	return internal.UpdateObject(service.db, BucketName, identifier, resourceControl)
 }
 
 // DeleteResourceControl deletes a ResourceControl object by ID
 func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/role/role.go

@@ -1,7 +1,7 @@
 package role
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "roles"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &set)
+	err := internal.GetObject(service.db, BucketName, identifier, &set)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 func (service *Service) Roles() ([]portainer.Role, error) {
 	var sets = make([]portainer.Role, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -67,7 +67,7 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 
 // CreateRole creates a new Role.
 func (service *Service) CreateRole(role *portainer.Role) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,5 +85,5 @@ func (service *Service) CreateRole(role *portainer.Role) error {
 // UpdateRole updates a role.
 func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, role)
+	return internal.UpdateObject(service.db, BucketName, identifier, role)
 }

api/bolt/schedule/schedule.go

@@ -1,7 +1,7 @@
 package schedule
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -14,18 +14,18 @@ const (
 
 // Service represents a service for managing schedule data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 	var schedule portainer.Schedule
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &schedule)
+	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
 	if err != nil {
 		return nil, err
 	}
@@ -45,20 +45,20 @@ func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule,
 // UpdateSchedule updates a schedule.
 func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, schedule)
+	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
 func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
 func (service *Service) Schedules() ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -82,7 +82,7 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
 	var schedules = make([]portainer.Schedule, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -105,7 +105,7 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 
 // CreateSchedule assign an ID to a new schedule and saves it.
 func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for schedules
@@ -125,5 +125,5 @@ func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
 
 // GetNextIdentifier returns the next identifier for a schedule.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }

api/bolt/services.go

@@ -1,282 +0,0 @@
-package bolt
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/customtemplate"
-	"github.com/portainer/portainer/api/bolt/dockerhub"
-	"github.com/portainer/portainer/api/bolt/edgegroup"
-	"github.com/portainer/portainer/api/bolt/edgejob"
-	"github.com/portainer/portainer/api/bolt/edgestack"
-	"github.com/portainer/portainer/api/bolt/endpoint"
-	"github.com/portainer/portainer/api/bolt/endpointgroup"
-	"github.com/portainer/portainer/api/bolt/endpointrelation"
-	"github.com/portainer/portainer/api/bolt/extension"
-	"github.com/portainer/portainer/api/bolt/helmuserrepository"
-	"github.com/portainer/portainer/api/bolt/registry"
-	"github.com/portainer/portainer/api/bolt/resourcecontrol"
-	"github.com/portainer/portainer/api/bolt/role"
-	"github.com/portainer/portainer/api/bolt/schedule"
-	"github.com/portainer/portainer/api/bolt/settings"
-	"github.com/portainer/portainer/api/bolt/ssl"
-	"github.com/portainer/portainer/api/bolt/stack"
-	"github.com/portainer/portainer/api/bolt/tag"
-	"github.com/portainer/portainer/api/bolt/team"
-	"github.com/portainer/portainer/api/bolt/teammembership"
-	"github.com/portainer/portainer/api/bolt/tunnelserver"
-	"github.com/portainer/portainer/api/bolt/user"
-	"github.com/portainer/portainer/api/bolt/version"
-	"github.com/portainer/portainer/api/bolt/webhook"
-)
-
-func (store *Store) initServices() error {
-	authorizationsetService, err := role.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.RoleService = authorizationsetService
-
-	customTemplateService, err := customtemplate.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.CustomTemplateService = customTemplateService
-
-	dockerhubService, err := dockerhub.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.DockerHubService = dockerhubService
-
-	edgeStackService, err := edgestack.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeStackService = edgeStackService
-
-	edgeGroupService, err := edgegroup.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeGroupService = edgeGroupService
-
-	edgeJobService, err := edgejob.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EdgeJobService = edgeJobService
-
-	endpointgroupService, err := endpointgroup.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointGroupService = endpointgroupService
-
-	endpointService, err := endpoint.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointService = endpointService
-
-	endpointRelationService, err := endpointrelation.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.EndpointRelationService = endpointRelationService
-
-	extensionService, err := extension.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ExtensionService = extensionService
-
-	helmUserRepositoryService, err := helmuserrepository.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.HelmUserRepositoryService = helmUserRepositoryService
-
-	registryService, err := registry.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.RegistryService = registryService
-
-	resourcecontrolService, err := resourcecontrol.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ResourceControlService = resourcecontrolService
-
-	settingsService, err := settings.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.SettingsService = settingsService
-
-	sslSettingsService, err := ssl.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.SSLSettingsService = sslSettingsService
-
-	stackService, err := stack.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.StackService = stackService
-
-	tagService, err := tag.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TagService = tagService
-
-	teammembershipService, err := teammembership.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TeamMembershipService = teammembershipService
-
-	teamService, err := team.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TeamService = teamService
-
-	tunnelServerService, err := tunnelserver.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.TunnelServerService = tunnelServerService
-
-	userService, err := user.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.UserService = userService
-
-	versionService, err := version.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.VersionService = versionService
-
-	webhookService, err := webhook.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.WebhookService = webhookService
-
-	scheduleService, err := schedule.NewService(store.connection)
-	if err != nil {
-		return err
-	}
-	store.ScheduleService = scheduleService
-
-	return nil
-}
-
-// CustomTemplate gives access to the CustomTemplate data management layer
-func (store *Store) CustomTemplate() portainer.CustomTemplateService {
-	return store.CustomTemplateService
-}
-
-// EdgeGroup gives access to the EdgeGroup data management layer
-func (store *Store) EdgeGroup() portainer.EdgeGroupService {
-	return store.EdgeGroupService
-}
-
-// EdgeJob gives access to the EdgeJob data management layer
-func (store *Store) EdgeJob() portainer.EdgeJobService {
-	return store.EdgeJobService
-}
-
-// EdgeStack gives access to the EdgeStack data management layer
-func (store *Store) EdgeStack() portainer.EdgeStackService {
-	return store.EdgeStackService
-}
-
-// Environment(Endpoint) gives access to the Environment(Endpoint) data management layer
-func (store *Store) Endpoint() portainer.EndpointService {
-	return store.EndpointService
-}
-
-// EndpointGroup gives access to the EndpointGroup data management layer
-func (store *Store) EndpointGroup() portainer.EndpointGroupService {
-	return store.EndpointGroupService
-}
-
-// EndpointRelation gives access to the EndpointRelation data management layer
-func (store *Store) EndpointRelation() portainer.EndpointRelationService {
-	return store.EndpointRelationService
-}
-
-// HelmUserRepository access the helm user repository settings
-func (store *Store) HelmUserRepository() portainer.HelmUserRepositoryService {
-	return store.HelmUserRepositoryService
-}
-
-// Registry gives access to the Registry data management layer
-func (store *Store) Registry() portainer.RegistryService {
-	return store.RegistryService
-}
-
-// ResourceControl gives access to the ResourceControl data management layer
-func (store *Store) ResourceControl() portainer.ResourceControlService {
-	return store.ResourceControlService
-}
-
-// Role gives access to the Role data management layer
-func (store *Store) Role() portainer.RoleService {
-	return store.RoleService
-}
-
-// Settings gives access to the Settings data management layer
-func (store *Store) Settings() portainer.SettingsService {
-	return store.SettingsService
-}
-
-// SSLSettings gives access to the SSL Settings data management layer
-func (store *Store) SSLSettings() portainer.SSLSettingsService {
-	return store.SSLSettingsService
-}
-
-// Stack gives access to the Stack data management layer
-func (store *Store) Stack() portainer.StackService {
-	return store.StackService
-}
-
-// Tag gives access to the Tag data management layer
-func (store *Store) Tag() portainer.TagService {
-	return store.TagService
-}
-
-// TeamMembership gives access to the TeamMembership data management layer
-func (store *Store) TeamMembership() portainer.TeamMembershipService {
-	return store.TeamMembershipService
-}
-
-// Team gives access to the Team data management layer
-func (store *Store) Team() portainer.TeamService {
-	return store.TeamService
-}
-
-// TunnelServer gives access to the TunnelServer data management layer
-func (store *Store) TunnelServer() portainer.TunnelServerService {
-	return store.TunnelServerService
-}
-
-// User gives access to the User data management layer
-func (store *Store) User() portainer.UserService {
-	return store.UserService
-}
-
-// Version gives access to the Version data management layer
-func (store *Store) Version() portainer.VersionService {
-	return store.VersionService
-}
-
-// Webhook gives access to the Webhook data management layer
-func (store *Store) Webhook() portainer.WebhookService {
-	return store.WebhookService
-}

api/bolt/settings/settings.go

@@ -1,8 +1,10 @@
 package settings
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -11,20 +13,20 @@ const (
 	settingsKey = "SETTINGS"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings
 
-	err := internal.GetObject(service.connection, BucketName, []byte(settingsKey), &settings)
+	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(settingsKey), settings)
+	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), settings)
 }

api/bolt/ssl/ssl.go

@@ -1,46 +0,0 @@
-package ssl
-
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-)
-
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "ssl"
-	key        = "SSL"
-)
-
-// Service represents a service for managing ssl data.
-type Service struct {
-	connection *internal.DbConnection
-}
-
-// NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		connection: connection,
-	}, nil
-}
-
-// Settings retrieve the ssl settings object.
-func (service *Service) Settings() (*portainer.SSLSettings, error) {
-	var settings portainer.SSLSettings
-
-	err := internal.GetObject(service.connection, BucketName, []byte(key), &settings)
-	if err != nil {
-		return nil, err
-	}
-
-	return &settings, nil
-}
-
-// UpdateSettings persists a SSLSettings object.
-func (service *Service) UpdateSettings(settings *portainer.SSLSettings) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(key), settings)
-}

api/bolt/stack/stack.go

@@ -1,14 +1,11 @@
 package stack
 
 import (
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
-	pkgerrors "github.com/pkg/errors"
 )
 
 const (
@@ -16,20 +13,20 @@ const (
 	BucketName = "stacks"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -38,7 +35,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 	var stack portainer.Stack
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	err := internal.GetObject(service.db, BucketName, identifier, &stack)
 	if err != nil {
 		return nil, err
 	}
@@ -50,7 +47,7 @@ func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
 func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 	var stack *portainer.Stack
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -81,7 +78,7 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 func (service *Service) Stacks() ([]portainer.Stack, error) {
 	var stacks = make([]portainer.Stack, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -102,12 +99,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 
 // GetNextIdentifier returns the next identifier for a stack.
 func (service *Service) GetNextIdentifier() int {
-	return internal.GetNextIdentifier(service.connection, BucketName)
+	return internal.GetNextIdentifier(service.db, BucketName)
 }
 
 // CreateStack creates a new stack.
 func (service *Service) CreateStack(stack *portainer.Stack) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		// We manually manage sequences for stacks
@@ -128,84 +125,11 @@ func (service *Service) CreateStack(stack *portainer.Stack) error {
 // UpdateStack updates a stack.
 func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, stack)
+	return internal.UpdateObject(service.db, BucketName, identifier, stack)
 }
 
 // DeleteStack deletes a stack.
 func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
-}
-
-// StackByWebhookID returns a pointer to a stack object by webhook ID.
-// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
-func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
-	if id == "" {
-		return nil, pkgerrors.New("webhook ID can't be empty string")
-	}
-	var stack portainer.Stack
-	found := false
-
-	err := service.connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-		cursor := bucket.Cursor()
-
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var t struct {
-				AutoUpdate *struct {
-					WebhookID string `json:"Webhook"`
-				} `json:"AutoUpdate"`
-			}
-
-			err := internal.UnmarshalObject(v, &t)
-			if err != nil {
-				return err
-			}
-
-			if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) {
-				found = true
-				err := internal.UnmarshalObject(v, &stack)
-				if err != nil {
-					return err
-				}
-				break
-			}
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-	if !found {
-		return nil, errors.ErrObjectNotFound
-	}
-
-	return &stack, nil
-}
-
-// RefreshableStacks returns stacks that are configured for a periodic update
-func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
-	stacks := make([]portainer.Stack, 0)
-	err := service.connection.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-		cursor := bucket.Cursor()
-
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			stack := portainer.Stack{}
-			err := internal.UnmarshalObject(v, &stack)
-			if err != nil {
-				return err
-			}
-
-			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
-				stacks = append(stacks, stack)
-			}
-		}
-
-		return nil
-	})
-
-	return stacks, err
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/stack/tests/stack_test.go

@@ -1,105 +0,0 @@
-package tests
-
-import (
-	"testing"
-	"time"
-
-	"github.com/gofrs/uuid"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt"
-	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/filesystem"
-	"github.com/stretchr/testify/assert"
-)
-
-func newGuidString(t *testing.T) string {
-	uuid, err := uuid.NewV4()
-	assert.NoError(t, err)
-
-	return uuid.String()
-}
-
-type stackBuilder struct {
-	t     *testing.T
-	count int
-	store *bolt.Store
-}
-
-func TestService_StackByWebhookID(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
-	}
-	store, teardown := bolt.MustNewTestStore(true)
-	defer teardown()
-
-	b := stackBuilder{t: t, store: store}
-	b.createNewStack(newGuidString(t))
-	for i := 0; i < 10; i++ {
-		b.createNewStack("")
-	}
-	webhookID := newGuidString(t)
-	stack := b.createNewStack(webhookID)
-
-	// can find a stack by webhook ID
-	got, err := store.StackService.StackByWebhookID(webhookID)
-	assert.NoError(t, err)
-	assert.Equal(t, stack, *got)
-
-	// returns nil and object not found error if there's no stack associated with the webhook
-	got, err = store.StackService.StackByWebhookID(newGuidString(t))
-	assert.Nil(t, got)
-	assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound)
-}
-
-func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
-	b.count++
-	stack := portainer.Stack{
-		ID:           portainer.StackID(b.count),
-		Name:         "Name",
-		Type:         portainer.DockerComposeStack,
-		EndpointID:   2,
-		EntryPoint:   filesystem.ComposeFileDefaultName,
-		Env:          []portainer.Pair{{"Name1", "Value1"}},
-		Status:       portainer.StackStatusActive,
-		CreationDate: time.Now().Unix(),
-		ProjectPath:  "/tmp/project",
-		CreatedBy:    "test",
-	}
-
-	if webhookID == "" {
-		if b.count%2 == 0 {
-			stack.AutoUpdate = &portainer.StackAutoUpdate{
-				Interval: "",
-				Webhook:  "",
-			}
-		} // else keep AutoUpdate nil
-	} else {
-		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
-	}
-
-	err := b.store.StackService.CreateStack(&stack)
-	assert.NoError(b.t, err)
-
-	return stack
-}
-
-func Test_RefreshableStacks(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
-	}
-	store, teardown := bolt.MustNewTestStore(true)
-	defer teardown()
-
-	staticStack := portainer.Stack{ID: 1}
-	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
-	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
-
-	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
-		err := store.Stack().CreateStack(stack)
-		assert.NoError(t, err)
-	}
-
-	stacks, err := store.Stack().RefreshableStacks()
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
-}

api/bolt/tag/tag.go

@@ -1,7 +1,7 @@
 package tag
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "tags"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -33,7 +33,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Tags() ([]portainer.Tag, error) {
 	var tags = make([]portainer.Tag, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -57,7 +57,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 	var tag portainer.Tag
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &tag)
+	err := internal.GetObject(service.db, BucketName, identifier, &tag)
 	if err != nil {
 		return nil, err
 	}
@@ -67,7 +67,7 @@ func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
 
 // CreateTag creates a new tag.
 func (service *Service) CreateTag(tag *portainer.Tag) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -85,11 +85,11 @@ func (service *Service) CreateTag(tag *portainer.Tag) error {
 // UpdateTag updates a tag.
 func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, tag)
+	return internal.UpdateObject(service.db, BucketName, identifier, tag)
 }
 
 // DeleteTag deletes a tag.
 func (service *Service) DeleteTag(ID portainer.TagID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/team/team.go

@@ -15,20 +15,20 @@ const (
 	BucketName = "teams"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -37,7 +37,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 	var team portainer.Team
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &team)
+	err := internal.GetObject(service.db, BucketName, identifier, &team)
 	if err != nil {
 		return nil, err
 	}
@@ -49,7 +49,7 @@ func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 	var team *portainer.Team
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -80,7 +80,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 func (service *Service) Teams() ([]portainer.Team, error) {
 	var teams = make([]portainer.Team, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -102,12 +102,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 // UpdateTeam saves a Team.
 func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, team)
+	return internal.UpdateObject(service.db, BucketName, identifier, team)
 }
 
 // CreateTeam creates a new Team.
 func (service *Service) CreateTeam(team *portainer.Team) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -125,5 +125,5 @@ func (service *Service) CreateTeam(team *portainer.Team) error {
 // DeleteTeam deletes a Team.
 func (service *Service) DeleteTeam(ID portainer.TeamID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/teammembership/teammembership.go

@@ -1,7 +1,7 @@
 package teammembership
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
@@ -12,20 +12,20 @@ const (
 	BucketName = "team_membership"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 	var membership portainer.TeamMembership
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &membership)
+	err := internal.GetObject(service.db, BucketName, identifier, &membership)
 	if err != nil {
 		return nil, err
 	}
@@ -46,7 +46,7 @@ func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portaine
 func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -69,7 +69,7 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -95,7 +95,7 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -120,12 +120,12 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 // UpdateTeamMembership saves a TeamMembership object.
 func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
 	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.connection, BucketName, identifier, membership)
+	return internal.UpdateObject(service.db, BucketName, identifier, membership)
 }
 
 // CreateTeamMembership creates a new TeamMembership object.
 func (service *Service) CreateTeamMembership(membership *portainer.TeamMembership) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -143,12 +143,12 @@ func (service *Service) CreateTeamMembership(membership *portainer.TeamMembershi
 // DeleteTeamMembership deletes a TeamMembership object.
 func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
 func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -173,7 +173,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
 func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()

api/bolt/teststore.go

@@ -1,68 +0,0 @@
-package bolt
-
-import (
-	"io/ioutil"
-	"log"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/filesystem"
-)
-
-var errTempDir = errors.New("can't create a temp dir")
-
-func MustNewTestStore(init bool) (*Store, func()) {
-	store, teardown, err := NewTestStore(init)
-	if err != nil {
-		if !errors.Is(err, errTempDir) {
-			teardown()
-		}
-		log.Fatal(err)
-	}
-
-	return store, teardown
-}
-
-func NewTestStore(init bool) (*Store, func(), error) {
-	// Creates unique temp directory in a concurrency friendly manner.
-	dataStorePath, err := ioutil.TempDir("", "boltdb")
-	if err != nil {
-		return nil, nil, errors.Wrap(errTempDir, err.Error())
-	}
-
-	fileService, err := filesystem.NewService(dataStorePath, "")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	store := NewStore(dataStorePath, fileService)
-	err = store.Open()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if init {
-		err = store.Init()
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	teardown := func() {
-		teardown(store, dataStorePath)
-	}
-
-	return store, teardown, nil
-}
-
-func teardown(store *Store, dataStorePath string) {
-	err := store.Close()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = os.RemoveAll(dataStorePath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-}

api/bolt/tunnelserver/tunnelserver.go

@@ -1,8 +1,10 @@
 package tunnelserver
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -11,20 +13,20 @@ const (
 	infoKey    = "INFO"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -32,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 	var info portainer.TunnelServerInfo
 
-	err := internal.GetObject(service.connection, BucketName, []byte(infoKey), &info)
+	err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +44,5 @@ func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
 
 // UpdateInfo persists a TunnelServerInfo object.
 func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
-	return internal.UpdateObject(service.connection, BucketName, []byte(infoKey), settings)
+	return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
 }

api/bolt/user/user.go

@@ -15,20 +15,20 @@ const (
 	BucketName = "users"
 )
 
-// Service represents a service for managing environment(endpoint) data.
+// Service represents a service for managing endpoint data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -37,7 +37,7 @@ func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 	var user portainer.User
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &user)
+	err := internal.GetObject(service.db, BucketName, identifier, &user)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +51,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 
 	username = strings.ToLower(username)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -81,7 +81,7 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 func (service *Service) Users() ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -103,7 +103,7 @@ func (service *Service) Users() ([]portainer.User, error) {
 // UsersByRole return an array containing all the users with the specified role.
 func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
 	var users = make([]portainer.User, 0)
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -128,12 +128,12 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
 	identifier := internal.Itob(int(ID))
 	user.Username = strings.ToLower(user.Username)
-	return internal.UpdateObject(service.connection, BucketName, identifier, user)
+	return internal.UpdateObject(service.db, BucketName, identifier, user)
 }
 
 // CreateUser creates a new user.
 func (service *Service) CreateUser(user *portainer.User) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()
@@ -152,5 +152,5 @@ func (service *Service) CreateUser(user *portainer.User) error {
 // DeleteUser deletes a user.
 func (service *Service) DeleteUser(ID portainer.UserID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }

api/bolt/version/version.go

@@ -15,23 +15,22 @@ const (
 	versionKey  = "DB_VERSION"
 	instanceKey = "INSTANCE_ID"
 	editionKey  = "EDITION"
-	updatingKey = "DB_UPDATING"
 )
 
 // Service represents a service to manage stored versions.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -39,7 +38,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) DBVersion() (int, error) {
 	var data []byte
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(versionKey))
@@ -76,7 +75,7 @@ func (service *Service) Edition() (portainer.SoftwareEdition, error) {
 
 // StoreDBVersion store the database version.
 func (service *Service) StoreDBVersion(version int) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(strconv.Itoa(version))
@@ -84,26 +83,11 @@ func (service *Service) StoreDBVersion(version int) error {
 	})
 }
 
-// IsUpdating retrieves the database updating status.
-func (service *Service) IsUpdating() (bool, error) {
-	isUpdating, err := service.getKey(updatingKey)
-	if err != nil {
-		return false, err
-	}
-
-	return strconv.ParseBool(string(isUpdating))
-}
-
-// StoreIsUpdating store the database updating status.
-func (service *Service) StoreIsUpdating(isUpdating bool) error {
-	return service.setKey(updatingKey, strconv.FormatBool(isUpdating))
-}
-
 // InstanceID retrieves the stored instance ID.
 func (service *Service) InstanceID() (string, error) {
 	var data []byte
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(instanceKey))
@@ -125,7 +109,7 @@ func (service *Service) InstanceID() (string, error) {
 
 // StoreInstanceID store the instance ID.
 func (service *Service) StoreInstanceID(ID string) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(ID)
@@ -136,7 +120,7 @@ func (service *Service) StoreInstanceID(ID string) error {
 func (service *Service) getKey(key string) ([]byte, error) {
 	var data []byte
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		value := bucket.Get([]byte(key))
@@ -158,7 +142,7 @@ func (service *Service) getKey(key string) ([]byte, error) {
 }
 
 func (service *Service) setKey(key string, value string) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		data := []byte(value)

api/bolt/webhook/webhook.go

@@ -1,7 +1,7 @@
 package webhook
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
@@ -15,18 +15,18 @@ const (
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection *internal.DbConnection
+	db *bolt.DB
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection *internal.DbConnection) (*Service, error) {
-	err := internal.CreateBucket(connection, BucketName)
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Service{
-		connection: connection,
+		db: db,
 	}, nil
 }
 
@@ -34,7 +34,7 @@ func NewService(connection *internal.DbConnection) (*Service, error) {
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		cursor := bucket.Cursor()
@@ -58,7 +58,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 	var webhook portainer.Webhook
 	identifier := internal.Itob(int(ID))
 
-	err := internal.GetObject(service.connection, BucketName, identifier, &webhook)
+	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +70,7 @@ func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, err
 func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -101,7 +101,7 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
 	var webhook *portainer.Webhook
 
-	err := service.connection.View(func(tx *bolt.Tx) error {
+	err := service.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
@@ -131,12 +131,12 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 // DeleteWebhook deletes a webhook.
 func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
 	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.connection, BucketName, identifier)
+	return internal.DeleteObject(service.db, BucketName, identifier)
 }
 
 // CreateWebhook assign an ID to a new webhook and saves it.
 func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
-	return service.connection.Update(func(tx *bolt.Tx) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
 		id, _ := bucket.NextSequence()

api/chisel/schedules.go

@@ -6,7 +6,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-// AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
+// AddEdgeJob register an EdgeJob inside the tunnel details associated to an endpoint.
 func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *portainer.EdgeJob) {
 	tunnel := service.GetTunnelDetails(endpointID)
 

api/chisel/service.go

@@ -1,18 +1,15 @@
 package chisel
 
 import (
-	"context"
 	"fmt"
-	"github.com/portainer/portainer/api/http/proxy"
 	"log"
-	"net/http"
 	"strconv"
 	"time"
 
 	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
 	cmap "github.com/orcaman/concurrent-map"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
@@ -32,68 +29,16 @@ type Service struct {
 	dataStore         portainer.DataStore
 	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
-	shutdownCtx       context.Context
-	ProxyManager      *proxy.Manager
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(dataStore portainer.DataStore, shutdownCtx context.Context) *Service {
+func NewService(dataStore portainer.DataStore) *Service {
 	return &Service{
 		tunnelDetailsMap: cmap.New(),
 		dataStore:        dataStore,
-		shutdownCtx:      shutdownCtx,
 	}
 }
 
-// pingAgent ping the given agent so that the agent can keep the tunnel alive
-func (service *Service) pingAgent(endpointID portainer.EndpointID) error{
-	tunnel := service.GetTunnelDetails(endpointID)
-	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
-	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
-	if err != nil {
-		return err
-	}
-
-	httpClient := &http.Client{
-		Timeout: 3 * time.Second,
-	}
-	_, err = httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
-func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
-	go func() {
-		log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: start for %.0f minutes]\n", endpointID, maxAlive.Minutes())
-		maxAliveTicker := time.NewTicker(maxAlive)
-		defer maxAliveTicker.Stop()
-		pingTicker := time.NewTicker(tunnelCleanupInterval)
-		defer pingTicker.Stop()
-
-		for {
-			select {
-			case <-pingTicker.C:
-				service.SetTunnelStatusToActive(endpointID)
-				err := service.pingAgent(endpointID)
-				if err != nil {
-					log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [warning: ping agent err=%s]\n", endpointID, err)
-				}
-			case <-maxAliveTicker.C:
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as %.0f minutes timeout]\n", endpointID, maxAlive.Minutes())
-				return
-			case <-ctx.Done():
-				err := ctx.Err()
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as err=%s]\n", endpointID, err)
-				return
-			}
-		}
-	}()
-}
-
 // StartTunnelServer starts a tunnel server on the specified addr and port.
 // It uses a seed to generate a new private/public key pair. If the seed cannot
 // be found inside the database, it will generate a new one randomly and persist it.
@@ -138,11 +83,6 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
 	return nil
 }
 
-// StopTunnelServer stops tunnel http server
-func (service *Service) StopTunnelServer() error {
-	return service.chiselServer.Close()
-}
-
 func (service *Service) retrievePrivateKeySeed() (string, error) {
 	var serverInfo *portainer.TunnelServerInfo
 
@@ -168,16 +108,13 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 func (service *Service) startTunnelVerificationLoop() {
 	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
 	ticker := time.NewTicker(tunnelCleanupInterval)
+	stopSignal := make(chan struct{})
 
 	for {
 		select {
 		case <-ticker.C:
 			service.checkTunnels()
-		case <-service.shutdownCtx.Done():
-			log.Println("[DEBUG] Shutting down tunnel service")
-			if err := service.StopTunnelServer(); err != nil {
-				log.Printf("Stopped tunnel service: %s", err)
-			}
+		case <-stopSignal:
 			ticker.Stop()
 			return
 		}
@@ -193,7 +130,7 @@ func (service *Service) checkTunnels() {
 		}
 
 		elapsed := time.Since(tunnel.LastActivity)
-		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
+		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: endpoint tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
 
 		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
 			continue
@@ -208,22 +145,27 @@ func (service *Service) checkTunnels() {
 
 			endpointID, err := strconv.Atoi(item.Key)
 			if err != nil {
-				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
 			}
 
 			err = service.snapshotEnvironment(portainer.EndpointID(endpointID), tunnel.Port)
 			if err != nil {
-				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge endpoint (id: %s): %s", item.Key, err)
 			}
 		}
 
-		endpointID, err := strconv.Atoi(item.Key)
-		if err != nil {
-			log.Printf("[ERROR] [chisel,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
-			continue
+		if len(tunnel.Jobs) > 0 {
+			endpointID, err := strconv.Atoi(item.Key)
+			if err != nil {
+				log.Printf("[ERROR] [chisel,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
+				continue
+			}
+
+			service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
+		} else {
+			service.tunnelDetailsMap.Remove(item.Key)
 		}
 
-		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
 	}
 }
 

api/chisel/tunnel.go

@@ -38,7 +38,7 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
-// GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
+// GetTunnelDetails returns information about the tunnel associated to an endpoint.
 func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
 	key := strconv.Itoa(int(endpointID))
 
@@ -56,48 +56,7 @@ func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *porta
 	}
 }
 
-// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portainer.TunnelDetails, error) {
-	tunnel := service.GetTunnelDetails(endpoint.ID)
-
-	if tunnel.Status == portainer.EdgeAgentActive {
-		// update the LastActivity
-		service.SetTunnelStatusToActive(endpoint.ID)
-	}
-
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return nil, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := service.dataStore.Settings().Settings()
-			if err != nil {
-				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		waitForAgentToConnect := 2 * time.Duration(endpoint.EdgeCheckinInterval)
-
-		for waitForAgentToConnect >= 0 {
-			waitForAgentToConnect--
-			time.Sleep(time.Second)
-			tunnel = service.GetTunnelDetails(endpoint.ID)
-			if tunnel.Status == portainer.EdgeAgentActive {
-				break
-			}
-		}
-	}
-
-	tunnel = service.GetTunnelDetails(endpoint.ID)
-
-	return tunnel, nil
-}
-
-// SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
+// SetTunnelStatusToActive update the status of the tunnel associated to the specified endpoint.
 // It sets the status to ACTIVE.
 func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
 	tunnel := service.GetTunnelDetails(endpointID)
@@ -109,7 +68,7 @@ func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID)
 	service.tunnelDetailsMap.Set(key, tunnel)
 }
 
-// SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
+// SetTunnelStatusToIdle update the status of the tunnel associated to the specified endpoint.
 // It sets the status to IDLE.
 // It removes any existing credentials associated to the tunnel.
 func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
@@ -127,15 +86,13 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 
 	key := strconv.Itoa(int(endpointID))
 	service.tunnelDetailsMap.Set(key, tunnel)
-
-	service.ProxyManager.DeleteEndpointProxy(endpointID)
 }
 
-// SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
+// SetTunnelStatusToRequired update the status of the tunnel associated to the specified endpoint.
 // It sets the status to REQUIRED.
 // If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
 // and generate temporary credentials that can be used to establish a reverse tunnel on that port.
-// Credentials are encrypted using the Edge ID associated to the environment(endpoint).
+// Credentials are encrypted using the Edge ID associated to the endpoint.
 func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
 	tunnel := service.GetTunnelDetails(endpointID)
 

api/cli/cli.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"time"
 
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -18,7 +18,7 @@ import (
 type Service struct{}
 
 var (
-	errInvalidEndpointProtocol       = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errInvalidEndpointProtocol       = errors.New("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
 	errSocketOrNamedPipeNotFound     = errors.New("Unable to locate Unix socket or named pipe")
 	errInvalidSnapshotInterval       = errors.New("Invalid snapshot interval")
 	errAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
@@ -30,12 +30,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 	flags := &portainer.CLIFlags{
 		Addr:                      kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
-		AddrHTTPS:                 kingpin.Flag("bind-https", "Address and port to serve Portainer via https").Default(defaultHTTPSBindAddress).String(),
 		TunnelAddr:                kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
+		EndpointURL:               kingpin.Flag("host", "Endpoint URL").Short('H').String(),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
 		NoAnalytics:               kingpin.Flag("no-analytics", "Disable Analytics in app (deprecated)").Bool(),
 		TLS:                       kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
@@ -43,12 +42,10 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TLSCacert:                 kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:                   kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
 		TLSKey:                    kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
-		HTTPDisabled:              kingpin.Flag("http-disabled", "Serve portainer only on https").Default(defaultHTTPDisabled).Bool(),
-		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
-		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
-		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
-		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(),
+		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
+		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
+		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
+		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:                    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
@@ -95,10 +92,6 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 	if *flags.NoAnalytics {
 		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
 	}
-
-	if *flags.SSL {
-		log.Println("Warning: SSL is enabled by default and there is no need for the --ssl flag. It has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
-	}
 }
 
 func validateEndpointURL(endpointURL string) error {

api/cli/confirm.go

@@ -1,24 +0,0 @@
-package cli
-
-import (
-	"bufio"
-	"log"
-	"os"
-	"strings"
-)
-
-// Confirm starts a rollback db cli application
-func Confirm(message string) (bool, error) {
-	log.Printf("%s [y/N]", message)
-
-	reader := bufio.NewReader(os.Stdin)
-	answer, err := reader.ReadString('\n')
-	if err != nil {
-		return false, err
-	}
-	answer = strings.Replace(answer, "\n", "", -1)
-	answer = strings.ToLower(answer)
-
-	return answer == "y" || answer == "yes", nil
-
-}

api/cli/defaults.go

@@ -4,7 +4,6 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
-	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "/data"
@@ -14,7 +13,6 @@ const (
 	defaultTLSCACertPath       = "/certs/ca.pem"
 	defaultTLSCertPath         = "/certs/cert.pem"
 	defaultTLSKeyPath          = "/certs/key.pem"
-	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"

api/cli/defaults_windows.go

@@ -2,7 +2,6 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
-	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "C:\\data"
@@ -12,7 +11,6 @@ const (
 	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
 	defaultTLSCertPath         = "C:\\certs\\cert.pem"
 	defaultTLSKeyPath          = "C:\\certs\\key.pem"
-	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"

api/cmd/portainer/log.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	"log"
-
-	"github.com/sirupsen/logrus"
-)
-
-func configureLogger() {
-	logger := logrus.New() // logger is to implicitly substitute stdlib's log
-	log.SetOutput(logger.Writer())
-
-	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
-	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatter)
-
-	logger.SetLevel(logrus.DebugLevel)
-	logrus.SetLevel(logrus.DebugLevel)
-}

api/cmd/portainer/main.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"context"
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
@@ -13,7 +13,6 @@ import (
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/docker"
 
-	"github.com/portainer/libhelm"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
@@ -21,17 +20,13 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/api/http/proxy"
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
-	"github.com/portainer/portainer/api/internal/authorization"
-	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
-	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
+	"github.com/portainer/portainer/api/libcompose"
 	"github.com/portainer/portainer/api/oauth"
-	"github.com/portainer/portainer/api/scheduler"
-	"github.com/portainer/portainer/api/stacks"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -56,22 +51,15 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
-	store := bolt.NewStore(dataStorePath, fileService)
-	err := store.Open()
+func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
+	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
-		log.Fatalf("failed opening store: %v", err)
+		log.Fatalf("failed creating data store: %v", err)
 	}
 
-	if rollback {
-		err := store.Rollback(false)
-		if err != nil {
-			log.Fatalf("failed rolling back: %s", err)
-		}
-
-		log.Println("Exiting rollback")
-		os.Exit(0)
-		return nil
+	err = store.Open()
+	if err != nil {
+		log.Fatalf("failed opening store: %v", err)
 	}
 
 	err = store.Init()
@@ -79,39 +67,28 @@ func initDataStore(dataStorePath string, rollback bool, fileService portainer.Fi
 		log.Fatalf("failed initializing data store: %v", err)
 	}
 
-	err = store.MigrateData(false)
+	err = store.MigrateData()
 	if err != nil {
 		log.Fatalf("failed migration: %v", err)
 	}
-
-	go shutdownDatastore(shutdownCtx, store)
 	return store
 }
 
-func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStore) {
-	<-shutdownCtx.Done()
-	datastore.Close()
-}
-
-func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
-	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
-	if err != nil {
-		log.Fatalf("failed creating compose manager: %s", err)
+func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
+	composeWrapper := exec.NewComposeWrapper(assetsPath, proxyManager)
+	if composeWrapper != nil {
+		return composeWrapper
 	}
 
-	return composeWrapper
+	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 }
 
-func initSwarmStackManager(assetsPath string, configPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
-	return exec.NewSwarmStackManager(assetsPath, configPath, signatureService, fileService, reverseTunnelService)
+func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
+	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, assetsPath)
-}
-
-func initHelmPackageManager(assetsPath string) (libhelm.HelmPackageManager, error) {
-	return libhelm.NewHelmPackageManager(libhelm.HelmConfig{BinaryPath: assetsPath})
+func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(assetsPath)
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -124,7 +101,7 @@ func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error)
 		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
 		dataStore.Settings().UpdateSettings(settings)
 	}
-	jwtService, err := jwt.NewService(settings.UserSessionTimeout, dataStore)
+	jwtService, err := jwt.NewService(settings.UserSessionTimeout)
 	if err != nil {
 		return nil, err
 	}
@@ -151,36 +128,19 @@ func initGitService() portainer.GitService {
 	return git.NewService()
 }
 
-func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore portainer.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
-	slices := strings.Split(addr, ":")
-	host := slices[0]
-	if host == "" {
-		host = "0.0.0.0"
-	}
-
-	sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)
-
-	err := sslService.Init(host, certPath, keyPath)
-	if err != nil {
-		return nil, err
-	}
-
-	return sslService, nil
-}
-
 func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore portainer.DataStore) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string) *kubecli.ClientFactory {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID)
 }
 
-func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
+	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter)
 	if err != nil {
 		return nil, err
 	}
@@ -188,10 +148,24 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
-func initStatus(instanceID string) *portainer.Status {
+func loadEdgeJobsFromDatabase(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService) error {
+	edgeJobs, err := dataStore.EdgeJob().EdgeJobs()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeJob := range edgeJobs {
+		for endpointID := range edgeJob.Endpoints {
+			reverseTunnelService.AddEdgeJob(endpointID, &edgeJob)
+		}
+	}
+
+	return nil
+}
+
+func initStatus(flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
-		Version:    portainer.APIVersion,
-		InstanceID: instanceID,
+		Version: portainer.APIVersion,
 	}
 }
 
@@ -205,7 +179,6 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 	settings.SnapshotInterval = *flags.SnapshotInterval
 	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
 	settings.EnableTelemetry = true
-	settings.OAuthSettings.SSO = true
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -215,26 +188,7 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 		settings.BlackListedLabels = *flags.Labels
 	}
 
-	err = dataStore.Settings().UpdateSettings(settings)
-	if err != nil {
-		return err
-	}
-
-	httpEnabled := !*flags.HTTPDisabled
-
-	sslSettings, err := dataStore.SSLSettings().Settings()
-	if err != nil {
-		return err
-	}
-
-	sslSettings.HTTPEnabled = httpEnabled
-
-	err = dataStore.SSLSettings().UpdateSettings(sslSettings)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return dataStore.Settings().UpdateSettings(settings)
 }
 
 func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
@@ -300,7 +254,6 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
-			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -328,7 +281,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.Dat
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 	}
 
 	return dataStore.Endpoint().CreateEndpoint(endpoint)
@@ -362,7 +315,6 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 			AllowVolumeBrowserForRegularUsers: false,
 			EnableHostManagementFeatures:      false,
 
-			AllowSysctlSettingForRegularUsers:         true,
 			AllowBindMountsForRegularUsers:            true,
 			AllowPrivilegedModeForRegularUsers:        true,
 			AllowHostNamespaceForRegularUsers:         true,
@@ -374,7 +326,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore,
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 	}
 
 	return dataStore.Endpoint().CreateEndpoint(endpoint)
@@ -391,7 +343,7 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	}
 
 	if len(endpoints) > 0 {
-		log.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
+		log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
 		return nil
 	}
 
@@ -401,12 +353,28 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snap
 	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
-func buildServer(flags *portainer.CLIFlags) portainer.Server {
-	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
+func terminateIfNoAdminCreated(dataStore portainer.DataStore) {
+	timer1 := time.NewTimer(5 * time.Minute)
+	<-timer1.C
+
+	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		log.Fatalf("failed getting admin user: %v", err)
+	}
+
+	if len(users) == 0 {
+		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
+		return
+	}
+}
+
+func main() {
+	flags := initCLI()
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, *flags.Rollback, fileService, shutdownCtx)
+	dataStore := initDataStore(*flags.Data, fileService)
+	defer dataStore.Close()
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -427,22 +395,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	digitalSignatureService := initDigitalSignatureService()
 
-	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	sslSettings, err := sslService.GetSSLSettings()
-	if err != nil {
-		log.Fatalf("failed to get ssl settings: %s", err)
-	}
-
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatalf("failed initializing key pai: %v", err)
 	}
 
-	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
+	reverseTunnelService := chisel.NewService(dataStore)
 
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
@@ -450,40 +408,24 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
+	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID)
 
-	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
+	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory)
 	if err != nil {
 		log.Fatalf("failed initializing snapshot service: %v", err)
 	}
 	snapshotService.Start()
 
-	authorizationService := authorization.NewService(dataStore)
-	authorizationService.K8sClientFactory = kubernetesClientFactory
-
+	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
+	if err != nil {
+		log.Fatalf("failed initializing swarm stack manager: %v", err)
+	}
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
-
-	kubeConfigService := kubernetes.NewKubeConfigCAService(*flags.AddrHTTPS, sslSettings.CertPath)
-
 	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
 
-	reverseTunnelService.ProxyManager = proxyManager
+	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
 
-	dockerConfigPath := fileService.GetDockerConfigPath()
-
-	composeStackManager := initComposeStackManager(*flags.Assets, dockerConfigPath, reverseTunnelService, proxyManager)
-
-	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService)
-	if err != nil {
-		log.Fatalf("failed initializing swarm stack manager: %s", err)
-	}
-
-	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
-
-	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
-	if err != nil {
-		log.Fatalf("failed initializing helm package manager: %s", err)
-	}
+	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)
@@ -492,16 +434,16 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		}
 	}
 
-	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
+	err = loadEdgeJobsFromDatabase(dataStore, reverseTunnelService)
 	if err != nil {
 		log.Fatalf("failed loading edge jobs from database: %v", err)
 	}
 
-	applicationStatus := initStatus(instanceID)
+	applicationStatus := initStatus(flags)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatalf("failed initializing environment: %v", err)
+		log.Fatalf("failed initializing endpoint: %v", err)
 	}
 
 	adminPasswordHash := ""
@@ -540,33 +482,22 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		}
 	}
 
+	go terminateIfNoAdminCreated(dataStore)
+
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting tunnel server: %s", err)
+		log.Fatalf("failed starting tunnel server: %v", err)
 	}
 
-	sslDBSettings, err := dataStore.SSLSettings().Settings()
-	if err != nil {
-		log.Fatalf("failed to fetch ssl settings from DB")
-	}
-
-	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
-	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
-
-	return &http.Server{
-		AuthorizationService:        authorizationService,
+	var server portainer.Server = &http.Server{
 		ReverseTunnelService:        reverseTunnelService,
 		Status:                      applicationStatus,
 		BindAddress:                 *flags.Addr,
-		BindAddressHTTPS:            *flags.AddrHTTPS,
-		HTTPEnabled:                 sslDBSettings.HTTPEnabled,
 		AssetsPath:                  *flags.Assets,
 		DataStore:                   dataStore,
 		SwarmStackManager:           swarmStackManager,
 		ComposeStackManager:         composeStackManager,
 		KubernetesDeployer:          kubernetesDeployer,
-		HelmPackageManager:          helmPackageManager,
 		CryptoService:               cryptoService,
 		JWTService:                  jwtService,
 		FileService:                 fileService,
@@ -575,28 +506,18 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		GitService:                  gitService,
 		ProxyManager:                proxyManager,
 		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		KubeConfigService:           kubeConfigService,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
-		SSLService:                  sslService,
+		SSL:                         *flags.SSL,
+		SSLCert:                     *flags.SSLCert,
+		SSLKey:                      *flags.SSLKey,
 		DockerClientFactory:         dockerClientFactory,
 		KubernetesClientFactory:     kubernetesClientFactory,
-		Scheduler:                   scheduler,
-		ShutdownCtx:                 shutdownCtx,
-		ShutdownTrigger:             shutdownTrigger,
-		StackDeployer:               stackDeployer,
-	}
-}
-
-func main() {
-	flags := initCLI()
-
-	configureLogger()
-
-	for {
-		server := buildServer(flags)
-		log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
-		err := server.Start()
-		log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
+	}
+
+	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)
+	err = server.Start()
+	if err != nil {
+		log.Fatalf("failed starting server: %v", err)
 	}
 }

api/crypto/aes.go

@@ -1,70 +0,0 @@
-package crypto
-
-import (
-	"crypto/aes"
-	"crypto/cipher"
-	"io"
-
-	"golang.org/x/crypto/scrypt"
-)
-
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
-
-var emptySalt []byte = make([]byte, 0, 0)
-
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
-func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
-	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
-func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
-	if err != nil {
-		return nil, err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	reader := &cipher.StreamReader{S: stream, R: input}
-
-	return reader, nil
-}

api/crypto/aes_test.go

@@ -1,132 +0,0 @@
-package crypto
-
-import (
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to decrypt file")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
-}
-
-func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte(""))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(""))
-	assert.Nil(t, err, "Failed to decrypt file")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
-}
-
-func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
-
-	var (
-		originFilePath    = filepath.Join(tmpdir, "origin")
-		encryptedFilePath = filepath.Join(tmpdir, "encrypted")
-		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
-	)
-
-	content := []byte("content")
-	ioutil.WriteFile(originFilePath, content, 0600)
-
-	originFile, _ := os.Open(originFilePath)
-	defer originFile.Close()
-
-	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
-
-	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
-	assert.Nil(t, err, "Failed to encrypt a file")
-	encryptedContent, err := ioutil.ReadFile(encryptedFilePath)
-	assert.Nil(t, err, "Couldn't read encrypted file")
-	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
-
-	encryptedFileReader, _ := os.Open(encryptedFilePath)
-	defer encryptedFileReader.Close()
-
-	decryptedFileWriter, _ := os.Create(decryptedFilePath)
-	defer decryptedFileWriter.Close()
-
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := ioutil.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
-}

api/crypto/ecdsa.go

@@ -22,7 +22,7 @@ const (
 )
 
 // ECDSAService is a service used to create digital signatures when communicating with
-// an agent based environment(endpoint). It will automatically generates a key pair using ECDSA or
+// an agent based environment. It will automatically generates a key pair using ECDSA or
 // can also reuse an existing ECDSA key pair.
 type ECDSAService struct {
 	privateKey    *ecdsa.PrivateKey

api/docker/client.go

@@ -34,15 +34,15 @@ func NewClientFactory(signatureService portainer.DigitalSignatureService, revers
 }
 
 // createClient is a generic function to create a Docker client based on
-// a specific environment(endpoint) configuration. The nodeName parameter can be used
-// with an agent enabled environment(endpoint) to target a specific node in an agent cluster.
+// a specific endpoint configuration. The nodeName parameter can be used
+// with an agent enabled endpoint to target a specific node in an agent cluster.
 func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string) (*client.Client, error) {
 	if endpoint.Type == portainer.AzureEnvironment {
 		return nil, errUnsupportedEnvironmentType
 	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
 		return createAgentClient(endpoint, factory.signatureService, nodeName)
 	} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		return createEdgeClient(endpoint, factory.signatureService, factory.reverseTunnelService, nodeName)
+		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
 	}
 
 	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
@@ -71,31 +71,18 @@ func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	)
 }
 
-func createEdgeClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
+func createEdgeClient(endpoint *portainer.Endpoint, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
 	httpCli, err := httpClient(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	signature, err := signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
-	if err != nil {
-		return nil, err
-	}
-
-	headers := map[string]string{
-		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
-		portainer.PortainerAgentSignatureHeader: signature,
-	}
-
+	headers := map[string]string{}
 	if nodeName != "" {
 		headers[portainer.PortainerAgentTargetHeader] = nodeName
 	}
 
-	tunnel, err := reverseTunnelService.GetActiveTunnel(endpoint)
-	if err != nil {
-		return nil, err
-	}
-	
+	tunnel := reverseTunnelService.GetTunnelDetails(endpoint.ID)
 	endpointURL := fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
 
 	return client.NewClientWithOpts(

api/docker/errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 // Docker errors
 var (
-	ErrUnableToPingEndpoint = errors.New("Unable to communicate with the environment")
+	ErrUnableToPingEndpoint = errors.New("Unable to communicate with the endpoint")
 )

api/docker/snapshot.go

@@ -12,7 +12,7 @@ import (
 	"github.com/portainer/portainer/api"
 )
 
-// Snapshotter represents a service used to create environment(endpoint) snapshots
+// Snapshotter represents a service used to create endpoint snapshots
 type Snapshotter struct {
 	clientFactory *ClientFactory
 }
@@ -24,7 +24,7 @@ func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
 	}
 }
 
-// CreateSnapshot creates a snapshot of a specific Docker environment(endpoint)
+// CreateSnapshot creates a snapshot of a specific Docker endpoint
 func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
 	cli, err := snapshotter.clientFactory.CreateClient(endpoint, "")
 	if err != nil {
@@ -47,44 +47,44 @@ func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Dock
 
 	err = snapshotInfo(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	if snapshot.Swarm {
 		err = snapshotSwarmServices(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [environment: %s] [err: %s]", endpoint.Name, err)
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [endpoint: %s] [err: %s]", endpoint.Name, err)
 		}
 
 		err = snapshotNodes(snapshot, cli)
 		if err != nil {
-			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [environment: %s] [err: %s]", endpoint.Name, err)
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [endpoint: %s] [err: %s]", endpoint.Name, err)
 		}
 	}
 
 	err = snapshotContainers(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotImages(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotVolumes(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotNetworks(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	err = snapshotVersion(snapshot, cli)
 	if err != nil {
-		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [environment: %s] [err: %s]", endpoint.Name, err)
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [endpoint: %s] [err: %s]", endpoint.Name, err)
 	}
 
 	snapshot.Time = time.Now().Unix()

api/exec/common.go

@@ -1,5 +0,0 @@
-package exec
-
-import "regexp"
-
-var stackNameNormalizeRegex = regexp.MustCompile("[^-_a-z0-9]+")

api/exec/compose_stack.go

@@ -1,141 +0,0 @@
-package exec
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	libstack "github.com/portainer/docker-compose-wrapper"
-	"github.com/portainer/docker-compose-wrapper/compose"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/proxy"
-	"github.com/portainer/portainer/api/http/proxy/factory"
-)
-
-// ComposeStackManager is a wrapper for docker-compose binary
-type ComposeStackManager struct {
-	deployer     libstack.Deployer
-	proxyManager *proxy.Manager
-}
-
-// NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
-func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
-	deployer, err := compose.NewComposeDeployer(binaryPath, configPath)
-	if err != nil {
-		return nil, err
-	}
-
-	return &ComposeStackManager{
-		deployer:     deployer,
-		proxyManager: proxyManager,
-	}, nil
-}
-
-// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
-func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
-	return portainer.ComposeSyntaxMaxVersion
-}
-
-// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
-func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	url, proxy, err := manager.fetchEndpointProxy(endpoint)
-	if err != nil {
-		return errors.Wrap(err, "failed to fetch environment proxy")
-	}
-
-	if proxy != nil {
-		defer proxy.Close()
-	}
-
-	envFilePath, err := createEnvFile(stack)
-	if err != nil {
-		return errors.Wrap(err, "failed to create env file")
-	}
-
-	filePaths := getStackFiles(stack)
-	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFilePath)
-	return errors.Wrap(err, "failed to deploy a stack")
-}
-
-// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
-func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	url, proxy, err := manager.fetchEndpointProxy(endpoint)
-	if err != nil {
-		return err
-	}
-	if proxy != nil {
-		defer proxy.Close()
-	}
-
-	filePaths := getStackFiles(stack)
-	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths)
-	return errors.Wrap(err, "failed to remove a stack")
-}
-
-// NormalizeStackName returns a new stack name with unsupported characters replaced
-func (manager *ComposeStackManager) NormalizeStackName(name string) string {
-	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
-}
-
-func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
-	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
-		return "", nil, nil
-	}
-
-	proxy, err := manager.proxyManager.CreateAgentProxyServer(endpoint)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
-}
-
-func createEnvFile(stack *portainer.Stack) (string, error) {
-	if stack.Env == nil || len(stack.Env) == 0 {
-		return "", nil
-	}
-
-	envFilePath := path.Join(stack.ProjectPath, "stack.env")
-
-	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return "", err
-	}
-
-	for _, v := range stack.Env {
-		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
-	}
-	envfile.Close()
-
-	return "stack.env", nil
-}
-
-// getStackFiles returns list of stack's confile file paths.
-// items in the list would be sanitized according to following criterias:
-// 1. no empty paths
-// 2. no "../xxx" paths that are trying to escape stack folder
-// 3. no dir paths
-// 4. root paths would be made relative
-func getStackFiles(stack *portainer.Stack) []string {
-	paths := make([]string, 0, len(stack.AdditionalFiles)+1)
-
-	for _, p := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
-		if strings.HasPrefix(p, "/") {
-			p = `.` + p
-		}
-
-		if p == `` || p == `.` || strings.HasPrefix(p, `..`) || strings.HasSuffix(p, string(filepath.Separator)) {
-			continue
-		}
-
-		paths = append(paths, p)
-	}
-
-	return paths
-}

api/exec/compose_stack_test.go

@@ -1,84 +0,0 @@
-package exec
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_createEnvFile(t *testing.T) {
-	dir := t.TempDir()
-
-	tests := []struct {
-		name         string
-		stack        *portainer.Stack
-		expected     string
-		expectedFile bool
-	}{
-		{
-			name: "should not add env file option if stack doesn't have env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-			},
-			expected: "",
-		},
-		{
-			name: "should not add env file option if stack's env variables are empty",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env:         []portainer.Pair{},
-			},
-			expected: "",
-		},
-		{
-			name: "should add env file option if stack has env variables",
-			stack: &portainer.Stack{
-				ProjectPath: dir,
-				Env: []portainer.Pair{
-					{Name: "var1", Value: "value1"},
-					{Name: "var2", Value: "value2"},
-				},
-			},
-			expected: "var1=value1\nvar2=value2\n",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result, _ := createEnvFile(tt.stack)
-
-			if tt.expected != "" {
-				assert.Equal(t, "stack.env", result)
-
-				f, _ := os.Open(path.Join(dir, "stack.env"))
-				content, _ := ioutil.ReadAll(f)
-
-				assert.Equal(t, tt.expected, string(content))
-			} else {
-				assert.Equal(t, "", result)
-			}
-		})
-	}
-}
-
-func Test_getStackFiles(t *testing.T) {
-	stack := &portainer.Stack{
-		EntryPoint: "./file", // picks entry point
-		AdditionalFiles: []string{
-			``,                  // ignores empty string
-			`.`,                 // ignores .
-			`..`,                // ignores ..
-			`./dir/`,            // ignrores paths that end with trailing /
-			`/with-root-prefix`, // replaces "root" based paths with relative
-			`./relative`,        // keeps relative paths
-			`../escape`,         // prevents dir escape
-		},
-	}
-
-	filePaths := getStackFiles(stack)
-	assert.ElementsMatch(t, filePaths, []string{`./file`, `./with-root-prefix`, `./relative`})
-}

api/exec/compose_wrapper.go

@@ -0,0 +1,132 @@
+package exec
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+)
+
+// ComposeWrapper is a wrapper for docker-compose binary
+type ComposeWrapper struct {
+	binaryPath   string
+	proxyManager *proxy.Manager
+}
+
+// NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
+func NewComposeWrapper(binaryPath string, proxyManager *proxy.Manager) *ComposeWrapper {
+	if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
+		return nil
+	}
+
+	return &ComposeWrapper{
+		binaryPath:   binaryPath,
+		proxyManager: proxyManager,
+	}
+}
+
+// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
+func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
+	return portainer.ComposeSyntaxMaxVersion
+}
+
+// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
+func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	_, err := w.command([]string{"up", "-d"}, stack, endpoint)
+	return err
+}
+
+// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
+func (w *ComposeWrapper) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	_, err := w.command([]string{"down", "--remove-orphans"}, stack, endpoint)
+	return err
+}
+
+func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpoint *portainer.Endpoint) ([]byte, error) {
+	if endpoint == nil {
+		return nil, errors.New("cannot call a compose command on an empty endpoint")
+	}
+
+	program := programPath(w.binaryPath, "docker-compose")
+
+	options := setComposeFile(stack)
+
+	options = addProjectNameOption(options, stack)
+	options, err := addEnvFileOption(options, stack)
+	if err != nil {
+		return nil, err
+	}
+
+	if !(endpoint.URL == "" || strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://")) {
+
+		proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
+		if err != nil {
+			return nil, err
+		}
+
+		defer proxy.Close()
+
+		options = append(options, "-H", fmt.Sprintf("http://127.0.0.1:%d", proxy.Port))
+	}
+
+	args := append(options, command...)
+
+	var stderr bytes.Buffer
+	cmd := exec.Command(program, args...)
+	cmd.Stderr = &stderr
+
+	out, err := cmd.Output()
+	if err != nil {
+		return out, errors.New(stderr.String())
+	}
+
+	return out, nil
+}
+
+func setComposeFile(stack *portainer.Stack) []string {
+	options := make([]string, 0)
+
+	if stack == nil || stack.EntryPoint == "" {
+		return options
+	}
+
+	composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	options = append(options, "-f", composeFilePath)
+	return options
+}
+
+func addProjectNameOption(options []string, stack *portainer.Stack) []string {
+	if stack == nil || stack.Name == "" {
+		return options
+	}
+
+	options = append(options, "-p", stack.Name)
+	return options
+}
+
+func addEnvFileOption(options []string, stack *portainer.Stack) ([]string, error) {
+	if stack == nil || stack.Env == nil || len(stack.Env) == 0 {
+		return options, nil
+	}
+
+	envFilePath := path.Join(stack.ProjectPath, "stack.env")
+
+	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return options, err
+	}
+
+	for _, v := range stack.Env {
+		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
+	}
+	envfile.Close()
+
+	options = append(options, "--env-file", envFilePath)
+	return options, nil
+}

api/exec/compose_wrapper_integration_test.go

@@ -1,7 +1,8 @@
+// +build integration
+
 package exec
 
 import (
-	"context"
 	"fmt"
 	"log"
 	"os"
@@ -32,9 +33,7 @@ func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
 		Name:        "project-name",
 	}
 
-	endpoint := &portainer.Endpoint{
-		URL: "unix://",
-	}
+	endpoint := &portainer.Endpoint{}
 
 	return stack, endpoint
 }
@@ -43,23 +42,18 @@ func Test_UpAndDown(t *testing.T) {
 
 	stack, endpoint := setup(t)
 
-	w, err := NewComposeStackManager("", "", nil)
-	if err != nil {
-		t.Fatalf("Failed creating manager: %s", err)
-	}
+	w := NewComposeWrapper("", nil)
 
-	ctx := context.TODO()
-
-	err = w.Up(ctx, stack, endpoint)
+	err := w.Up(stack, endpoint)
 	if err != nil {
 		t.Fatalf("Error calling docker-compose up: %s", err)
 	}
 
-	if !containerExists(composedContainerName) {
+	if containerExists(composedContainerName) == false {
 		t.Fatal("container should exist")
 	}
 
-	err = w.Down(ctx, stack, endpoint)
+	err = w.Down(stack, endpoint)
 	if err != nil {
 		t.Fatalf("Error calling docker-compose down: %s", err)
 	}
@@ -69,13 +63,13 @@ func Test_UpAndDown(t *testing.T) {
 	}
 }
 
-func containerExists(containerName string) bool {
-	cmd := exec.Command("docker", "ps", "-a", "-f", fmt.Sprintf("name=%s", containerName))
+func containerExists(contaierName string) bool {
+	cmd := exec.Command(osProgram("docker"), "ps", "-a", "-f", fmt.Sprintf("name=%s", contaierName))
 
 	out, err := cmd.Output()
 	if err != nil {
 		log.Fatalf("failed to list containers: %s", err)
 	}
 
-	return strings.Contains(string(out), containerName)
+	return strings.Contains(string(out), contaierName)
 }

api/exec/compose_wrapper_test.go

@@ -0,0 +1,143 @@
+package exec
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_setComposeFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		stack    *portainer.Stack
+		expected []string
+	}{
+		{
+			name:     "should return empty result if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should return empty result if stack don't have entrypoint",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should allow file name and dir",
+			stack: &portainer.Stack{
+				ProjectPath: "dir",
+				EntryPoint:  "file",
+			},
+			expected: []string{"-f", path.Join("dir", "file")},
+		},
+		{
+			name: "should allow file name only",
+			stack: &portainer.Stack{
+				EntryPoint: "file",
+			},
+			expected: []string{"-f", "file"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := setComposeFile(tt.stack)
+			assert.ElementsMatch(t, tt.expected, result)
+		})
+	}
+}
+
+func Test_addProjectNameOption(t *testing.T) {
+	tests := []struct {
+		name     string
+		stack    *portainer.Stack
+		expected []string
+	}{
+		{
+			name:     "should not add project option if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should not add project option if stack doesn't have name",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should add project name option if stack has a name",
+			stack: &portainer.Stack{
+				Name: "project-name",
+			},
+			expected: []string{"-p", "project-name"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			options := []string{"-a", "b"}
+			result := addProjectNameOption(options, tt.stack)
+			assert.ElementsMatch(t, append(options, tt.expected...), result)
+		})
+	}
+}
+
+func Test_addEnvFileOption(t *testing.T) {
+	dir := t.TempDir()
+
+	tests := []struct {
+		name            string
+		stack           *portainer.Stack
+		expected        []string
+		expectedContent string
+	}{
+		{
+			name:     "should not add env file option if stack is missing",
+			stack:    nil,
+			expected: []string{},
+		},
+		{
+			name:     "should not add env file option if stack doesn't have env variables",
+			stack:    &portainer.Stack{},
+			expected: []string{},
+		},
+		{
+			name: "should not add env file option if stack's env variables are empty",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env:         []portainer.Pair{},
+			},
+			expected: []string{},
+		},
+		{
+			name: "should add env file option if stack has env variables",
+			stack: &portainer.Stack{
+				ProjectPath: dir,
+				Env: []portainer.Pair{
+					{Name: "var1", Value: "value1"},
+					{Name: "var2", Value: "value2"},
+				},
+			},
+			expected:        []string{"--env-file", path.Join(dir, "stack.env")},
+			expectedContent: "var1=value1\nvar2=value2\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			options := []string{"-a", "b"}
+			result, _ := addEnvFileOption(options, tt.stack)
+			assert.ElementsMatch(t, append(options, tt.expected...), result)
+
+			if tt.expectedContent != "" {
+				f, _ := os.Open(path.Join(dir, "stack.env"))
+				content, _ := ioutil.ReadAll(f)
+
+				assert.Equal(t, tt.expectedContent, string(content))
+			}
+		})
+	}
+}

api/exec/exectest/kubernetes_mocks.go

@@ -1,23 +0,0 @@
-package exectest
-
-import (
-	portainer "github.com/portainer/portainer/api"
-)
-
-type kubernetesMockDeployer struct{}
-
-func NewKubernetesDeployer() portainer.KubernetesDeployer {
-	return &kubernetesMockDeployer{}
-}
-
-func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return "", nil
-}
-
-func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return "", nil
-}
-
-func (deployer *kubernetesMockDeployer) ConvertCompose(data []byte) ([]byte, error) {
-	return nil, nil
-}

api/exec/kubernetes_deploy.go

@@ -2,92 +2,43 @@ package exec
 
 import (
 	"bytes"
-	"fmt"
+	"errors"
+	"io/ioutil"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
 
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/http/proxy"
-	"github.com/portainer/portainer/api/http/proxy/factory"
-	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
-	"github.com/portainer/portainer/api/kubernetes/cli"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
-// KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
+// KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath                  string
-	dataStore                   portainer.DataStore
-	reverseTunnelService        portainer.ReverseTunnelService
-	signatureService            portainer.DigitalSignatureService
-	kubernetesClientFactory     *cli.ClientFactory
-	kubernetesTokenCacheManager *kubernetes.TokenCacheManager
-	proxyManager                *proxy.Manager
+	binaryPath string
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath:                  binaryPath,
-		dataStore:                   datastore,
-		reverseTunnelService:        reverseTunnelService,
-		signatureService:            signatureService,
-		kubernetesClientFactory:     kubernetesClientFactory,
-		kubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		proxyManager:                proxyManager,
+		binaryPath: binaryPath,
 	}
 }
 
-func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
-	kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
+// Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
+// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
+// Otherwise it will use kubectl to deploy the manifest.
+func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+	if composeFormat {
+		convertedData, err := deployer.convertComposeData(data)
+		if err != nil {
+			return nil, err
+		}
+		data = string(convertedData)
+	}
+
+	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
 	if err != nil {
-		return "", err
-	}
-
-	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(int(endpoint.ID))
-
-	tokenManager, err := kubernetes.NewTokenManager(kubeCLI, deployer.dataStore, tokenCache, setLocalAdminToken)
-	if err != nil {
-		return "", err
-	}
-
-	user, err := deployer.dataStore.User().User(userID)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch the user")
-	}
-
-	if user.Role == portainer.AdministratorRole {
-		return tokenManager.GetAdminServiceAccountToken(), nil
-	}
-
-	token, err := tokenManager.GetUserServiceAccountToken(int(user.ID), endpoint.ID)
-	if err != nil {
-		return "", err
-	}
-
-	if token == "" {
-		return "", fmt.Errorf("can not get a valid user service account token")
-	}
-	return token, nil
-}
-
-// Deploy upserts Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("apply", userID, endpoint, manifestFiles, namespace)
-}
-
-// Remove deletes Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("delete", userID, endpoint, manifestFiles, namespace)
-}
-
-func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment)
-	if err != nil {
-		return "", errors.Wrap(err, "failed generating a user token")
+		return nil, err
 	}
 
 	command := path.Join(deployer.binaryPath, "kubectl")
@@ -95,45 +46,27 @@ func (deployer *KubernetesDeployer) command(operation string, userID portainer.U
 		command = path.Join(deployer.binaryPath, "kubectl.exe")
 	}
 
-	args := []string{"--token", token}
-	if namespace != "" {
-		args = append(args, "--namespace", namespace)
-	}
-
-	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		url, proxy, err := deployer.getAgentURL(endpoint)
-		if err != nil {
-			return "", errors.WithMessage(err, "failed generating endpoint URL")
-		}
-
-		defer proxy.Close()
-		args = append(args, "--server", url)
-		args = append(args, "--insecure-skip-tls-verify")
-	}
-
-	if operation == "delete" {
-		args = append(args, "--ignore-not-found=true")
-	}
-
-	args = append(args, operation)
-	for _, path := range manifestFiles {
-		args = append(args, "-f", strings.TrimSpace(path))
-	}
+	args := make([]string, 0)
+	args = append(args, "--server", endpoint.URL)
+	args = append(args, "--insecure-skip-tls-verify")
+	args = append(args, "--token", string(token))
+	args = append(args, "--namespace", namespace)
+	args = append(args, "apply", "-f", "-")
 
 	var stderr bytes.Buffer
 	cmd := exec.Command(command, args...)
 	cmd.Stderr = &stderr
+	cmd.Stdin = strings.NewReader(data)
 
 	output, err := cmd.Output()
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to execute kubectl command: %q", stderr.String())
+		return nil, errors.New(stderr.String())
 	}
 
-	return string(output), nil
+	return output, nil
 }
 
-// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
-func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error) {
+func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")
@@ -145,7 +78,7 @@ func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error)
 	var stderr bytes.Buffer
 	cmd := exec.Command(command, args...)
 	cmd.Stderr = &stderr
-	cmd.Stdin = bytes.NewReader(data)
+	cmd.Stdin = strings.NewReader(data)
 
 	output, err := cmd.Output()
 	if err != nil {
@@ -154,12 +87,3 @@ func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error)
 
 	return output, nil
 }
-
-func (deployer *KubernetesDeployer) getAgentURL(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
-	proxy, err := deployer.proxyManager.CreateAgentProxyServer(endpoint)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return fmt.Sprintf("http://127.0.0.1:%d/kubernetes", proxy.Port), proxy, nil
-}

api/exec/swarm_stack.go

@@ -9,16 +9,14 @@ import (
 	"os/exec"
 	"path"
 	"runtime"
-	"strings"
 
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/internal/stackutils"
+	"github.com/portainer/portainer/api"
 )
 
 // SwarmStackManager represents a service for managing stacks.
 type SwarmStackManager struct {
 	binaryPath           string
-	configPath           string
+	dataPath             string
 	signatureService     portainer.DigitalSignatureService
 	fileService          portainer.FileService
 	reverseTunnelService portainer.ReverseTunnelService
@@ -26,16 +24,16 @@ type SwarmStackManager struct {
 
 // NewSwarmStackManager initializes a new SwarmStackManager service.
 // It also updates the configuration of the Docker CLI binary.
-func NewSwarmStackManager(binaryPath, configPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
+func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
 	manager := &SwarmStackManager{
 		binaryPath:           binaryPath,
-		configPath:           configPath,
+		dataPath:             dataPath,
 		signatureService:     signatureService,
 		fileService:          fileService,
 		reverseTunnelService: reverseTunnelService,
 	}
 
-	err := manager.updateDockerCLIConfiguration(manager.configPath)
+	err := manager.updateDockerCLIConfiguration(dataPath)
 	if err != nil {
 		return nil, err
 	}
@@ -44,60 +42,51 @@ func NewSwarmStackManager(binaryPath, configPath string, signatureService portai
 }
 
 // Login executes the docker login command against a list of registries (including DockerHub).
-func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoint *portainer.Endpoint) error {
-	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
-	if err != nil {
-		return err
-	}
+func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	for _, registry := range registries {
 		if registry.Authentication {
 			registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL)
 			runCommandAndCaptureStdErr(command, registryArgs, nil, "")
 		}
 	}
-	return nil
+
+	if dockerhub.Authentication {
+		dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password)
+		runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "")
+	}
 }
 
 // Logout executes the docker logout command.
 func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
-	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
-	if err != nil {
-		return err
-	}
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	args = append(args, "logout")
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
-	filePaths := stackutils.GetStackFilePaths(stack)
-	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
-	if err != nil {
-		return err
-	}
+	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 
 	if prune {
-		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth")
+		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
 	} else {
-		args = append(args, "stack", "deploy", "--with-registry-auth")
+		args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
 	}
 
-	args = configureFilePaths(args, filePaths)
-	args = append(args, stack.Name)
-
 	env := make([]string, 0)
 	for _, envvar := range stack.Env {
 		env = append(env, envvar.Name+"="+envvar.Value)
 	}
-	return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath)
+
+	stackFolder := path.Dir(stackFilePath)
+	return runCommandAndCaptureStdErr(command, args, env, stackFolder)
 }
 
 // Remove executes the docker stack rm command.
 func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
-	if err != nil {
-		return err
-	}
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	args = append(args, "stack", "rm", stack.Name)
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
@@ -121,7 +110,7 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor
 	return nil
 }
 
-func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, configPath string, endpoint *portainer.Endpoint) (string, []string, error) {
+func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
 	// Assume Linux as a default
 	command := path.Join(binaryPath, "docker")
 
@@ -130,14 +119,11 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, config
 	}
 
 	args := make([]string, 0)
-	args = append(args, "--config", configPath)
+	args = append(args, "--config", dataPath)
 
 	endpointURL := endpoint.URL
 	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		tunnel, err := manager.reverseTunnelService.GetActiveTunnel(endpoint)
-		if err != nil {
-			return "", nil, err
-		}
+		tunnel := manager.reverseTunnelService.GetTunnelDetails(endpoint.ID)
 		endpointURL = fmt.Sprintf("tcp://127.0.0.1:%d", tunnel.Port)
 	}
 
@@ -157,11 +143,11 @@ func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, config
 		}
 	}
 
-	return command, args, nil
+	return command, args
 }
 
-func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string) error {
-	configFilePath := path.Join(configPath, "config.json")
+func (manager *SwarmStackManager) updateDockerCLIConfiguration(dataPath string) error {
+	configFilePath := path.Join(dataPath, "config.json")
 	config, err := manager.retrieveConfigurationFromDisk(configFilePath)
 	if err != nil {
 		return err
@@ -203,14 +189,3 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 
 	return config, nil
 }
-
-func (manager *SwarmStackManager) NormalizeStackName(name string) string {
-	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
-}
-
-func configureFilePaths(args []string, filePaths []string) []string {
-	for _, path := range filePaths {
-		args = append(args, "--compose-file", path)
-	}
-	return args
-}

api/exec/swarm_stack_test.go

@@ -1,15 +0,0 @@
-package exec
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConfigFilePaths(t *testing.T) {
-	args := []string{"stack", "deploy", "--with-registry-auth"}
-	filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"}
-	expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"}
-	output := configureFilePaths(args, filePaths)
-	assert.ElementsMatch(t, expected, output, "wrong output file paths")
-}

api/exec/utils.go

@@ -0,0 +1,24 @@
+package exec
+
+import (
+	"os/exec"
+	"path/filepath"
+	"runtime"
+)
+
+func osProgram(program string) string {
+	if runtime.GOOS == "windows" {
+		program += ".exe"
+	}
+	return program
+}
+
+func programPath(rootPath, program string) string {
+	return filepath.Join(rootPath, osProgram(program))
+}
+
+// IsBinaryPresent returns true if corresponding program exists on PATH
+func IsBinaryPresent(program string) bool {
+	_, err := exec.LookPath(program)
+	return err == nil
+}

api/exec/utils_test.go

@@ -0,0 +1,16 @@
+package exec
+
+import (
+	"testing"
+)
+
+func Test_isBinaryPresent(t *testing.T) {
+
+	if !IsBinaryPresent("docker") {
+		t.Error("expect docker binary to exist on the path")
+	}
+
+	if IsBinaryPresent("executable-with-this-name-should-not-exist") {
+		t.Error("expect binary with a random name to be missing on the path")
+	}
+}

api/filesystem/copy.go

@@ -1,82 +0,0 @@
-package filesystem
-
-import (
-	"errors"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// CopyPath copies file or directory defined by the path to the toDir path
-func CopyPath(path string, toDir string) error {
-	info, err := os.Stat(path)
-	if err != nil && errors.Is(err, os.ErrNotExist) {
-		// skip copy if file does not exist
-		return nil
-	}
-
-	if !info.IsDir() {
-		destination := filepath.Join(toDir, info.Name())
-		return copyFile(path, destination)
-	}
-
-	return CopyDir(path, toDir, true)
-}
-
-// CopyDir copies contents of fromDir to toDir.
-// When keepParent is true, contents will be copied with their immediate parent dir,
-// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/<children>
-func CopyDir(fromDir, toDir string, keepParent bool) error {
-	cleanedSourcePath := filepath.Clean(fromDir)
-	parentDirectory := filepath.Dir(cleanedSourcePath)
-	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		var destination string
-		if keepParent {
-			destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
-		} else {
-			destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath))
-		}
-
-		if destination == "" {
-			return nil
-		}
-
-		if info.IsDir() {
-			return nil // skip directory creations
-		}
-
-		if info.Mode()&os.ModeSymlink != 0 { // entry is a symlink
-			return nil // don't copy symlinks
-		}
-
-		return copyFile(path, destination)
-	})
-
-	return err
-}
-
-// copies regular a file from src to dst
-func copyFile(src, dst string) error {
-	from, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer from.Close()
-
-	// has to include 'execute' bit, otherwise fails. MkdirAll follows `mkdir -m` restrictions
-	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
-		return err
-	}
-	to, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer to.Close()
-
-	_, err = io.Copy(to, from)
-	return err
-}

api/filesystem/copy_test.go

@@ -1,92 +0,0 @@
-package filesystem
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyFile("does-not-exist", tmpdir)
-	assert.Error(t, err)
-}
-
-func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
-
-	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
-	assert.NoError(t, err)
-
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := CopyDir("./testdata/copy_test", destination, true)
-	assert.NoError(t, err)
-
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
-}
-
-func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := CopyDir("./testdata/copy_test", destination, false)
-	assert.NoError(t, err)
-
-	assert.FileExists(t, filepath.Join(destination, "outer"))
-	assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile"))
-	assert.FileExists(t, filepath.Join(destination, "dir", "inner"))
-}
-
-func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := CopyPath("does-not-exists", tmpdir)
-	assert.NoError(t, err)
-
-	assert.NoFileExists(t, tmpdir)
-}
-
-func Test_CopyPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutil.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
-
-	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
-	err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
-	assert.NoError(t, err)
-
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
-	assert.NoError(t, err)
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_CopyPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutil.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := CopyPath("./testdata/copy_test", destination)
-	assert.NoError(t, err)
-
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
-}

api/filesystem/filesystem.go

@@ -9,7 +9,7 @@ import (
 	"io/ioutil"
 
 	"github.com/gofrs/uuid"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 
 	"io"
 	"os"
@@ -31,8 +31,6 @@ const (
 	ComposeStorePath = "compose"
 	// ComposeFileDefaultName represents the default name of a compose file.
 	ComposeFileDefaultName = "docker-compose.yml"
-	// ManifestFileDefaultName represents the default name of a k8s manifest file.
-	ManifestFileDefaultName = "k8s-deployment.yml"
 	// EdgeStackStorePath represents the subfolder where edge stack files are stored in the file store folder.
 	EdgeStackStorePath = "edge_stacks"
 	// PrivateKeyFile represents the name on disk of the file containing the private key.
@@ -43,8 +41,6 @@ const (
 	BinaryStorePath = "bin"
 	// EdgeJobStorePath represents the subfolder where schedule files are stored.
 	EdgeJobStorePath = "edge_jobs"
-	// DockerConfigPath represents the subfolder where docker configuration is stored.
-	DockerConfigPath = "docker_config"
 	// ExtensionRegistryManagementStorePath represents the subfolder where files related to the
 	// registry management extension are stored.
 	ExtensionRegistryManagementStorePath = "extensions"
@@ -52,12 +48,6 @@ const (
 	CustomTemplateStorePath = "custom_templates"
 	// TempPath represent the subfolder where temporary files are saved
 	TempPath = "tmp"
-	// SSLCertPath represents the default ssl certificates path
-	SSLCertPath = "certs"
-	// DefaultSSLCertFilename represents the default ssl certificate file name
-	DefaultSSLCertFilename = "cert.pem"
-	// DefaultSSLKeyFilename represents the default ssl key file name
-	DefaultSSLKeyFilename = "key.pem"
 )
 
 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -82,11 +72,6 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
-	err = service.createDirectoryInStore(SSLCertPath)
-	if err != nil {
-		return nil, err
-	}
-
 	err = service.createDirectoryInStore(TLSStorePath)
 	if err != nil {
 		return nil, err
@@ -102,11 +87,6 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
-	err = service.createDirectoryInStore(DockerConfigPath)
-	if err != nil {
-		return nil, err
-	}
-
 	return service, nil
 }
 
@@ -115,11 +95,6 @@ func (service *Service) GetBinaryFolder() string {
 	return path.Join(service.fileStorePath, BinaryStorePath)
 }
 
-// GetDockerConfigPath returns the full path to the docker config store on the filesystem
-func (service *Service) GetDockerConfigPath() string {
-	return path.Join(service.fileStorePath, DockerConfigPath)
-}
-
 // RemoveDirectory removes a directory on the filesystem.
 func (service *Service) RemoveDirectory(directoryPath string) error {
 	return os.RemoveAll(directoryPath)
@@ -131,66 +106,6 @@ func (service *Service) GetStackProjectPath(stackIdentifier string) string {
 	return path.Join(service.fileStorePath, ComposeStorePath, stackIdentifier)
 }
 
-// Copy copies the file on fromFilePath to toFilePath
-// if toFilePath exists func will fail unless deleteIfExists is true
-func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error {
-	exists, err := service.FileExists(fromFilePath)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return errors.New("File doesn't exist")
-	}
-
-	finput, err := os.Open(fromFilePath)
-	if err != nil {
-		return err
-	}
-
-	defer finput.Close()
-
-	exists, err = service.FileExists(toFilePath)
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		if !deleteIfExists {
-			return errors.New("Destination file exists")
-		}
-
-		err := os.Remove(toFilePath)
-		if err != nil {
-			return err
-		}
-	}
-
-	foutput, err := os.Create(toFilePath)
-	if err != nil {
-		return err
-	}
-
-	defer foutput.Close()
-
-	buf := make([]byte, 1024)
-	for {
-		n, err := finput.Read(buf)
-		if err != nil && err != io.EOF {
-			return err
-		}
-		if n == 0 {
-			break
-		}
-
-		if _, err := foutput.Write(buf[:n]); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // StoreStackFileFromBytes creates a subfolder in the ComposeStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
@@ -288,7 +203,7 @@ func (service *Service) StoreTLSFileFromBytes(folder string, fileType portainer.
 	return path.Join(service.fileStorePath, tlsFilePath), nil
 }
 
-// GetPathForTLSFile returns the absolute path to a specific TLS file for an environment(endpoint).
+// GetPathForTLSFile returns the absolute path to a specific TLS file for an endpoint.
 func (service *Service) GetPathForTLSFile(folder string, fileType portainer.TLSFileType) (string, error) {
 	var fileName string
 	switch fileType {
@@ -364,7 +279,13 @@ func (service *Service) WriteJSONToFile(path string, content interface{}) error
 
 // FileExists checks for the existence of the specified file.
 func (service *Service) FileExists(filePath string) (bool, error) {
-	return FileExists(filePath)
+	if _, err := os.Stat(filePath); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
 }
 
 // KeyPairFilesExist checks for the existence of the key files.
@@ -584,88 +505,3 @@ func (service *Service) GetTemporaryPath() (string, error) {
 
 	return path.Join(service.fileStorePath, TempPath, uid.String()), nil
 }
-
-// GetDataStorePath returns path to data folder
-func (service *Service) GetDatastorePath() string {
-	return service.dataStorePath
-}
-
-func (service *Service) wrapFileStore(filepath string) string {
-	return path.Join(service.fileStorePath, filepath)
-}
-
-func defaultCertPathUnderFileStore() (string, string) {
-	certPath := path.Join(SSLCertPath, DefaultSSLCertFilename)
-	keyPath := path.Join(SSLCertPath, DefaultSSLKeyFilename)
-	return certPath, keyPath
-}
-
-// GetDefaultSSLCertsPath returns the ssl certs path
-func (service *Service) GetDefaultSSLCertsPath() (string, string) {
-	certPath, keyPath := defaultCertPathUnderFileStore()
-	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath)
-}
-
-// StoreSSLCertPair stores a ssl certificate pair
-func (service *Service) StoreSSLCertPair(cert, key []byte) (string, string, error) {
-	certPath, keyPath := defaultCertPathUnderFileStore()
-
-	r := bytes.NewReader(cert)
-	err := service.createFileInStore(certPath, r)
-	if err != nil {
-		return "", "", err
-	}
-
-	r = bytes.NewReader(key)
-	err = service.createFileInStore(keyPath, r)
-	if err != nil {
-		return "", "", err
-	}
-
-	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath), nil
-}
-
-// CopySSLCertPair copies a ssl certificate pair
-func (service *Service) CopySSLCertPair(certPath, keyPath string) (string, string, error) {
-	defCertPath, defKeyPath := service.GetDefaultSSLCertsPath()
-
-	err := service.Copy(certPath, defCertPath, false)
-	if err != nil {
-		return "", "", err
-	}
-
-	err = service.Copy(keyPath, defKeyPath, false)
-	if err != nil {
-		return "", "", err
-	}
-
-	return defCertPath, defKeyPath, nil
-}
-
-// FileExists checks for the existence of the specified file.
-func FileExists(filePath string) (bool, error) {
-	if _, err := os.Stat(filePath); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
-}
-
-func MoveDirectory(originalPath, newPath string) error {
-	if _, err := os.Stat(originalPath); err != nil {
-		return err
-	}
-
-	alreadyExists, err := FileExists(newPath)
-	if err != nil {
-		return err
-	}
-
-	if alreadyExists {
-		return errors.New("Target path already exists")
-	}
-
-	return os.Rename(originalPath, newPath)
-}

api/filesystem/filesystem_fileexists_test.go

@@ -1,55 +0,0 @@
-package filesystem
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_fileSystemService_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
-	service := createService(t)
-	testHelperFileExists_fileExists(t, service.FileExists)
-}
-
-func Test_fileSystemService_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
-	service := createService(t)
-	testHelperFileExists_fileNotExists(t, service.FileExists)
-}
-
-func Test_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) {
-	testHelperFileExists_fileExists(t, FileExists)
-}
-
-func Test_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) {
-	testHelperFileExists_fileNotExists(t, FileExists)
-}
-
-func testHelperFileExists_fileExists(t *testing.T, checker func(path string) (bool, error)) {
-	file, err := os.CreateTemp("", t.Name())
-	assert.NoError(t, err, "CreateTemp should not fail")
-
-	t.Cleanup(func() {
-		os.RemoveAll(file.Name())
-	})
-
-	exists, err := checker(file.Name())
-	assert.NoError(t, err, "FileExists should not fail")
-
-	assert.True(t, exists)
-}
-
-func testHelperFileExists_fileNotExists(t *testing.T, checker func(path string) (bool, error)) {
-	filePath := path.Join(os.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int()))
-
-	err := os.RemoveAll(filePath)
-	assert.NoError(t, err, "RemoveAll should not fail")
-
-	exists, err := checker(filePath)
-	assert.NoError(t, err, "FileExists should not fail")
-
-	assert.False(t, exists)
-}